@cleocode/core 2026.10.16 → 2026.10.17-rc.ci.687
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-provenance.json +2 -2
- package/dist/cloud/journal-tallies.d.ts +92 -0
- package/dist/cloud/journal-tallies.d.ts.map +1 -0
- package/dist/cloud/journal-tallies.js +172 -0
- package/dist/cloud/journal-tallies.js.map +1 -0
- package/dist/cloud/manifest-check.d.ts +12 -0
- package/dist/cloud/manifest-check.d.ts.map +1 -1
- package/dist/cloud/manifest-check.js +19 -0
- package/dist/cloud/manifest-check.js.map +1 -1
- package/dist/cloud/nexus-vault.d.ts +2 -0
- package/dist/cloud/nexus-vault.d.ts.map +1 -1
- package/dist/cloud/nexus-vault.js +438 -9
- package/dist/cloud/nexus-vault.js.map +1 -1
- package/dist/release/ci-evidence.d.ts +1 -1
- package/dist/release/ci-evidence.d.ts.map +1 -1
- package/dist/release/ci-evidence.js +12 -0
- package/dist/release/ci-evidence.js.map +1 -1
- package/dist/store/portable-bundle-scan.d.ts +19 -0
- package/dist/store/portable-bundle-scan.d.ts.map +1 -1
- package/dist/store/portable-bundle-scan.js +38 -0
- package/dist/store/portable-bundle-scan.js.map +1 -1
- package/dist/store/portable-bundle.d.ts +13 -1
- package/dist/store/portable-bundle.d.ts.map +1 -1
- package/dist/store/portable-bundle.js +13 -1
- package/dist/store/portable-bundle.js.map +1 -1
- package/dist/store/sync/inbox.d.ts +6 -0
- package/dist/store/sync/inbox.d.ts.map +1 -1
- package/dist/store/sync/inbox.js +9 -0
- package/dist/store/sync/inbox.js.map +1 -1
- package/dist/store/sync/segments.d.ts +10 -2
- package/dist/store/sync/segments.d.ts.map +1 -1
- package/dist/store/sync/segments.js +8 -2
- package/dist/store/sync/segments.js.map +1 -1
- package/dist/tasks/affected-packages.d.ts +19 -0
- package/dist/tasks/affected-packages.d.ts.map +1 -1
- package/dist/tasks/affected-packages.js +28 -0
- package/dist/tasks/affected-packages.js.map +1 -1
- package/dist/tasks/affected-scope.d.ts.map +1 -1
- package/dist/tasks/affected-scope.js +33 -8
- package/dist/tasks/affected-scope.js.map +1 -1
- package/dist/tasks/done-plan.d.ts.map +1 -1
- package/dist/tasks/done-plan.js +12 -4
- package/dist/tasks/done-plan.js.map +1 -1
- package/dist/tasks/evidence.d.ts.map +1 -1
- package/dist/tasks/evidence.js +65 -0
- package/dist/tasks/evidence.js.map +1 -1
- package/dist/tasks/tool-resolver.d.ts.map +1 -1
- package/dist/tasks/tool-resolver.js +4 -2
- package/dist/tasks/tool-resolver.js.map +1 -1
- package/package.json +18 -13
|
@@ -72,13 +72,17 @@ import { writeRestoreMarker } from '../store/restore-marker.js';
|
|
|
72
72
|
import { FIRST_OPEN_LOCK_SUFFIX } from '../store/sqlite.js';
|
|
73
73
|
import { isSyncFlagOn, setSyncFlag, UNRELEASED_FLAGS } from '../store/sync/flags.js';
|
|
74
74
|
import { completeGenesis, cutGenesisWithSnapshot, GenesisRacedError, GenesisTakenOverError, genesisCutOf, genesisPending, joinStream, readGenesisCut, } from '../store/sync/genesis.js';
|
|
75
|
+
import { UNRESOLVED_INBOX_STATUSES } from '../store/sync/inbox.js';
|
|
75
76
|
import { pullStream, readStreamCursor, } from '../store/sync/pull.js';
|
|
76
77
|
import { pushStream } from '../store/sync/push.js';
|
|
77
78
|
import { MergeKeyCollisionError, mergeKeyCollisions, reconcileCopy, } from '../store/sync/reconcile-copy.js';
|
|
78
|
-
import { replayPinOf } from '../store/sync/replay-pin.js';
|
|
79
|
+
import { migrationJournalHash, replayPinOf } from '../store/sync/replay-pin.js';
|
|
79
80
|
import { activeReplica, fileIdentity, readActiveReplicaId, rebindAfterVaultRestore, reconcileDue, } from '../store/sync/replica.js';
|
|
80
81
|
import { ReplicaRegistry } from '../store/sync/replica-registry.js';
|
|
82
|
+
import { hasTable } from '../store/sync/schema.js';
|
|
83
|
+
import { sealBacklog } from '../store/sync/seal-backlog.js';
|
|
81
84
|
import { sealPending } from '../store/sync/sealer.js';
|
|
85
|
+
import { unpushedSegments } from '../store/sync/segments.js';
|
|
82
86
|
import { firstBadTxnSignature, signTxn } from '../store/sync/txn-signing.js';
|
|
83
87
|
import { buildVaultManifest, carryMachineState, compareVaultManifests, emptyVaultTableHash, isVaultAnnotationKey, isVaultForkManifest, sameVaultManifest, VAULT_FILES_KEY, VAULT_FORK_KEY, VAULT_FORMAT_KEY, VAULT_GLOBAL_CONFIG_LOCAL_KEYS, VAULT_MANIFEST_FORMAT_VERSION, vaultDatabaseEntry, vaultDatabaseKey, vaultFileDigest, vaultFilesEntry, vaultForkEntry, vaultFormatEntry, vaultManifestFormat, vaultStripColumns, } from '../store/vault-manifest.js';
|
|
84
88
|
import { foreignWriterLeases, storeOpenElsewhere } from '../store/writer-lease.js';
|
|
@@ -86,8 +90,9 @@ import { autoSyncSuppressedBy } from './auto-sync.js';
|
|
|
86
90
|
import { deriveKey, uuidv7 } from './crypto.js';
|
|
87
91
|
import { NexusError } from './http.js';
|
|
88
92
|
import { cursorFromCheckpoint, initialPullCursor, Journal } from './journal.js';
|
|
93
|
+
import { journalTallies, readJournalWindow, settleShortfalls, } from './journal-tallies.js';
|
|
89
94
|
import { signerKeys } from './keys.js';
|
|
90
|
-
import { manifestVersion } from './manifest-check.js';
|
|
95
|
+
import { isSnapshotAuthoritativeKey, manifestVersion, schemaRises, } from './manifest-check.js';
|
|
91
96
|
import { canonicalGlobalReplicaBinder } from './nexus-attach.js';
|
|
92
97
|
import { NexusAccountError } from './nexus-auth.js';
|
|
93
98
|
import { nexusQueryPath } from './nexus-cloud.js';
|
|
@@ -365,7 +370,7 @@ async function replaySegments(journal, cursor, signers) {
|
|
|
365
370
|
* What works on a stream the change journal writes (its head snapshot is
|
|
366
371
|
* checkpoint/v3), for every message that would otherwise say "push" (T13034).
|
|
367
372
|
*/
|
|
368
|
-
const JOURNAL_STREAM_REMEDY = 'on this stream local changes travel through the change journal (`sync.push`),
|
|
373
|
+
const JOURNAL_STREAM_REMEDY = 'on this stream local changes travel through the change journal (`sync.push`), and with it on a vault push checkpoints the journal (T12999); a store that has not joined it joins with `cleo sync enable push`, which keeps its own rows and merges them into the stream (T13466); `cleo cloud pull`, `cleo cloud restore` and `cleo cloud verify` still work';
|
|
369
374
|
/** Whether a snapshot is checkpoint/v3: the change journal writes its stream (T13034). */
|
|
370
375
|
function isJournalSnapshot(cp) {
|
|
371
376
|
return cp !== null && cp !== undefined && manifestVersion(cp.manifest) === 3;
|
|
@@ -576,6 +581,10 @@ function compareWithSynced(local, snapshot, git, changes) {
|
|
|
576
581
|
const JOURNAL_SNAPSHOT_CARRIED = new Set([
|
|
577
582
|
'_sync_row_meta',
|
|
578
583
|
'_sync_field_leave',
|
|
584
|
+
// The checkpoint's unresolved transactions travel in its bundle (§2.11 §6; the
|
|
585
|
+
// export keeps only those rows): a restore pulls on after coversSeq, so a
|
|
586
|
+
// pending one would never arrive again (T12999).
|
|
587
|
+
'_sync_inbox',
|
|
579
588
|
]);
|
|
580
589
|
const VAULT_CLEARED_JOURNAL_TABLES = [
|
|
581
590
|
'_sync_capture',
|
|
@@ -598,8 +607,12 @@ const VAULT_CLEARED_JOURNAL_TABLES = [
|
|
|
598
607
|
* global snapshot carries its credential cells sealed under the account key
|
|
599
608
|
* when `seal` is given, so a device that restores it can use them (T13530).
|
|
600
609
|
*/
|
|
601
|
-
async function exportVaultBundle(t, outputPath, label, tracked, seal) {
|
|
610
|
+
async function exportVaultBundle(t, outputPath, label, tracked, seal, inspect) {
|
|
602
611
|
await exportPortableBundle({
|
|
612
|
+
...(inspect ? { inspectSnapshot: inspect } : {}),
|
|
613
|
+
// Only the unresolved receive rows travel: a journal checkpoint's pending
|
|
614
|
+
// transactions, never the whole history (T12999).
|
|
615
|
+
keepRows: { _sync_inbox: { column: 'status', values: UNRESOLVED_INBOX_STATUSES } },
|
|
603
616
|
scope: t.scope === 'global' ? 'global' : 'project',
|
|
604
617
|
...(t.scope === 'project' ? { projectRoot: t.storeRoot } : {}),
|
|
605
618
|
...(t.scope === 'global'
|
|
@@ -728,15 +741,15 @@ async function snapshotManifest(extractDir, manifest, t) {
|
|
|
728
741
|
return { vault, dbPath, files: digests };
|
|
729
742
|
}
|
|
730
743
|
/** Export and extract the store into `work`, returning its bundle and vault manifest. */
|
|
731
|
-
async function exportAndRead(t, work, label, tracked, seal) {
|
|
744
|
+
async function exportAndRead(t, work, label, tracked, seal, inspect) {
|
|
732
745
|
const bundlePath = path.join(work, 'snapshot.cleobundle.tar.gz');
|
|
733
|
-
await exportVaultBundle(t, bundlePath, label, tracked, seal);
|
|
746
|
+
await exportVaultBundle(t, bundlePath, label, tracked, seal, inspect);
|
|
734
747
|
const extractDir = path.join(work, 'x');
|
|
735
748
|
fs.mkdirSync(extractDir);
|
|
736
749
|
await tarExtract({ file: bundlePath, cwd: extractDir });
|
|
737
750
|
const bundleManifest = JSON.parse(fs.readFileSync(path.join(extractDir, 'manifest.json'), 'utf8'));
|
|
738
|
-
const { vault, files } = await snapshotManifest(extractDir, bundleManifest, t);
|
|
739
|
-
return { bundlePath, vault, gitTracked: snapshotGitTracked(bundleManifest, t), files };
|
|
751
|
+
const { vault, files, dbPath } = await snapshotManifest(extractDir, bundleManifest, t);
|
|
752
|
+
return { bundlePath, vault, gitTracked: snapshotGitTracked(bundleManifest, t), files, dbPath };
|
|
740
753
|
}
|
|
741
754
|
/** The paths a project snapshot's bundle marks as git-tracked; a global one has none (T13019, T13038). */
|
|
742
755
|
function snapshotGitTracked(manifest, t) {
|
|
@@ -928,6 +941,260 @@ async function appendCheckpoint(a) {
|
|
|
928
941
|
}
|
|
929
942
|
return { cp, deltaSegmentSeq, cursor: pushedWith };
|
|
930
943
|
}
|
|
944
|
+
/** Whether this store's `sync.push` is on: its vault push checkpoints the journal (T12999). */
|
|
945
|
+
async function journalPushOn(t) {
|
|
946
|
+
const { openNativeDatabase } = await import('../store/sqlite-native.js');
|
|
947
|
+
const db = openNativeDatabase(t.dbPath, { readonly: true, enableWal: false });
|
|
948
|
+
try {
|
|
949
|
+
return isSyncFlagOn(db, 'sync.push');
|
|
950
|
+
}
|
|
951
|
+
finally {
|
|
952
|
+
db.close();
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
/** A ref and deltas in the wire's normal form: only tables with a non-zero count. */
|
|
956
|
+
function normalRefDeltas(list) {
|
|
957
|
+
const out = [];
|
|
958
|
+
for (const v of list) {
|
|
959
|
+
const deltas = Object.fromEntries(Object.entries(v.deltas).filter(([, c]) => c.created > 0 || c.deleted > 0));
|
|
960
|
+
if (Object.keys(deltas).length > 0)
|
|
961
|
+
out.push({ ref: v.ref, deltas });
|
|
962
|
+
}
|
|
963
|
+
return out;
|
|
964
|
+
}
|
|
965
|
+
/**
|
|
966
|
+
* What a journal checkpoint's snapshot holds of the journal, read from the
|
|
967
|
+
* snapshot as taken (before the export clears the local journal): its pull
|
|
968
|
+
* position, and what it holds that the stream does not have yet (live
|
|
969
|
+
* captures, sealed transactions not packed, segments not pushed). A
|
|
970
|
+
* snapshot whose store moved after the sync is not the sealed state.
|
|
971
|
+
*/
|
|
972
|
+
async function snapshotJournalState(snapshotPath, stream, replicaId) {
|
|
973
|
+
const { openNativeDatabase } = await import('../store/sqlite-native.js');
|
|
974
|
+
const db = openNativeDatabase(snapshotPath, { readonly: true, enableWal: false });
|
|
975
|
+
try {
|
|
976
|
+
const unpacked = hasTable(db, '_sync_txn')
|
|
977
|
+
? Number(db
|
|
978
|
+
.prepare("SELECT count(*) AS n FROM _sync_txn WHERE state = 'sealed' AND replica = ?")
|
|
979
|
+
.get(replicaId).n)
|
|
980
|
+
: 0;
|
|
981
|
+
return {
|
|
982
|
+
cursor: readStreamCursor(db, stream),
|
|
983
|
+
unsent: sealBacklog(db).live + unpacked + unpushedSegments(db, stream, replicaId).length,
|
|
984
|
+
};
|
|
985
|
+
}
|
|
986
|
+
finally {
|
|
987
|
+
db.close();
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
/**
|
|
991
|
+
* `cleo cloud push` on a store that writes its stream's change journal
|
|
992
|
+
* (`sync.push` on; T12999, journal spec §2.11): seal, push and pull the
|
|
993
|
+
* journal first, then snapshot the store as a checkpoint/v3 at the journal's
|
|
994
|
+
* sealed high-water mark (the snapshot's pull position). Its counts reconcile
|
|
995
|
+
* through the journal's tallies (pending and voided transactions of the
|
|
996
|
+
* window), never a synthetic delta segment: the segments already declare
|
|
997
|
+
* those rows, and a delta would count them twice. The vault-only entries
|
|
998
|
+
* (files, other databases), which the journal never carries, are
|
|
999
|
+
* snapshot-authoritative on the server. A write that lands between the sync
|
|
1000
|
+
* and the snapshot makes the snapshot more than the sealed state: the push
|
|
1001
|
+
* syncs and snapshots once more, then refuses.
|
|
1002
|
+
*/
|
|
1003
|
+
async function pushJournalCheckpoint(conn, key, opts, scope, warnings) {
|
|
1004
|
+
for (let attempt = 0;; attempt++) {
|
|
1005
|
+
const synced0 = await syncOneStream({ conn, key }, opts, scope);
|
|
1006
|
+
if (synced0.status !== 'synced') {
|
|
1007
|
+
throw vaultError('E_NEXUS_SYNC_REFUSED', `the change journal did not sync before the snapshot (${synced0.status}${synced0.refused ? `: ${synced0.refused}` : ''})`, 'nothing was snapshotted; run `cleo cloud sync`, then push again');
|
|
1008
|
+
}
|
|
1009
|
+
// Resolved again: a pull that rebound at head moved the store to its successor replica.
|
|
1010
|
+
const t = await resolveTarget(conn, key.masterKey, { ...opts, scope }, 'push');
|
|
1011
|
+
const replicaId = t.replicaId;
|
|
1012
|
+
if (!replicaId) {
|
|
1013
|
+
throw vaultError('E_NEXUS_VAULT_NOT_LINKED', 'this copy of the project is not attached from this device', 'run `cleo project link`');
|
|
1014
|
+
}
|
|
1015
|
+
const journal = journalFor(conn, t);
|
|
1016
|
+
const head = await streamHead(conn, t.streamId);
|
|
1017
|
+
const checkpoints = head.headCheckpointId ? await listCheckpoints(conn, t.streamId) : [];
|
|
1018
|
+
const parent = checkpoints.find((c) => c.checkpointId === head.headCheckpointId) ?? null;
|
|
1019
|
+
if (parent === null || !isJournalSnapshot(parent)) {
|
|
1020
|
+
throw vaultError('E_NEXUS_VAULT_REFUSED', `${t.streamId}'s head snapshot ${head.headCheckpointId ?? '(none)'} is not a journal checkpoint, but this store writes the stream's change journal`);
|
|
1021
|
+
}
|
|
1022
|
+
journal.verifyCheckpoint(parent, key.signers);
|
|
1023
|
+
const ahead = aheadOfThisCleo(t, Math.max(head.maxSchemaVersion ?? 0, parent.manifest.schemaVersion), parent);
|
|
1024
|
+
if (ahead)
|
|
1025
|
+
throw ahead;
|
|
1026
|
+
const { state: synced } = syncedState(conn, t, checkpoints, head.headCheckpointId);
|
|
1027
|
+
// Another device's snapshot since this store last synced: journal movement
|
|
1028
|
+
// is fine (the sync just pulled it), but its files and databases travel only
|
|
1029
|
+
// in snapshots, and this one would drop them. Only `--force` pushes over
|
|
1030
|
+
// them, labelled a fork on the checkpoint, as a vault push does (T13007).
|
|
1031
|
+
const lastSynced = checkpoints.find((c) => c.checkpointId === synced?.lastCheckpointId);
|
|
1032
|
+
const vaultOnlyMoved = parent.checkpointId !== lastSynced?.checkpointId &&
|
|
1033
|
+
(lastSynced === undefined ||
|
|
1034
|
+
[
|
|
1035
|
+
...new Set([
|
|
1036
|
+
...Object.keys(parent.manifest.tables),
|
|
1037
|
+
...Object.keys(lastSynced.manifest.tables),
|
|
1038
|
+
]),
|
|
1039
|
+
]
|
|
1040
|
+
.filter(isSnapshotAuthoritativeKey)
|
|
1041
|
+
.some((k) => parent.manifest.tables[k]?.hash !== lastSynced.manifest.tables[k]?.hash ||
|
|
1042
|
+
parent.manifest.tables[k]?.rows !== lastSynced.manifest.tables[k]?.rows));
|
|
1043
|
+
if (vaultOnlyMoved && opts.force !== true) {
|
|
1044
|
+
throw vaultError('E_NEXUS_VAULT_BEHIND', `another device pushed snapshot ${parent.checkpointId} after this machine's last sync, with files or databases this machine does not have`, "run `cleo cloud pull` first (on a store that writes the change journal it brings in only the snapshot's files and databases), then push again; or `--force` to push this store as a labelled fork");
|
|
1045
|
+
}
|
|
1046
|
+
const git = gitTracking(t);
|
|
1047
|
+
if (git.mode === 'unknown' && attempt === 0)
|
|
1048
|
+
warnings.push(gitUnknownWarning(t));
|
|
1049
|
+
const work = tempDir('cleo-vault-push-');
|
|
1050
|
+
try {
|
|
1051
|
+
let snap = null;
|
|
1052
|
+
const { bundlePath, vault, gitTracked, files } = await exportAndRead(t, work, `cloud-vault-${scope}`, trackedForPush(t, git, synced), { masterKey: key.masterKey, warn: (w) => warnings.push(w) }, async (snapshotPath) => {
|
|
1053
|
+
const state = await snapshotJournalState(snapshotPath, t.streamId, replicaId);
|
|
1054
|
+
let read = null;
|
|
1055
|
+
if (state.cursor !== null &&
|
|
1056
|
+
state.unsent === 0 &&
|
|
1057
|
+
parent.coversSeq <= state.cursor.after) {
|
|
1058
|
+
const window = await readJournalWindow(journal, parent, key.signers, state.cursor.after);
|
|
1059
|
+
const { openNativeDatabase } = await import('../store/sqlite-native.js');
|
|
1060
|
+
const snapDb = openNativeDatabase(snapshotPath, { readonly: true, enableWal: false });
|
|
1061
|
+
try {
|
|
1062
|
+
read = {
|
|
1063
|
+
window,
|
|
1064
|
+
tallies: journalTallies(snapDb, t.streamId, parent, window),
|
|
1065
|
+
// The journal in force from each rise: the snapshot's, which replays at the newest schema.
|
|
1066
|
+
pin: replayPinOf(snapDb, schemaRises(parent.manifest.schemaVersion, window.schemaVersions).map((r) => ({
|
|
1067
|
+
...r,
|
|
1068
|
+
journal: migrationJournalHash(snapDb),
|
|
1069
|
+
}))),
|
|
1070
|
+
};
|
|
1071
|
+
}
|
|
1072
|
+
finally {
|
|
1073
|
+
snapDb.close();
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
snap = { ...state, read };
|
|
1077
|
+
});
|
|
1078
|
+
// Read through a const: TypeScript does not see the callback assign it.
|
|
1079
|
+
const taken = snap;
|
|
1080
|
+
const cursor = taken?.cursor ?? null;
|
|
1081
|
+
if (taken === null || cursor === null || taken.unsent > 0) {
|
|
1082
|
+
if (attempt === 0)
|
|
1083
|
+
continue;
|
|
1084
|
+
throw vaultError('E_NEXUS_SYNC_REFUSED', `${t.streamId}: this store changed while it was snapshotted (${taken === null || cursor === null ? 'no pull position' : `${taken.unsent} change(s) not on the stream yet`}), so the snapshot is not the journal's sealed state`, 'nothing was written; push again once the store is quiet');
|
|
1085
|
+
}
|
|
1086
|
+
if (parent.coversSeq > cursor.after) {
|
|
1087
|
+
throw vaultError('E_NEXUS_VAULT_BEHIND', `another device checkpointed ${t.streamId} at seq ${parent.coversSeq}, past this store's pull position ${cursor.after}`, 'run `cleo cloud sync`, then push again');
|
|
1088
|
+
}
|
|
1089
|
+
if (taken?.read == null) {
|
|
1090
|
+
// @sync-invariant none:local-only unreachable: a clean snapshot at or past the parent was read
|
|
1091
|
+
throw new Error('pushJournalCheckpoint: the snapshot was not read');
|
|
1092
|
+
}
|
|
1093
|
+
const { window, pin } = taken.read;
|
|
1094
|
+
const names = await deviceNames(conn);
|
|
1095
|
+
if (window.txns.length === 0 &&
|
|
1096
|
+
compareVaultManifests(vault, comparable(parent.manifest, git)).every((r) => r.match)) {
|
|
1097
|
+
saveSynced(conn, t, parent, { gitTracked, files: syncFiles(files) });
|
|
1098
|
+
warnings.push(...conn.state.drainWarnings());
|
|
1099
|
+
return {
|
|
1100
|
+
apiUrl: conn.apiUrl,
|
|
1101
|
+
scope,
|
|
1102
|
+
streamId: t.streamId,
|
|
1103
|
+
status: 'up-to-date',
|
|
1104
|
+
snapshot: snapshotOf(parent, names),
|
|
1105
|
+
parentCheckpointId: parent.parentCheckpointId,
|
|
1106
|
+
deltaSegmentSeq: null,
|
|
1107
|
+
lease: null,
|
|
1108
|
+
forked: false,
|
|
1109
|
+
warnings,
|
|
1110
|
+
};
|
|
1111
|
+
}
|
|
1112
|
+
const tables = {
|
|
1113
|
+
...vault.tables,
|
|
1114
|
+
[VAULT_FORMAT_KEY]: vaultFormatEntry(hashKeyOf(t.dataKey)),
|
|
1115
|
+
};
|
|
1116
|
+
for (const table of Object.keys(parent.manifest.tables)) {
|
|
1117
|
+
if (isVaultAnnotationKey(table))
|
|
1118
|
+
continue;
|
|
1119
|
+
tables[table] ??= { rows: 0, hash: buildEmptyHash(t, table) };
|
|
1120
|
+
}
|
|
1121
|
+
const local = {};
|
|
1122
|
+
for (const [table, e] of Object.entries(tables))
|
|
1123
|
+
local[table] = e.rows;
|
|
1124
|
+
const keys = [...new Set([...Object.keys(tables), ...Object.keys(parent.manifest.tables)])]
|
|
1125
|
+
.filter((k) => !isVaultAnnotationKey(k))
|
|
1126
|
+
.sort();
|
|
1127
|
+
const { tallies, unexplained } = settleShortfalls(taken.read.tallies, local, keys.filter((k) => !isSnapshotAuthoritativeKey(k)));
|
|
1128
|
+
if (unexplained.length > 0) {
|
|
1129
|
+
throw vaultError('E_NEXUS_VAULT_REFUSED', `this store's rows do not reconcile with ${t.streamId}'s change journal (${unexplained.slice(0, 8).join(', ')}): a write landed outside the journal`, 'nothing was written; run `cleo cloud sync`, then push again');
|
|
1130
|
+
}
|
|
1131
|
+
const vaultOnlyChanged = keys.some((k) => isSnapshotAuthoritativeKey(k) && local[k] !== (parent.manifest.tables[k]?.rows ?? 0));
|
|
1132
|
+
const acquired = await acquireLease(conn, t, opts.force === true, opts.force === true ? 'cleo cloud push --force' : 'cleo cloud push');
|
|
1133
|
+
const forked = vaultOnlyMoved || acquired.forked;
|
|
1134
|
+
if (forked)
|
|
1135
|
+
tables[VAULT_FORK_KEY] = vaultForkEntry(hashKeyOf(t.dataKey), parent.checkpointId);
|
|
1136
|
+
try {
|
|
1137
|
+
conn.state.markPushInFlight(conn.apiUrl, conn.userId, t.streamId, t.storeRoot, parent.checkpointId, { gitTracked: t.scope === 'project' ? gitTracked : [], files: syncFiles(files) });
|
|
1138
|
+
let cp;
|
|
1139
|
+
try {
|
|
1140
|
+
cp = await journal.pushCheckpoint({
|
|
1141
|
+
bundle: fs.readFileSync(bundlePath),
|
|
1142
|
+
manifest: {
|
|
1143
|
+
schemaVersion: SYNC_SCHEMA_VERSION,
|
|
1144
|
+
tables,
|
|
1145
|
+
pending: tallies.pending,
|
|
1146
|
+
voided: normalRefDeltas(tallies.voided),
|
|
1147
|
+
revived: [],
|
|
1148
|
+
pruned: {},
|
|
1149
|
+
replayPin: pin,
|
|
1150
|
+
},
|
|
1151
|
+
cursor: {
|
|
1152
|
+
after: cursor.after,
|
|
1153
|
+
knowsAllReplicas: cursor.knowsAllReplicas,
|
|
1154
|
+
replicas: { ...cursor.replicas },
|
|
1155
|
+
},
|
|
1156
|
+
parentCheckpointId: parent.checkpointId,
|
|
1157
|
+
});
|
|
1158
|
+
}
|
|
1159
|
+
catch (err) {
|
|
1160
|
+
if (err instanceof NexusError && err.code === 'E_LINEAGE') {
|
|
1161
|
+
throw vaultError('E_NEXUS_VAULT_BEHIND', 'another device pushed a snapshot while this one was uploading', 'run `cleo cloud sync`, then push again');
|
|
1162
|
+
}
|
|
1163
|
+
if (err instanceof NexusError &&
|
|
1164
|
+
(err.code === 'E_REGRESSION' || err.code === 'E_VALIDATION')) {
|
|
1165
|
+
throw vaultError('E_NEXUS_VAULT_REFUSED', `the server refused the snapshot: ${err.message}`, vaultOnlyChanged
|
|
1166
|
+
? 'nothing was written; this snapshot changed files or databases the journal never carries, and a Cleo Nexus server without snapshot-authoritative keys (T12999) reconciles them against the journal: upgrade the server, then push again (if another device wrote during the push, `cleo cloud sync` first)'
|
|
1167
|
+
: 'nothing was written; another device may have written during the push: run `cleo cloud sync`, then push again');
|
|
1168
|
+
}
|
|
1169
|
+
throw err;
|
|
1170
|
+
}
|
|
1171
|
+
saveSynced(conn, t, cp, { gitTracked, files: syncFiles(files) });
|
|
1172
|
+
if (opts.hold !== true)
|
|
1173
|
+
await releaseLeaseQuietly(conn, t);
|
|
1174
|
+
warnings.push(...conn.state.drainWarnings());
|
|
1175
|
+
return {
|
|
1176
|
+
apiUrl: conn.apiUrl,
|
|
1177
|
+
scope,
|
|
1178
|
+
streamId: t.streamId,
|
|
1179
|
+
status: 'pushed',
|
|
1180
|
+
snapshot: snapshotOf(cp, names),
|
|
1181
|
+
parentCheckpointId: cp.parentCheckpointId,
|
|
1182
|
+
deltaSegmentSeq: null,
|
|
1183
|
+
lease: opts.hold === true ? acquired.lease : null,
|
|
1184
|
+
forked,
|
|
1185
|
+
warnings,
|
|
1186
|
+
};
|
|
1187
|
+
}
|
|
1188
|
+
catch (err) {
|
|
1189
|
+
await releaseLeaseQuietly(conn, t);
|
|
1190
|
+
throw err;
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
finally {
|
|
1194
|
+
fs.rmSync(work, { recursive: true, force: true });
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
931
1198
|
async function pushNexusVaultImpl(opts = {}) {
|
|
932
1199
|
const conn = await connectNexusVault(opts);
|
|
933
1200
|
const key = await unlockNexusAccountKey(conn);
|
|
@@ -941,6 +1208,11 @@ async function pushNexusVaultImpl(opts = {}) {
|
|
|
941
1208
|
// Past its genesis cut, this store's changes travel through the change
|
|
942
1209
|
// journal: a vault delta would count them twice (T12343 S4-1b).
|
|
943
1210
|
const cut = await readGenesisCut(t.dbPath, t.streamId);
|
|
1211
|
+
// With sync.push on (and released), the snapshot checkpoints the journal (T12999); off, it stays refused.
|
|
1212
|
+
const pushReleased = !UNRELEASED_FLAGS.has('sync.push') || opts.allowUnreleased === true;
|
|
1213
|
+
if (cut !== undefined && pushReleased && (await journalPushOn(t))) {
|
|
1214
|
+
return pushJournalCheckpoint(conn, key, opts, scope, warnings);
|
|
1215
|
+
}
|
|
944
1216
|
if (cut !== undefined) {
|
|
945
1217
|
throw streamUpgradedError(`this store started ${t.streamId}'s change journal (genesis cut at capture ${cut}); its changes are pushed as journal segments, not vault snapshots`, `nothing was written; ${JOURNAL_STREAM_REMEDY}`);
|
|
946
1218
|
}
|
|
@@ -2364,6 +2636,98 @@ function keepGlobalConfigLocalKeys(configPath, local) {
|
|
|
2364
2636
|
fs.renameSync(tmp, configPath);
|
|
2365
2637
|
}
|
|
2366
2638
|
/** `cleo cloud pull` and `cleo cloud restore`. */
|
|
2639
|
+
/**
|
|
2640
|
+
* `cleo cloud pull` on a store that writes its stream's change journal
|
|
2641
|
+
* (T12999): its journal tables are already on the stream (the journal carries
|
|
2642
|
+
* them both ways), so only what snapshots alone carry is pulled: the head's
|
|
2643
|
+
* plain files and other databases (the snapshot-authoritative entries).
|
|
2644
|
+
* `cleo.db` is never touched. Refused when this store's own files or databases
|
|
2645
|
+
* changed since its last sync; a store that never synced keeps the files it
|
|
2646
|
+
* has (T13020). The head is then recorded as synced, so a journal push
|
|
2647
|
+
* follows it.
|
|
2648
|
+
*/
|
|
2649
|
+
async function pullVaultOnlyEntries(a) {
|
|
2650
|
+
const { conn, key, t, journal, target, last, git, synced, warnings } = a;
|
|
2651
|
+
const read = await localManifest(t, git, untrackedComparison(t, git, synced));
|
|
2652
|
+
if (read !== null && last !== undefined) {
|
|
2653
|
+
const fileChanges = fileChangesSinceSync(read.files, t, git, synced);
|
|
2654
|
+
const changed = compareWithSynced(read.manifest, last.manifest, git, fileChanges)
|
|
2655
|
+
.filter((d) => isSnapshotAuthoritativeKey(d.table) && !d.match)
|
|
2656
|
+
.map((d) => d.table === VAULT_FILES_KEY && fileChanges !== null && fileChanges.length > 0
|
|
2657
|
+
? `${d.table}: ${fileChanges.slice(0, 5).join(', ')}${fileChanges.length > 5 ? ', …' : ''}`
|
|
2658
|
+
: d.table);
|
|
2659
|
+
if (changed.length > 0) {
|
|
2660
|
+
throw vaultError('E_NEXUS_VAULT_LOCAL_CHANGES', `this store's files or databases changed since its last cloud sync (${changed.slice(0, 8).join(', ')}); pulling the snapshot's would overwrite them`, 'run `cleo cloud push --force` to keep them (a labelled fork), or `cleo cloud pull --force` to take the cloud copy (a safety backup is taken first)');
|
|
2661
|
+
}
|
|
2662
|
+
}
|
|
2663
|
+
const restored = await journal.restoreCheckpoint(target.checkpointId, key.signers);
|
|
2664
|
+
const work = tempDir('cleo-vault-pull-');
|
|
2665
|
+
try {
|
|
2666
|
+
const bundlePath = path.join(work, 'snapshot.cleobundle.tar.gz');
|
|
2667
|
+
fs.writeFileSync(bundlePath, restored.bundle);
|
|
2668
|
+
const staging = path.join(work, 'staging');
|
|
2669
|
+
fs.mkdirSync(staging);
|
|
2670
|
+
const { extractDir, manifest } = await extractPortableBundle(bundlePath, staging);
|
|
2671
|
+
const staged = await snapshotManifest(extractDir, manifest, t);
|
|
2672
|
+
const diff = compareVaultManifests(staged.vault, restored.checkpoint.manifest).filter((d) => !d.match);
|
|
2673
|
+
if (diff.length > 0) {
|
|
2674
|
+
throw vaultError('E_NEXUS_VAULT_VERIFY_FAILED', `snapshot ${target.checkpointId} does not match its manifest (${diff.map((d) => d.table).join(', ')}); nothing was pulled`);
|
|
2675
|
+
}
|
|
2676
|
+
await assertStoreQuiescent(t);
|
|
2677
|
+
const safetyBackup = await exportSafetyBundle(t, 'pre-restore', 'cloud-vault-pre-restore');
|
|
2678
|
+
const section = t.scope === 'global' ? manifest.global?.home : manifest.projects[0];
|
|
2679
|
+
const { sectionRoot, primaryRel } = sectionScan(t);
|
|
2680
|
+
const globalConfig = t.scope === 'global' ? path.join(t.storeRoot, 'config.json') : null;
|
|
2681
|
+
const ownConfigKeys = globalConfig !== null && fs.existsSync(globalConfig)
|
|
2682
|
+
? readGlobalConfigLocalKeys(globalConfig)
|
|
2683
|
+
: new Map();
|
|
2684
|
+
// Never-synced local copies stay (T13020); machine-local and git-tracked paths are never placed (T13005, T13019).
|
|
2685
|
+
const placeable = (rel, gitTracked) => machineLocalFile(rel) === null &&
|
|
2686
|
+
git.mode !== 'unknown' &&
|
|
2687
|
+
!(git.mode !== 'none' && (gitTracked || git.tracked.has(rel))) &&
|
|
2688
|
+
(last !== undefined || !fs.existsSync(path.join(sectionRoot, rel)));
|
|
2689
|
+
for (const f of section?.files ?? []) {
|
|
2690
|
+
if (!placeable(f.relPath, t.scope === 'project' && f.gitTracked === true))
|
|
2691
|
+
continue;
|
|
2692
|
+
const dst = path.join(sectionRoot, f.relPath);
|
|
2693
|
+
fs.mkdirSync(path.dirname(dst), { recursive: true });
|
|
2694
|
+
replaceFile(dst, { copyFrom: path.join(extractDir, f.bundlePath) });
|
|
2695
|
+
}
|
|
2696
|
+
for (const d of section?.databases ?? []) {
|
|
2697
|
+
// The journal carries cleo.db's tables: never replaced here.
|
|
2698
|
+
if (d.relPath === primaryRel || !placeable(d.relPath, false))
|
|
2699
|
+
continue;
|
|
2700
|
+
const dst = path.join(sectionRoot, d.relPath);
|
|
2701
|
+
fs.mkdirSync(path.dirname(dst), { recursive: true });
|
|
2702
|
+
for (const suffix of ['-wal', '-shm'])
|
|
2703
|
+
fs.rmSync(`${dst}${suffix}`, { force: true });
|
|
2704
|
+
replaceFile(dst, { copyFrom: path.join(extractDir, d.bundlePath) });
|
|
2705
|
+
}
|
|
2706
|
+
if (globalConfig !== null && fs.existsSync(globalConfig)) {
|
|
2707
|
+
keepGlobalConfigLocalKeys(globalConfig, ownConfigKeys);
|
|
2708
|
+
}
|
|
2709
|
+
saveSynced(conn, t, restored.checkpoint, {
|
|
2710
|
+
gitTracked: snapshotGitTracked(manifest, t),
|
|
2711
|
+
files: syncFiles(staged.files),
|
|
2712
|
+
});
|
|
2713
|
+
return {
|
|
2714
|
+
apiUrl: conn.apiUrl,
|
|
2715
|
+
scope: t.scope,
|
|
2716
|
+
streamId: t.streamId,
|
|
2717
|
+
status: 'restored',
|
|
2718
|
+
snapshot: snapshotOf(restored.checkpoint, a.names),
|
|
2719
|
+
target: t.storeRoot,
|
|
2720
|
+
verified: true,
|
|
2721
|
+
tables: Object.keys(restored.checkpoint.manifest.tables).filter((k) => !isVaultAnnotationKey(k)).length,
|
|
2722
|
+
safetyBackup,
|
|
2723
|
+
replica: null,
|
|
2724
|
+
warnings: [...warnings, ...conn.state.drainWarnings()],
|
|
2725
|
+
};
|
|
2726
|
+
}
|
|
2727
|
+
finally {
|
|
2728
|
+
fs.rmSync(work, { recursive: true, force: true });
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2367
2731
|
async function restoreNexusVaultImpl(opts) {
|
|
2368
2732
|
const conn = await connectNexusVault(opts);
|
|
2369
2733
|
// Read-only: a restore never mints, escrows or certifies (T12974).
|
|
@@ -2444,6 +2808,27 @@ async function restoreNexusVaultImpl(opts) {
|
|
|
2444
2808
|
const hasLocal = fs.existsSync(t.dbPath);
|
|
2445
2809
|
const force = opts.force === true;
|
|
2446
2810
|
const last = synced ? trusted.find((c) => c.checkpointId === synced.lastCheckpointId) : undefined;
|
|
2811
|
+
// A store that writes the journal pulls only what snapshots alone carry (T12999).
|
|
2812
|
+
if (opts.mode === 'pull' &&
|
|
2813
|
+
!force &&
|
|
2814
|
+
opts.checkpointId === undefined &&
|
|
2815
|
+
hasLocal &&
|
|
2816
|
+
targetCp !== undefined &&
|
|
2817
|
+
isJournalSnapshot(targetCp) &&
|
|
2818
|
+
(await readGenesisCut(t.dbPath, t.streamId)) !== undefined) {
|
|
2819
|
+
return pullVaultOnlyEntries({
|
|
2820
|
+
conn,
|
|
2821
|
+
key,
|
|
2822
|
+
t,
|
|
2823
|
+
journal,
|
|
2824
|
+
target: targetCp,
|
|
2825
|
+
last,
|
|
2826
|
+
git,
|
|
2827
|
+
synced,
|
|
2828
|
+
warnings,
|
|
2829
|
+
names,
|
|
2830
|
+
});
|
|
2831
|
+
}
|
|
2447
2832
|
const local = hasLocal && !force ? await localManifest(t, git, untrackedComparison(t, git, synced)) : null;
|
|
2448
2833
|
if (local !== null) {
|
|
2449
2834
|
const hasRows = Object.entries(local.manifest.tables).some(([name, x]) => x.rows > 0 && name !== VAULT_FILES_KEY);
|
|
@@ -2848,6 +3233,31 @@ async function deepVerify(journal, signers, headCp, snapshots) {
|
|
|
2848
3233
|
segments: { from, checked: count, ok: problem === null, problem },
|
|
2849
3234
|
};
|
|
2850
3235
|
}
|
|
3236
|
+
/**
|
|
3237
|
+
* The cloud side of `cleo cloud verify` for a store that writes the stream's
|
|
3238
|
+
* change journal (T12999; journal spec §2.11): the rows the journal's tallies
|
|
3239
|
+
* leave past the head checkpoint (counts only: tallies carry no hashes),
|
|
3240
|
+
* whether the stream holds segments this store has not pulled, and whether
|
|
3241
|
+
* the store holds changes it has not pushed.
|
|
3242
|
+
*/
|
|
3243
|
+
async function journalVerifySide(journal, signers, t, headCp) {
|
|
3244
|
+
const window = await readJournalWindow(journal, headCp, signers);
|
|
3245
|
+
const { openNativeDatabase } = await import('../store/sqlite-native.js');
|
|
3246
|
+
const db = openNativeDatabase(t.dbPath, { readonly: true, enableWal: false });
|
|
3247
|
+
try {
|
|
3248
|
+
const { rows } = journalTallies(db, t.streamId, headCp, window);
|
|
3249
|
+
const after = readStreamCursor(db, t.streamId)?.after ?? headCp.coversSeq;
|
|
3250
|
+
const unpushed = t.replicaId ? unpushedSegments(db, t.streamId, t.replicaId).length : 0;
|
|
3251
|
+
return {
|
|
3252
|
+
rows,
|
|
3253
|
+
cloudAdvanced: after < window.head,
|
|
3254
|
+
localChanged: unpushed > 0 || sealBacklog(db).live > 0,
|
|
3255
|
+
};
|
|
3256
|
+
}
|
|
3257
|
+
finally {
|
|
3258
|
+
db.close();
|
|
3259
|
+
}
|
|
3260
|
+
}
|
|
2851
3261
|
/** `cleo cloud verify`: local integrity, local vs head per table, and every device's newest snapshot vs the head. */
|
|
2852
3262
|
async function verifyNexusVaultImpl(opts = {}) {
|
|
2853
3263
|
const conn = await connectNexusVault(opts);
|
|
@@ -2872,9 +3282,24 @@ async function verifyNexusVaultImpl(opts = {}) {
|
|
|
2872
3282
|
// Files changed since the last sync, path by path (T13038).
|
|
2873
3283
|
const fileChanges = read ? fileChangesSinceSync(read.files, t, git, synced) : null;
|
|
2874
3284
|
const last = synced ? trusted.find((c) => c.checkpointId === synced.lastCheckpointId) : undefined;
|
|
2875
|
-
|
|
3285
|
+
let tables = headCp
|
|
2876
3286
|
? compareWithSynced(local, headCp.manifest, git, headCp.checkpointId === last?.checkpointId ? fileChanges : null)
|
|
2877
3287
|
: [];
|
|
3288
|
+
// A store that writes the stream's change journal is compared with the
|
|
3289
|
+
// journal's tallies past the head, never the head's frozen counts (T12999).
|
|
3290
|
+
const journalSide = headCp &&
|
|
3291
|
+
isJournalSnapshot(headCp) &&
|
|
3292
|
+
(await readGenesisCut(t.dbPath, t.streamId)) !== undefined
|
|
3293
|
+
? await journalVerifySide(journal, key.signers, t, headCp)
|
|
3294
|
+
: null;
|
|
3295
|
+
if (journalSide !== null) {
|
|
3296
|
+
tables = tables.map((r) => {
|
|
3297
|
+
if (isSnapshotAuthoritativeKey(r.table))
|
|
3298
|
+
return r;
|
|
3299
|
+
const cloudRows = journalSide.rows[r.table] ?? 0;
|
|
3300
|
+
return { ...r, cloudRows, match: (r.localRows ?? 0) === cloudRows };
|
|
3301
|
+
});
|
|
3302
|
+
}
|
|
2878
3303
|
let verdict;
|
|
2879
3304
|
if (headUntrusted)
|
|
2880
3305
|
verdict = 'untrusted';
|
|
@@ -2882,6 +3307,10 @@ async function verifyNexusVaultImpl(opts = {}) {
|
|
|
2882
3307
|
verdict = 'empty';
|
|
2883
3308
|
else if (tables.every((x) => x.match))
|
|
2884
3309
|
verdict = 'match';
|
|
3310
|
+
else if (journalSide !== null) {
|
|
3311
|
+
const { localChanged, cloudAdvanced } = journalSide;
|
|
3312
|
+
verdict = localChanged && cloudAdvanced ? 'diverged' : cloudAdvanced ? 'behind' : 'ahead';
|
|
3313
|
+
}
|
|
2885
3314
|
else {
|
|
2886
3315
|
const localChanged = last
|
|
2887
3316
|
? compareWithSynced(local, last.manifest, git, fileChanges).some((d) => !d.match)
|