@cleocode/core 2026.10.17 → 2026.10.18-rc.ci.688

package/dist/cloud/nexus-vault.js

@@ -57,7 +57,7 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { ListCheckpointsResult, ListHomeReplicasResult, ListLeasesResult, ReplayPin, } from '@cleocode/contracts/cloud';
-import { NEXUS_CLOUD_MAX_PAGES, NEXUS_PAGE_LIMIT_MAX, nexusCloudDevicePageSchema, nexusCloudReplicaPageSchema, nexusCloudStatusSchema, } from '@cleocode/contracts/nexus-cloud.js';
+import { NEXUS_CLOUD_MAX_PAGES, NEXUS_FEATURE_MANIFEST_SNAPSHOT_AUTHORITATIVE, NEXUS_PAGE_LIMIT_MAX, nexusCloudDevicePageSchema, nexusCloudReplicaPageSchema, nexusCloudStatusSchema, } from '@cleocode/contracts/nexus-cloud.js';
 import { NEXUS_VAULT_LEASE_ROLE, nexusLeaseAcquireSchema, nexusStreamHeadSchema, } from '@cleocode/contracts/nexus-vault.js';
 import { SYNC_SCHEMA_VERSION } from '@cleocode/contracts/sync-schema.js';
 import { readDeclaredProjectIdentity } from '@cleocode/paths';
@@ -69,6 +69,7 @@ import { exportPortableBundle, globalHomeRules } from '../store/portable-bundle.
 import { extractPortableBundle, importPortableBundle } from '../store/portable-bundle-import.js';
 import { integrityCheck, PROJECT_SECTION_RULES, scanSection, sha256File, } from '../store/portable-bundle-scan.js';
 import { writeRestoreMarker } from '../store/restore-marker.js';
+import { ROW_IDENTITY_EXEMPT } from '../store/row-identity-registry.js';
 import { FIRST_OPEN_LOCK_SUFFIX } from '../store/sqlite.js';
 import { isSyncFlagOn, setSyncFlag, UNRELEASED_FLAGS } from '../store/sync/flags.js';
 import { completeGenesis, cutGenesisWithSnapshot, GenesisRacedError, GenesisTakenOverError, genesisCutOf, genesisPending, joinStream, readGenesisCut, } from '../store/sync/genesis.js';
@@ -422,7 +423,8 @@ function formatWarning(t, cp) {
     if (!cp)
         return null;
     const format = vaultManifestFormat(cp.manifest, hashKeyOf(t.dataKey));
-    if (format === VAULT_MANIFEST_FORMAT_VERSION)
+    // Format 2 is compared in its own terms (local rows counted, T13808).
+    if (format === VAULT_MANIFEST_FORMAT_VERSION || format === 2)
         return null;
     return {
         code: 'W_NEXUS_VAULT_FORMAT',
@@ -702,8 +704,10 @@ function comparable(m, git) {
  * The vault manifest of an extracted bundle: the syncing tables of its
  * `cleo.db`, one entry per other database of the section, and one entry for
  * its plain-file inventory (T12969), so a change anywhere in the bundle is seen.
+ * Checked against a checkpoint hashed under format 2 (`like`), rows local by
+ * row-identity spec are counted as that format did (T13808).
  */
-async function snapshotManifest(extractDir, manifest, t) {
+async function snapshotManifest(extractDir, manifest, t, like) {
     const section = t.scope === 'global' ? manifest.global?.home : manifest.projects[0];
     const entry = section?.databases.find((d) => d.role === 'primary' && d.relPath === 'cleo.db');
     if (!section || !entry) {
@@ -714,7 +718,14 @@ async function snapshotManifest(extractDir, manifest, t) {
     const root = t.scope === 'global' ? null : (manifest.projects[0]?.originalPath ?? section.originalRoot);
     const hashKey = hashKeyOf(t.dataKey);
     const dbPath = path.join(extractDir, entry.bundlePath);
-    const vault = buildVaultManifest(dbPath, { scope: tableScopeOf(t), hashKey, root }).manifest;
+    const likeFormat = like ? vaultManifestFormat(like, hashKey) : VAULT_MANIFEST_FORMAT_VERSION;
+    const built = buildVaultManifest(dbPath, {
+        scope: tableScopeOf(t),
+        hashKey,
+        root,
+        countLocalRows: likeFormat === 2,
+    });
+    const vault = built.manifest;
     for (const d of section.databases) {
         if (d === entry)
             continue;
@@ -738,7 +749,7 @@ async function snapshotManifest(extractDir, manifest, t) {
         files.push({ relPath: f.relPath, sha256: digest });
     }
     vault.tables[VAULT_FILES_KEY] = vaultFilesEntry(files, hashKey);
-    return { vault, dbPath, files: digests };
+    return { vault, dbPath, files: digests, localRows: built.localRows };
 }
 /** Export and extract the store into `work`, returning its bundle and vault manifest. */
 async function exportAndRead(t, work, label, tracked, seal, inspect) {
@@ -748,8 +759,15 @@ async function exportAndRead(t, work, label, tracked, seal, inspect) {
     fs.mkdirSync(extractDir);
     await tarExtract({ file: bundlePath, cwd: extractDir });
     const bundleManifest = JSON.parse(fs.readFileSync(path.join(extractDir, 'manifest.json'), 'utf8'));
-    const { vault, files, dbPath } = await snapshotManifest(extractDir, bundleManifest, t);
-    return { bundlePath, vault, gitTracked: snapshotGitTracked(bundleManifest, t), files, dbPath };
+    const { vault, files, dbPath, localRows } = await snapshotManifest(extractDir, bundleManifest, t);
+    return {
+        bundlePath,
+        vault,
+        gitTracked: snapshotGitTracked(bundleManifest, t),
+        files,
+        dbPath,
+        localRows,
+    };
 }
 /** The paths a project snapshot's bundle marks as git-tracked; a global one has none (T13019, T13038). */
 function snapshotGitTracked(manifest, t) {
@@ -769,15 +787,22 @@ function snapshotGitTracked(manifest, t) {
  * in `untracked` (git-tracked here, or marked by the snapshot it is compared
  * with) are left out of the inventory hash, and every inventory file's digest
  * is returned for a path-by-path comparison (T13038); with tracking unknown
- * the file inventory is left out entirely (T13019).
+ * the file inventory is left out entirely (T13019). Compared with a
+ * checkpoint hashed under format 2 (`like`), rows local by row-identity spec
+ * are counted as that format did (T13808).
  */
-async function localManifest(t, git, untracked) {
+async function localManifest(t, git, untracked, like) {
     if (!fs.existsSync(t.dbPath))
         return null;
     const { sectionRoot, scan, primaryRel } = sectionScan(t);
     const hashKey = hashKeyOf(t.dataKey);
     const root = t.scope === 'global' ? null : t.storeRoot;
-    const vault = buildVaultManifest(t.dbPath, { scope: tableScopeOf(t), hashKey, root }).manifest;
+    const vault = buildVaultManifest(t.dbPath, {
+        scope: tableScopeOf(t),
+        hashKey,
+        root,
+        countLocalRows: like !== undefined && vaultManifestFormat(like, hashKey) === 2,
+    }).manifest;
     for (const rel of scan.sqlite) {
         if (rel === primaryRel)
             continue;
@@ -941,6 +966,53 @@ async function appendCheckpoint(a) {
     }
     return { cp, deltaSegmentSeq, cursor: pushedWith };
 }
+/**
+ * Whether a refusal names an omitted snapshot-authoritative list (T13777): a
+ * parent checkpoint carried one and this push did not, the mark of a device on
+ * a cleo that predates the list.
+ */
+function authoritativeOmitted(err) {
+    if (!(err instanceof NexusError) || err.code !== 'E_MANIFEST_ACCOUNTING')
+        return false;
+    const verdict = err.details?.['verdict'];
+    const findings = typeof verdict === 'object' && verdict !== null && 'findings' in verdict
+        ? verdict.findings
+        : null;
+    return (Array.isArray(findings) &&
+        findings.some((f) => typeof f === 'object' &&
+            f !== null &&
+            'reason' in f &&
+            f.reason === 'authoritative-omitted'));
+}
+/**
+ * The rows a parent hashed under an older vault format counted that this
+ * format leaves out (T13808): rows local by row-identity spec. They never
+ * travelled, so the journal declares no delete for them; the checkpoint names
+ * them `pruned`, so the server's row rule takes them off the parent's count.
+ * Per table, the shortfall against the tallies, capped by the local rows this
+ * store holds now; a parent in the current format prunes nothing.
+ *
+ * @param parentFormat - The parent's {@link vaultManifestFormat}.
+ * @param tallies - The window's tallies.
+ * @param local - The store's rows per manifest key.
+ * @param localRows - Per table, the rows the manifest left out as local.
+ * @returns The tallies less the pruned rows, and the `pruned` map.
+ */
+export function formatTransitionPruned(parentFormat, tallies, local, localRows) {
+    const pruned = {};
+    if (parentFormat === 'unknown' || parentFormat >= VAULT_MANIFEST_FORMAT_VERSION) {
+        return { tallies, pruned };
+    }
+    const rows = { ...tallies.rows };
+    for (const [table, n] of Object.entries(localRows)) {
+        const take = Math.min(n, (rows[table] ?? 0) - (local[table] ?? 0));
+        if (take <= 0)
+            continue;
+        pruned[table] = take;
+        rows[table] = (rows[table] ?? 0) - take;
+    }
+    return { tallies: { ...tallies, rows }, pruned };
+}
 /** Whether this store's `sync.push` is on: its vault push checkpoints the journal (T12999). */
 async function journalPushOn(t) {
     const { openNativeDatabase } = await import('../store/sqlite-native.js');
@@ -987,6 +1059,30 @@ async function snapshotJournalState(snapshotPath, stream, replicaId) {
         db.close();
     }
 }
+/**
+ * The tables a journal checkpoint names as snapshot-authoritative (T13777): the manifest's portable
+ * tables that are still exempt from row identity, so the change journal never carries them and no
+ * segment declares their rows. Kept to the parent's list when it has one (the server allows only a
+ * shrinking list). `list` is null when the server does not take the field
+ * ({@link NEXUS_FEATURE_MANIFEST_SNAPSHOT_AUTHORITATIVE} absent from E3), so the manifest omits it.
+ */
+async function snapshotAuthoritativeTables(conn, t, keys, parent) {
+    const exempt = new Set(Object.keys(ROW_IDENTITY_EXEMPT[tableScopeOf(t)]));
+    const allowed = parent.manifest.snapshotAuthoritative;
+    // A parent with a list proves the server takes it, and omitting it would be refused as a reset.
+    if (allowed === undefined) {
+        const status = await conn.find(t.projectId !== null
+            ? nexusQueryPath('/v1/status', { projectId: t.projectId })
+            : '/v1/status', nexusCloudStatusSchema);
+        if (!(status?.features ?? []).includes(NEXUS_FEATURE_MANIFEST_SNAPSHOT_AUTHORITATIVE)) {
+            return { list: null, exempt };
+        }
+    }
+    return {
+        list: keys.filter((k) => exempt.has(k) && (allowed === undefined || allowed.includes(k))),
+        exempt,
+    };
+}
 /**
  * `cleo cloud push` on a store that writes its stream's change journal
  * (`sync.push` on; T12999, journal spec §2.11): seal, push and pull the
@@ -1049,7 +1145,7 @@ async function pushJournalCheckpoint(conn, key, opts, scope, warnings) {
         const work = tempDir('cleo-vault-push-');
         try {
             let snap = null;
-            const { bundlePath, vault, gitTracked, files } = await exportAndRead(t, work, `cloud-vault-${scope}`, trackedForPush(t, git, synced), { masterKey: key.masterKey, warn: (w) => warnings.push(w) }, async (snapshotPath) => {
+            const { bundlePath, vault, gitTracked, files, localRows } = await exportAndRead(t, work, `cloud-vault-${scope}`, trackedForPush(t, git, synced), { masterKey: key.masterKey, warn: (w) => warnings.push(w) }, async (snapshotPath) => {
                 const state = await snapshotJournalState(snapshotPath, t.streamId, replicaId);
                 let read = null;
                 if (state.cursor !== null &&
@@ -1124,9 +1220,16 @@ async function pushJournalCheckpoint(conn, key, opts, scope, warnings) {
             const keys = [...new Set([...Object.keys(tables), ...Object.keys(parent.manifest.tables)])]
                 .filter((k) => !isVaultAnnotationKey(k))
                 .sort();
-            const { tallies, unexplained } = settleShortfalls(taken.read.tallies, local, keys.filter((k) => !isSnapshotAuthoritativeKey(k)));
+            // Portable tables the journal does not carry yet are the snapshot's alone (T13777), named in
+            // the manifest only to a server that takes the list; the list only shrinks along the lineage.
+            const authoritative = await snapshotAuthoritativeTables(conn, t, keys, parent);
+            const formatPruned = formatTransitionPruned(vaultManifestFormat(parent.manifest, hashKeyOf(t.dataKey)), taken.read.tallies, local, localRows);
+            const { tallies, unexplained } = settleShortfalls(formatPruned.tallies, local, keys.filter((k) => !isSnapshotAuthoritativeKey(k) && !(authoritative.list?.includes(k) ?? false)));
             if (unexplained.length > 0) {
-                throw vaultError('E_NEXUS_VAULT_REFUSED', `this store's rows do not reconcile with ${t.streamId}'s change journal (${unexplained.slice(0, 8).join(', ')}): a write landed outside the journal`, 'nothing was written; run `cleo cloud sync`, then push again');
+                const exemptOnly = authoritative.list === null && unexplained.every((k) => authoritative.exempt.has(k));
+                throw vaultError('E_NEXUS_VAULT_REFUSED', `this store's rows do not reconcile with ${t.streamId}'s change journal (${unexplained.slice(0, 8).join(', ')}): ${exemptOnly ? 'these tables are not in the change journal yet, and this Cleo Nexus server does not take a snapshot-authoritative list (manifest.snapshot-authoritative)' : 'a write landed outside the journal'}`, exemptOnly
+                    ? 'nothing was written; upgrade the Cleo Nexus server, then push again'
+                    : 'nothing was written; run `cleo cloud sync`, then push again');
             }
             const vaultOnlyChanged = keys.some((k) => isSnapshotAuthoritativeKey(k) && local[k] !== (parent.manifest.tables[k]?.rows ?? 0));
             const acquired = await acquireLease(conn, t, opts.force === true, opts.force === true ? 'cleo cloud push --force' : 'cleo cloud push');
@@ -1145,8 +1248,9 @@ async function pushJournalCheckpoint(conn, key, opts, scope, warnings) {
                             pending: tallies.pending,
                             voided: normalRefDeltas(tallies.voided),
                             revived: [],
-                            pruned: {},
+                            pruned: formatPruned.pruned,
                             replayPin: pin,
+                            ...(authoritative.list !== null ? { snapshotAuthoritative: authoritative.list } : {}),
                         },
                         cursor: {
                             after: cursor.after,
@@ -1160,6 +1264,9 @@ async function pushJournalCheckpoint(conn, key, opts, scope, warnings) {
                     if (err instanceof NexusError && err.code === 'E_LINEAGE') {
                         throw vaultError('E_NEXUS_VAULT_BEHIND', 'another device pushed a snapshot while this one was uploading', 'run `cleo cloud sync`, then push again');
                     }
+                    if (authoritativeOmitted(err)) {
+                        throw vaultError('E_NEXUS_VAULT_REFUSED', `the server refused the snapshot: a device on ${t.streamId} sent a snapshot-authoritative list and this push did not carry it (${err.message})`, 'nothing was written; once one device sends the list, every device on the stream must run a cleo that sends it: upgrade cleo on the other devices, then push again');
+                    }
                     if (err instanceof NexusError &&
                         (err.code === 'E_REGRESSION' || err.code === 'E_VALIDATION')) {
                         throw vaultError('E_NEXUS_VAULT_REFUSED', `the server refused the snapshot: ${err.message}`, vaultOnlyChanged
@@ -1590,7 +1697,7 @@ async function enableSyncPushImpl(opts = {}) {
         // Unchanged since the restore: the rows match the checkpoint's manifest
         // (row data) here, and joinStream checks every row against its meta.
         const git = gitTracking(t);
-        const local = await localManifest(t, git, untrackedComparison(t, git, synced));
+        const local = await localManifest(t, git, untrackedComparison(t, git, synced), parent.manifest);
         const changed = local === null
             ? ['(no store)']
             : compareVaultManifests(local.manifest, comparable(parent.manifest, git))
@@ -2648,7 +2755,7 @@ function keepGlobalConfigLocalKeys(configPath, local) {
  */
 async function pullVaultOnlyEntries(a) {
     const { conn, key, t, journal, target, last, git, synced, warnings } = a;
-    const read = await localManifest(t, git, untrackedComparison(t, git, synced));
+    const read = await localManifest(t, git, untrackedComparison(t, git, synced), last?.manifest);
     if (read !== null && last !== undefined) {
         const fileChanges = fileChangesSinceSync(read.files, t, git, synced);
         const changed = compareWithSynced(read.manifest, last.manifest, git, fileChanges)
@@ -2668,7 +2775,7 @@ async function pullVaultOnlyEntries(a) {
         const staging = path.join(work, 'staging');
         fs.mkdirSync(staging);
         const { extractDir, manifest } = await extractPortableBundle(bundlePath, staging);
-        const staged = await snapshotManifest(extractDir, manifest, t);
+        const staged = await snapshotManifest(extractDir, manifest, t, restored.checkpoint.manifest);
         const diff = compareVaultManifests(staged.vault, restored.checkpoint.manifest).filter((d) => !d.match);
         if (diff.length > 0) {
             throw vaultError('E_NEXUS_VAULT_VERIFY_FAILED', `snapshot ${target.checkpointId} does not match its manifest (${diff.map((d) => d.table).join(', ')}); nothing was pulled`);
@@ -2793,7 +2900,8 @@ async function restoreNexusVaultImpl(opts) {
     // against its manifest here; say so instead of reporting a mismatch (T13034).
     const targetCp = trusted.find((c) => c.checkpointId === target);
     const format = targetCp ? vaultManifestFormat(targetCp.manifest, hashKeyOf(t.dataKey)) : null;
-    if (format !== null && format !== VAULT_MANIFEST_FORMAT_VERSION) {
+    // Format 2 is format 3 with local rows counted: a restore checks it that way (T13808).
+    if (format !== null && format !== VAULT_MANIFEST_FORMAT_VERSION && format !== 2) {
         throw vaultError('E_NEXUS_VAULT_VERIFY_FAILED', format === 'unknown'
             ? `snapshot ${target} records a vault manifest format this CLEO does not recognise (this CLEO hashes format ${VAULT_MANIFEST_FORMAT_VERSION}; a different vault format wrote it, most likely a newer CLEO), so it cannot be verified here; nothing was restored`
             : `snapshot ${target} was hashed under vault manifest format ${format}, older than this CLEO's (${VAULT_MANIFEST_FORMAT_VERSION}), so it cannot be verified here; nothing was restored`, format === 'unknown'
@@ -2829,7 +2937,9 @@ async function restoreNexusVaultImpl(opts) {
             names,
         });
     }
-    const local = hasLocal && !force ? await localManifest(t, git, untrackedComparison(t, git, synced)) : null;
+    const local = hasLocal && !force
+        ? await localManifest(t, git, untrackedComparison(t, git, synced), last?.manifest)
+        : null;
     if (local !== null) {
         const hasRows = Object.entries(local.manifest.tables).some(([name, x]) => x.rows > 0 && name !== VAULT_FILES_KEY);
         const fileChanges = fileChangesSinceSync(local.files, t, git, synced);
@@ -2910,7 +3020,7 @@ async function restoreNexusVaultImpl(opts) {
                         throw vaultError('E_NEXUS_VAULT_TARGET_OCCUPIED', `${t.storeRoot} already holds project ${present.projectId}, not ${incoming}`, 'pass --into an empty directory, or --force to replace it (a safety backup is taken first)');
                     }
                 }
-                const staged = await snapshotManifest(extractDir, manifest, t);
+                const staged = await snapshotManifest(extractDir, manifest, t, restored.checkpoint.manifest);
                 const diff = compareVaultManifests(staged.vault, restored.checkpoint.manifest).filter((d) => !d.match);
                 if (diff.length > 0) {
                     throw vaultError('E_NEXUS_VAULT_VERIFY_FAILED', `snapshot ${target} does not match its manifest (${diff.map((d) => d.table).join(', ')}); nothing was restored`);
@@ -3108,7 +3218,7 @@ async function nexusVaultStatusImpl(opts = {}) {
     const git = gitTracking(t);
     if (git.mode === 'unknown')
         warnings.push(gitUnknownWarning(t));
-    const local = await localManifest(t, git, untrackedComparison(t, git, synced));
+    const local = await localManifest(t, git, untrackedComparison(t, git, synced), last?.manifest);
     const pendingChanges = local && last
         ? compareWithSynced(local.manifest, last.manifest, git, fileChangesSinceSync(local.files, t, git, synced)).filter((d) => !d.match)
         : [];
@@ -3252,6 +3362,7 @@ async function journalVerifySide(journal, signers, t, headCp) {
             rows,
             cloudAdvanced: after < window.head,
             localChanged: unpushed > 0 || sealBacklog(db).live > 0,
+            authoritative: new Set(Object.keys(ROW_IDENTITY_EXEMPT[tableScopeOf(t)])),
         };
     }
     finally {
@@ -3277,7 +3388,7 @@ async function verifyNexusVaultImpl(opts = {}) {
     const git = gitTracking(t);
     if (git.mode === 'unknown')
         warnings.push(gitUnknownWarning(t));
-    const read = await localManifest(t, git, untrackedComparison(t, git, synced));
+    const read = await localManifest(t, git, untrackedComparison(t, git, synced), headCp?.manifest);
     const local = read?.manifest ?? { schemaVersion: SYNC_SCHEMA_VERSION, tables: {} };
     // Files changed since the last sync, path by path (T13038).
     const fileChanges = read ? fileChangesSinceSync(read.files, t, git, synced) : null;
@@ -3294,7 +3405,7 @@ async function verifyNexusVaultImpl(opts = {}) {
         : null;
     if (journalSide !== null) {
         tables = tables.map((r) => {
-            if (isSnapshotAuthoritativeKey(r.table))
+            if (isSnapshotAuthoritativeKey(r.table) || journalSide.authoritative.has(r.table))
                 return r;
             const cloudRows = journalSide.rows[r.table] ?? 0;
             return { ...r, cloudRows, match: (r.localRows ?? 0) === cloudRows };