@amalgm/shell 0.1.109 → 0.1.111

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -27,7 +27,6 @@ import { GroundCoordinator } from "./ground-coordination.js";
27
27
  import { submitMutationOperation } from "./paged-mutation-submit.js";
28
28
  import { adoptSnapshotDeliveryBaseline } from "./snapshot-delivery-baseline.js";
29
29
  import { WireClient, WireRequestError } from "./wire-client.js";
30
- const SMALL_CONTENT_UPLOAD_CONCURRENCY = 16;
31
30
  /** Cargo waves hold up to 16MiB of payload each; two in flight bounds the
32
31
  * upload path's memory while keeping the wire busy. */
33
32
  const CARGO_UPLOAD_CONCURRENCY = 2;
@@ -47,8 +46,24 @@ const DETECT_RETRY_MS = 250;
47
46
  /** A Detect walk yields the event loop this often so Send, Receive, and
48
47
  * Apply progress during a large reconciliation instead of queueing behind it. */
49
48
  const DETECT_WALK_YIELD_ENTRIES = 256;
49
+ /** File captures during a walk overlap reads and hashing across this many
50
+ * lanes; the directory traversal itself stays ordered and serial. */
51
+ const WALK_CAPTURE_CONCURRENCY = 8;
52
+ /** How many times one register attempt re-derives after ground moved under
53
+ * its capture. Each attempt re-walks at read+hash speed, so steady churn is
54
+ * surfaced instead of chased forever. */
55
+ const REGISTER_STALE_CAPTURE_ATTEMPTS = 3;
50
56
  const DOWNLOAD_CONTENT_RETRY_ATTEMPTS = 3;
51
57
  const APPLY_GROUND_WAIT_TIMEOUT_MS = 15_000;
58
+ function loadSnapshotPublication(entry) {
59
+ const snapshotRecords = snapshotFromRecords((JSON.parse(entry.snapshotBlob.toString("utf8")).records ?? [])).records;
60
+ const uploadIds = new Set(entry.uploadUuids);
61
+ return {
62
+ ...entry,
63
+ snapshotRecords,
64
+ uploadRecords: snapshotRecords.filter((record) => uploadIds.has(record.uuid)),
65
+ };
66
+ }
52
67
  const GROUND_ROW_COLUMNS = [
53
68
  "uuid", "resource_id", "root_uuid", "type", "parent_uuid", "name", "status", "version",
54
69
  "payload_version", "transport_version", "relative_path", "absolute_path", "device_number",
@@ -532,19 +547,30 @@ export class UserGroundHost {
532
547
  // unchanged while a contract cutover moved its cargo namespace, so
533
548
  // the publish path re-proves seal evidence and repairs any missing
534
549
  // uploads before this answer is lawful.
550
+ let staleRepair = false;
535
551
  try {
536
552
  const deliveryWatermark = await this.prepareSnapshotPublication(identity, { journey: "register", workspaceId });
537
553
  await this.publishMaterializedSnapshot(identity, new Set([workspaceId]), deliveryWatermark, { journey: "register", workspaceId });
538
554
  }
539
- finally {
555
+ catch (error) {
556
+ // A stale capture means a repair upload's evidence no longer matches
557
+ // the ground it was captured from. The full walk below recaptures
558
+ // and republishes current truth instead of failing the register.
559
+ if (!staleCapture(error)) {
560
+ this.finishColdOperation();
561
+ throw error;
562
+ }
563
+ staleRepair = true;
564
+ }
565
+ if (!staleRepair) {
540
566
  this.finishColdOperation();
567
+ return {
568
+ registered: true,
569
+ entityCount: countDescendantRows(database, workspaceId),
570
+ cloudHead: this.cloudState.checksum,
571
+ watch,
572
+ };
541
573
  }
542
- return {
543
- registered: true,
544
- entityCount: countDescendantRows(database, workspaceId),
545
- cloudHead: this.cloudState.checksum,
546
- watch,
547
- };
548
574
  }
549
575
  // Register is a cold declaration, not a filesystem edit. It performs one
550
576
  // complete reconciliation and publishes one graph head; it must not mint
@@ -578,75 +604,88 @@ export class UserGroundHost {
578
604
  const observations = this.watchHost.observations().filter((observation) => observation.directory === this.userRoot(identity)
579
605
  || pathWithin(contentOwner, observation.directory));
580
606
  this.watchDirty = false;
581
- const registrationStarted = performance.now();
582
- this.stage({
583
- primitive: "register", stage: "entity-registration", status: "started", workspaceId,
584
- });
585
- const scanned = await reconcileGround({
586
- identity,
587
- userRoot: this.userRoot(identity),
588
- database: this.databasePath(identity),
589
- cacheDir: this.cacheDir(identity),
590
- onScan: (evidence) => this.reportDetectScan(evidence),
591
- registrationWorkspaceId: workspaceId,
592
- suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
593
- });
594
- const retry = scanned.detection.find((plan) => plan.kind === "retry");
595
- if (retry?.kind === "retry") {
596
- throw new Error(retry.reasons.join("; "));
597
- }
598
- const registrationDuration = performance.now() - registrationStarted;
599
- const commitStarted = performance.now();
600
- this.stage({
601
- primitive: "register", stage: "entity-sqlite-commit", status: "started", workspaceId,
602
- });
603
- const committed = commitDetectedState(this.databasePath(identity), {
604
- notebookBasis: scanned.notebookBasis,
605
- acceptedMaterializedRows: scanned.acceptedMaterializedRows,
606
- materializedRemovals: scanned.materializedRemovals,
607
- acceptedNotebookRows: scanned.acceptedNotebookRows,
608
- notebookRemovals: scanned.notebookRemovals,
609
- });
610
- if (!committed) {
611
- throw new Error("verified ground advanced while registration was observing; retry");
612
- }
613
- this.stage({
614
- primitive: "register",
615
- stage: "entity-sqlite-commit",
616
- status: "completed",
617
- workspaceId,
618
- durationMs: performance.now() - commitStarted,
619
- });
620
- if (scanned.transferFailures.length > 0) {
621
- const details = scanned.transferFailures
622
- .map(({ rootId, reason }) => `${rootId}: ${reason}`)
623
- .join("; ");
624
- this.stage({
625
- primitive: "register",
626
- stage: "entity-registration",
627
- status: "failed",
628
- workspaceId,
629
- durationMs: registrationDuration,
630
- measurements: { roots: scanned.transferFailures.length },
631
- });
632
- throw pipelineStageError("entity registration", new Error(`repository state could not be captured: ${details}`));
633
- }
634
- this.stage({
635
- primitive: "register",
636
- stage: "entity-registration",
637
- status: "completed",
638
- workspaceId,
639
- durationMs: registrationDuration,
640
- measurements: {
641
- roots: scanned.publicationRootIds?.length ?? 0,
642
- rows: scanned.acceptedMaterializedRows.length,
643
- },
644
- });
645
- this.namedDetect?.refreshEnrollmentPolicy();
646
- if (!scanned.publicationRootIds) {
647
- throw new Error("cold registration reconciliation did not name its publication roots");
607
+ // Ground may move between a walk's capture and its upload. Each retry
608
+ // re-walks and republishes the ground as it now is — bounded, because
609
+ // a workspace under continuous mutation belongs to Watch afterwards,
610
+ // not to an endless register.
611
+ for (let attempt = 1;; attempt++) {
612
+ try {
613
+ const registrationStarted = performance.now();
614
+ this.stage({
615
+ primitive: "register", stage: "entity-registration", status: "started", workspaceId,
616
+ });
617
+ const scanned = await reconcileGround({
618
+ identity,
619
+ userRoot: this.userRoot(identity),
620
+ database: this.databasePath(identity),
621
+ cacheDir: this.cacheDir(identity),
622
+ onScan: (evidence) => this.reportDetectScan(evidence),
623
+ registrationWorkspaceId: workspaceId,
624
+ suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
625
+ });
626
+ const retry = scanned.detection.find((plan) => plan.kind === "retry");
627
+ if (retry?.kind === "retry") {
628
+ throw new Error(retry.reasons.join("; "));
629
+ }
630
+ const registrationDuration = performance.now() - registrationStarted;
631
+ const commitStarted = performance.now();
632
+ this.stage({
633
+ primitive: "register", stage: "entity-sqlite-commit", status: "started", workspaceId,
634
+ });
635
+ const committed = commitDetectedState(this.databasePath(identity), {
636
+ notebookBasis: scanned.notebookBasis,
637
+ acceptedMaterializedRows: scanned.acceptedMaterializedRows,
638
+ materializedRemovals: scanned.materializedRemovals,
639
+ acceptedNotebookRows: scanned.acceptedNotebookRows,
640
+ notebookRemovals: scanned.notebookRemovals,
641
+ });
642
+ if (!committed) {
643
+ throw new Error("verified ground advanced while registration was observing; retry");
644
+ }
645
+ this.stage({
646
+ primitive: "register",
647
+ stage: "entity-sqlite-commit",
648
+ status: "completed",
649
+ workspaceId,
650
+ durationMs: performance.now() - commitStarted,
651
+ });
652
+ if (scanned.transferFailures.length > 0) {
653
+ const details = scanned.transferFailures
654
+ .map(({ rootId, reason }) => `${rootId}: ${reason}`)
655
+ .join("; ");
656
+ this.stage({
657
+ primitive: "register",
658
+ stage: "entity-registration",
659
+ status: "failed",
660
+ workspaceId,
661
+ durationMs: registrationDuration,
662
+ measurements: { roots: scanned.transferFailures.length },
663
+ });
664
+ throw pipelineStageError("entity registration", new Error(`repository state could not be captured: ${details}`));
665
+ }
666
+ this.stage({
667
+ primitive: "register",
668
+ stage: "entity-registration",
669
+ status: "completed",
670
+ workspaceId,
671
+ durationMs: registrationDuration,
672
+ measurements: {
673
+ roots: scanned.publicationRootIds?.length ?? 0,
674
+ rows: scanned.acceptedMaterializedRows.length,
675
+ },
676
+ });
677
+ this.namedDetect?.refreshEnrollmentPolicy();
678
+ if (!scanned.publicationRootIds) {
679
+ throw new Error("cold registration reconciliation did not name its publication roots");
680
+ }
681
+ await this.publishMaterializedSnapshot(identity, new Set(scanned.publicationRootIds), deliveryWatermark, { journey: "register", workspaceId }, scanned.largeManifests);
682
+ break;
683
+ }
684
+ catch (error) {
685
+ if (!staleCapture(error) || attempt >= REGISTER_STALE_CAPTURE_ATTEMPTS)
686
+ throw error;
687
+ }
648
688
  }
649
- await this.publishMaterializedSnapshot(identity, new Set(scanned.publicationRootIds), deliveryWatermark, { journey: "register", workspaceId });
650
689
  this.watchHost.settle(observations);
651
690
  }
652
691
  catch (error) {
@@ -1154,27 +1193,15 @@ export class UserGroundHost {
1154
1193
  }
1155
1194
  return inline;
1156
1195
  }
1157
- async uploadSnapshotContent(identity, resourceId, records, trace) {
1196
+ async uploadSnapshotContent(identity, resourceId, records, trace, manifestHints) {
1158
1197
  const cache = this.cacheDir(identity);
1159
1198
  const byHash = new Map();
1160
1199
  for (const record of records) {
1161
1200
  const artifact = artifactForRecord(record);
1162
- if (artifact)
1163
- byHash.set(artifact.contentHash, artifact);
1201
+ if (artifact && !byHash.has(artifact.contentHash)) {
1202
+ byHash.set(artifact.contentHash, { artifact, uuid: record.uuid });
1203
+ }
1164
1204
  }
1165
- const sources = [...byHash.values()].map((artifact) => {
1166
- const file = contentCacheFile(cache, artifact.contentHash);
1167
- const manifest = readCachedManifest(cache, artifact.contentHash);
1168
- return {
1169
- artifact,
1170
- file,
1171
- manifest,
1172
- // A retained manifest can prove the remote inventory without local
1173
- // cargo bytes. The source file is required only when the authority
1174
- // reports a missing block.
1175
- bytes: manifest?.bytes ?? statSync(file).size,
1176
- };
1177
- });
1178
1205
  const started = performance.now();
1179
1206
  let uploadedChunks = 0;
1180
1207
  let reusedChunks = 0;
@@ -1192,29 +1219,94 @@ export class UserGroundHost {
1192
1219
  ...identifiers,
1193
1220
  measurements: {
1194
1221
  journey: trace?.journey ?? "bootstrap",
1195
- artifacts: sources.length,
1196
- declaredBytes: sources.reduce((total, source) => total + source.bytes, 0),
1222
+ artifacts: byHash.size,
1197
1223
  },
1198
1224
  });
1225
+ // The authority's seal inventory is the only permission to skip an
1226
+ // upload — asked before any source is touched, so sealed artifacts cost
1227
+ // zero local reads and a retried publication whose upload already sealed
1228
+ // completely never needs its captured ground to still exist.
1229
+ const alreadySealed = await this.sealedContentHashes(resourceId, [...byHash.keys()]);
1230
+ const unsealed = [...byHash.values()]
1231
+ .filter(({ artifact }) => !alreadySealed.has(artifact.contentHash));
1232
+ const rows = readGroundRowsByUuid(this.databasePath(identity), unsealed.map(({ uuid }) => uuid));
1233
+ // Each artifact uploads from the strongest evidence available: a cache
1234
+ // copy when another journey pinned one, otherwise the ground entity
1235
+ // itself. Register stages nothing, so the ground path is the common
1236
+ // case — and every read is re-proven against the captured hash, turning
1237
+ // ground churn into a stale capture instead of a wrong upload.
1238
+ const sources = unsealed.map(({ artifact, uuid }) => {
1239
+ const cached = contentCacheFile(cache, artifact.contentHash);
1240
+ const fromCache = existsSync(cached);
1241
+ const row = rows.get(uuid);
1242
+ if (!fromCache && artifact.kind === "git") {
1243
+ // Ground holds the working tree, never the transport parcel; a
1244
+ // lost parcel means recapturing the repository as it now is.
1245
+ throw new StaleCaptureError(`repository parcel ${artifact.contentHash} is no longer captured`);
1246
+ }
1247
+ if (!fromCache && !row) {
1248
+ throw new StaleCaptureError(`entity ${uuid} left the ground before uploading ${artifact.contentHash}`);
1249
+ }
1250
+ const file = fromCache ? cached : row.absolutePath;
1251
+ let linkTarget;
1252
+ let bytes;
1253
+ try {
1254
+ if (!fromCache && artifact.kind === "link") {
1255
+ linkTarget = Buffer.from(readlinkSync(file), "utf8");
1256
+ bytes = linkTarget.byteLength;
1257
+ }
1258
+ else {
1259
+ bytes = statSync(file).size;
1260
+ }
1261
+ }
1262
+ catch (error) {
1263
+ throw new StaleCaptureError(`source for ${artifact.contentHash} is no longer readable`, { cause: error });
1264
+ }
1265
+ const declared = manifestHints?.get(artifact.contentHash)
1266
+ ?? readCachedManifest(cache, artifact.contentHash)
1267
+ ?? undefined;
1268
+ return {
1269
+ artifact,
1270
+ file,
1271
+ fromCache,
1272
+ ...(linkTarget ? { linkTarget } : {}),
1273
+ // Retained evidence only counts while it still describes the exact
1274
+ // source being read; a disagreeing manifest is re-derived instead.
1275
+ ...(declared && declared.bytes === bytes ? { manifest: declared } : {}),
1276
+ bytes,
1277
+ };
1278
+ });
1279
+ const stale = (source, error) => {
1280
+ if (source.fromCache) {
1281
+ // A cache copy that no longer proves its hash is torn evidence:
1282
+ // purge it so the retry reads the ground itself.
1283
+ rmSync(source.file, { force: true });
1284
+ rmSync(manifestCacheFile(cache, source.artifact.contentHash), { force: true });
1285
+ }
1286
+ throw new StaleCaptureError(`${source.artifact.contentHash} no longer matches its source`, { cause: error });
1287
+ };
1199
1288
  // Complete small artifacts ride cargo waves: whole artifacts per frame,
1200
1289
  // verified and sealed by the authority in the same round trip, so the
1201
- // frame count scales with bytes moved, never with file count. A small
1202
- // artifact whose cached manifest survives without local bytes keeps the
1203
- // per-artifact path — its manifest alone can re-prove remote cargo.
1204
- // Large artifacts keep per-artifact transfer where chunk reuse pays.
1290
+ // frame count scales with bytes moved, never with file count. Large
1291
+ // artifacts keep per-artifact transfer where chunk reuse pays.
1205
1292
  const small = sources.filter((source) => source.bytes <= CHUNK_BYTES);
1206
- const waveSources = small.filter((source) => !source.manifest || existsSync(source.file));
1207
- const repairSources = small.filter((source) => source.manifest && !existsSync(source.file));
1208
1293
  const large = sources.filter((source) => source.bytes > CHUNK_BYTES);
1209
1294
  let cargoWaves = 0;
1210
1295
  const transferCargoWave = async (wave) => {
1211
1296
  const loaded = await Promise.all(wave.items.map(async (source) => {
1212
- const bytes = await readFile(source.file);
1213
- return {
1214
- source,
1215
- bytes,
1216
- manifest: deriveUploadManifest(source.artifact, bytes, sha256Hex),
1217
- };
1297
+ try {
1298
+ const bytes = source.linkTarget ?? await readFile(source.file);
1299
+ return {
1300
+ source,
1301
+ bytes,
1302
+ manifest: deriveUploadManifest(source.artifact, bytes, sha256Hex),
1303
+ };
1304
+ }
1305
+ catch (error) {
1306
+ if (sourceMismatch(error))
1307
+ stale(source, error);
1308
+ throw error;
1309
+ }
1218
1310
  }));
1219
1311
  const payload = Buffer.concat(loaded.map((item) => item.bytes));
1220
1312
  const encoded = await encodeContentWireBytes(payload);
@@ -1235,14 +1327,13 @@ export class UserGroundHost {
1235
1327
  if (!sealed.has(item.source.artifact.contentHash)) {
1236
1328
  throw new Error(`content authority did not seal ${item.source.artifact.contentHash}`);
1237
1329
  }
1238
- atomicWrite(manifestCacheFile(cache, item.source.artifact.contentHash), stableJson(item.manifest));
1239
1330
  uploadedChunks += item.manifest.chunks.length;
1240
1331
  }
1241
1332
  uploadedBytes += encoded.bytes.byteLength;
1242
1333
  cargoWaves += 1;
1243
1334
  };
1244
1335
  const transfer = async (source) => {
1245
- const { artifact, file, manifest: cachedManifest, bytes: fileBytes } = source;
1336
+ const { artifact, file, manifest: knownManifest, bytes: fileBytes } = source;
1246
1337
  const sourceFile = { handle: null };
1247
1338
  let manifestPresent = false;
1248
1339
  const putChunk = async (manifest, index, bytes) => {
@@ -1264,8 +1355,7 @@ export class UserGroundHost {
1264
1355
  try {
1265
1356
  const receipt = await uploadArtifact(artifact, {
1266
1357
  bytes: fileBytes,
1267
- ...(cachedManifest ? { manifest: cachedManifest } : {}),
1268
- ...(cachedManifest ? { manifestVerified: true } : {}),
1358
+ ...(knownManifest ? { manifest: knownManifest, manifestVerified: true } : {}),
1269
1359
  verify: async (contentHash) => await hashContentFile(file) === contentHash,
1270
1360
  read: async (offset, bytes) => {
1271
1361
  sourceFile.handle ??= open(file, "r");
@@ -1328,7 +1418,11 @@ export class UserGroundHost {
1328
1418
  });
1329
1419
  uploadedChunks += receipt.uploadedChunks;
1330
1420
  reusedChunks += receipt.reusedChunks;
1331
- atomicWrite(manifestCacheFile(cache, artifact.contentHash), stableJson(receipt.manifest));
1421
+ }
1422
+ catch (error) {
1423
+ if (sourceMismatch(error))
1424
+ stale(source, error);
1425
+ throw error;
1332
1426
  }
1333
1427
  finally {
1334
1428
  if (sourceFile.handle)
@@ -1336,8 +1430,7 @@ export class UserGroundHost {
1336
1430
  }
1337
1431
  };
1338
1432
  try {
1339
- await boundedForEach([...planCargoWaves(waveSources)], CARGO_UPLOAD_CONCURRENCY, transferCargoWave);
1340
- await boundedForEach(repairSources, SMALL_CONTENT_UPLOAD_CONCURRENCY, transfer);
1433
+ await boundedForEach([...planCargoWaves(small)], CARGO_UPLOAD_CONCURRENCY, transferCargoWave);
1341
1434
  for (const source of large) {
1342
1435
  await transfer(source);
1343
1436
  }
@@ -1349,7 +1442,8 @@ export class UserGroundHost {
1349
1442
  durationMs: performance.now() - started,
1350
1443
  measurements: {
1351
1444
  journey: trace?.journey ?? "bootstrap",
1352
- artifacts: sources.length,
1445
+ artifacts: byHash.size,
1446
+ reusedArtifacts: byHash.size - sources.length,
1353
1447
  declaredBytes: sources.reduce((total, source) => total + source.bytes, 0),
1354
1448
  uploadedBytes,
1355
1449
  uploadedChunks,
@@ -1854,7 +1948,7 @@ export class UserGroundHost {
1854
1948
  headVersion: first.baseVersion,
1855
1949
  checksum: "",
1856
1950
  deliveryWatermark: first.deliveryWatermark,
1857
- records: snapshotFromRecords(JSON.parse(first.snapshotJson).records || []).records,
1951
+ records: loadSnapshotPublication(first).snapshotRecords,
1858
1952
  };
1859
1953
  try {
1860
1954
  await this.publishPendingSnapshots(identity);
@@ -1862,9 +1956,11 @@ export class UserGroundHost {
1862
1956
  catch (error) {
1863
1957
  // A conclusive refusal settles the intent as rejected: the lookup that
1864
1958
  // follows fetches authoritative state and a later register supersedes
1865
- // the settled row, so a refused publication never wedges startup. An
1959
+ // the settled row, so a refused publication never wedges startup. A
1960
+ // stale capture already deleted its unsubmitted row — the intent will
1961
+ // be re-derived from current ground by the next register boundary. An
1866
1962
  // unknown outcome must keep blocking — the mutation may have landed.
1867
- if (!conclusivelyRefused(error))
1963
+ if (!conclusivelyRefused(error) && !staleCapture(error))
1868
1964
  throw error;
1869
1965
  this.cloudState = null;
1870
1966
  }
@@ -1875,12 +1971,13 @@ export class UserGroundHost {
1875
1971
  async prepareSnapshotPublication(identity, trace) {
1876
1972
  // An older durable publication resolves before the new boundary derives.
1877
1973
  // Unknown outcomes remain pending; a conclusive refusal may be replaced
1878
- // by the newer complete snapshot below.
1974
+ // by the newer complete snapshot below; a stale capture already deleted
1975
+ // its unsubmitted row and the boundary being prepared re-derives it.
1879
1976
  try {
1880
1977
  await this.publishPendingSnapshots(identity, trace);
1881
1978
  }
1882
1979
  catch (error) {
1883
- if (!conclusivelyRefused(error))
1980
+ if (!conclusivelyRefused(error) && !staleCapture(error))
1884
1981
  throw error;
1885
1982
  }
1886
1983
  await this.recordRail?.flush();
@@ -1898,7 +1995,7 @@ export class UserGroundHost {
1898
1995
  await this.applyRail?.flush();
1899
1996
  return deliveryWatermark;
1900
1997
  }
1901
- async publishMaterializedSnapshot(identity, publicationRootIds, deliveryWatermark, trace) {
1998
+ async publishMaterializedSnapshot(identity, publicationRootIds, deliveryWatermark, trace, manifestHints) {
1902
1999
  const state = this.cloudState;
1903
2000
  if (!state)
1904
2001
  throw new Error("cloud state is unavailable for graph publication");
@@ -1915,32 +2012,27 @@ export class UserGroundHost {
1915
2012
  throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
1916
2013
  }
1917
2014
  const snapshot = snapshotFromRecords(traveling);
1918
- const snapshotJson = stableJson(snapshot);
1919
- const checksum = sha256Hex(snapshotJson);
2015
+ // ONE canonical serialization: these exact bytes are the journal blob
2016
+ // and the checksum's input; they are never re-derived or re-parsed on
2017
+ // the in-process path.
2018
+ const snapshotBlob = Buffer.from(stableJson(snapshot), "utf8");
2019
+ const checksum = sha256Hex(snapshotBlob);
1920
2020
  const localIds = new Set(localRecords.map((record) => record.uuid));
1921
2021
  // Graph membership is never proof of sealed cargo: a record in the cloud
1922
2022
  // graph may name content sealed only under a prior content contract,
1923
2023
  // whose namespace the current readers never touch. Only the authority's
1924
- // (resource, hash, contract) seal evidence may skip an upload — and the
1925
- // question is asked even when the graph itself is unchanged, so Register
1926
- // always re-establishes cargo closure before returning.
1927
- const cargo = snapshot.records.flatMap((record) => {
1928
- if (!localIds.has(record.uuid))
1929
- return [];
1930
- const artifact = artifactForRecord(record);
1931
- return artifact ? [{ record, contentHash: artifact.contentHash }] : [];
1932
- });
1933
- const sealedArtifacts = await this.sealedContentHashes(state.resourceId, [...new Set(cargo.map((entry) => entry.contentHash))]);
1934
- const uploadRecords = cargo
1935
- .filter((entry) => !sealedArtifacts.has(entry.contentHash))
1936
- .map((entry) => entry.record);
2024
+ // (resource, hash, contract) seal evidence may skip an upload — the
2025
+ // sealed-first upload asks that exact question itself and moves only
2026
+ // what is unsealed, so Register always re-establishes cargo closure
2027
+ // before returning, even when the graph itself is unchanged.
2028
+ const cargoRecords = snapshot.records.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
1937
2029
  if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
1938
2030
  // The graph is already committed; only cargo can be missing. Repair
1939
2031
  // uploads need no journal row — a crash simply re-runs the same
1940
2032
  // repair on the next Register.
1941
- if (uploadRecords.length > 0) {
2033
+ if (cargoRecords.length > 0) {
1942
2034
  try {
1943
- await this.uploadSnapshotContent(identity, state.resourceId, uploadRecords, trace);
2035
+ await this.uploadSnapshotContent(identity, state.resourceId, cargoRecords, trace, manifestHints);
1944
2036
  }
1945
2037
  catch (error) {
1946
2038
  throw pipelineStageError("cloud upload", error);
@@ -1958,20 +2050,32 @@ export class UserGroundHost {
1958
2050
  return root !== null && replacementRoots.has(root);
1959
2051
  });
1960
2052
  const replacement = rootReplacementFromRecords(replacementRootIds, replacementRecords);
1961
- supersedeSnapshotJournal(this.databasePath(identity), {
2053
+ // The operation's canonical bytes: staged, hashed, and submitted as-is.
2054
+ const operationBlob = Buffer.from(stableJson(replacement), "utf8");
2055
+ const entry = {
1962
2056
  mutationId: randomUUID(),
1963
2057
  resourceId: state.resourceId,
1964
2058
  authorityEpoch: state.authorityEpoch,
1965
2059
  baseVersion: state.headVersion,
1966
2060
  snapshotChecksum: checksum,
1967
- snapshotJson,
1968
2061
  deliveryWatermark,
1969
- uploadRecords,
2062
+ operationKind: replacement.kind,
1970
2063
  replacementRootIds: replacement.rootIds,
1971
- replacementRecords: replacement.records,
2064
+ uploadUuids: cargoRecords.map((record) => record.uuid),
2065
+ snapshotBlob,
2066
+ operationBlob,
1972
2067
  status: "pending",
1973
- });
1974
- await this.publishPendingSnapshots(identity, trace);
2068
+ submitted: false,
2069
+ };
2070
+ supersedeSnapshotJournal(this.databasePath(identity), entry);
2071
+ // Publish the freshly journaled intent directly: the records are already
2072
+ // in memory and the operation bytes already exist — nothing is re-read,
2073
+ // re-parsed, or re-serialized between prepare and the wire.
2074
+ await this.publishLoadedSnapshot(identity, {
2075
+ ...entry,
2076
+ snapshotRecords: snapshot.records,
2077
+ uploadRecords: cargoRecords,
2078
+ }, trace, manifestHints);
1975
2079
  }
1976
2080
  /** Which of these heads does the authority hold sealed under the current
1977
2081
  * content contract? Batched so a whole workspace costs a handful of
@@ -1996,84 +2100,101 @@ export class UserGroundHost {
1996
2100
  }
1997
2101
  return sealed;
1998
2102
  }
1999
- async publishPendingSnapshots(identity, trace) {
2103
+ async publishPendingSnapshots(identity, trace, manifestHints) {
2000
2104
  for (const pending of readSnapshotPublications(this.databasePath(identity))) {
2001
2105
  // A rejected row is a settled fact, not an intent: the authority
2002
2106
  // conclusively refused this exact mutation, and only a newer complete
2003
2107
  // snapshot superseding it can publish this resource again.
2004
2108
  if (pending.status === "rejected")
2005
2109
  continue;
2006
- const snapshot = snapshotFromRecords(JSON.parse(pending.snapshotJson).records || []);
2007
- const replacement = rootReplacementFromRecords(pending.replacementRootIds, pending.replacementRecords);
2008
- try {
2009
- await this.uploadSnapshotContent(identity, pending.resourceId, pending.uploadRecords, trace);
2010
- }
2011
- catch (error) {
2012
- throw pipelineStageError("cloud upload", error);
2013
- }
2014
- let frame;
2015
- const graphCommitStarted = performance.now();
2016
- if (trace?.journey === "register") {
2017
- this.stage({
2018
- primitive: "register",
2019
- stage: "cloud-graph-commit",
2020
- status: "started",
2021
- ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2022
- ...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
2023
- });
2024
- }
2025
- try {
2026
- frame = await submitMutationOperation(this.wire, {
2027
- resourceId: pending.resourceId,
2028
- authorityEpoch: pending.authorityEpoch,
2029
- mutationId: pending.mutationId,
2030
- deviceId: identity.deviceId,
2031
- baseVersion: pending.baseVersion,
2032
- contract: ENTITY_CLOUD_CONTRACT,
2033
- schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2034
- operationKind: replacement.kind,
2035
- deliveryWatermark: pending.deliveryWatermark,
2036
- }, replacement);
2037
- }
2038
- catch (error) {
2039
- if (trace?.journey === "register") {
2040
- this.stage({
2041
- primitive: "register",
2042
- stage: "cloud-graph-commit",
2043
- status: "failed",
2044
- ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2045
- durationMs: performance.now() - graphCommitStarted,
2046
- });
2047
- }
2048
- // A non-retryable error frame is the authority's answer: this exact
2049
- // mutation is refused forever. Settle the row so it is never
2050
- // replayed and a newer snapshot may supersede it. Every other
2051
- // failure leaves the outcome unknown and the row pending.
2052
- if (error instanceof WireRequestError && error.retryable === false) {
2053
- markSnapshotJournalRejected(this.databasePath(identity), pending.mutationId);
2054
- }
2055
- throw pipelineStageError("cloud graph commit", error);
2110
+ await this.publishLoadedSnapshot(identity, loadSnapshotPublication(pending), trace, manifestHints);
2111
+ }
2112
+ }
2113
+ async publishLoadedSnapshot(identity, publication, trace, manifestHints) {
2114
+ try {
2115
+ await this.uploadSnapshotContent(identity, publication.resourceId, publication.uploadRecords, trace, manifestHints);
2116
+ }
2117
+ catch (error) {
2118
+ // An unsubmitted pending row is a local intent the authority has
2119
+ // never seen. When its captured ground moved before upload, the
2120
+ // intent is simply stale: delete it and let the caller re-derive
2121
+ // one from current ground. A submitted row never lands here — its
2122
+ // upload sealed completely before the first submit frame left.
2123
+ if (staleCapture(error) && !publication.submitted) {
2124
+ deleteJournalEntry(this.databasePath(identity), publication.mutationId);
2056
2125
  }
2126
+ throw pipelineStageError("cloud upload", error);
2127
+ }
2128
+ let frame;
2129
+ const graphCommitStarted = performance.now();
2130
+ if (trace?.journey === "register") {
2131
+ this.stage({
2132
+ primitive: "register",
2133
+ stage: "cloud-graph-commit",
2134
+ status: "started",
2135
+ ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2136
+ ...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
2137
+ });
2138
+ }
2139
+ if (!publication.submitted) {
2140
+ markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId);
2141
+ }
2142
+ const legMs = {};
2143
+ try {
2144
+ frame = await submitMutationOperation(this.wire, {
2145
+ resourceId: publication.resourceId,
2146
+ authorityEpoch: publication.authorityEpoch,
2147
+ mutationId: publication.mutationId,
2148
+ deviceId: identity.deviceId,
2149
+ baseVersion: publication.baseVersion,
2150
+ contract: ENTITY_CLOUD_CONTRACT,
2151
+ schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
2152
+ operationKind: publication.operationKind,
2153
+ deliveryWatermark: publication.deliveryWatermark,
2154
+ }, publication.operationBlob, (leg, durationMs) => { legMs[`${leg}Ms`] = durationMs; });
2155
+ }
2156
+ catch (error) {
2057
2157
  if (trace?.journey === "register") {
2058
2158
  this.stage({
2059
2159
  primitive: "register",
2060
2160
  stage: "cloud-graph-commit",
2061
- status: "completed",
2161
+ status: "failed",
2062
2162
  ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2063
2163
  durationMs: performance.now() - graphCommitStarted,
2064
- measurements: { version: Number(frame.version) },
2065
2164
  });
2066
2165
  }
2067
- this.cloudState = {
2068
- resourceId: pending.resourceId,
2069
- authorityEpoch: Number(frame.authority_epoch),
2070
- headVersion: Number(frame.version),
2071
- checksum: pending.snapshotChecksum,
2072
- deliveryWatermark: Number(frame.delivery_watermark),
2073
- records: snapshot.records,
2074
- };
2075
- deleteJournalEntry(this.databasePath(identity), pending.mutationId);
2166
+ // A non-retryable error frame is the authority's answer: this exact
2167
+ // mutation is refused forever. Settle the row so it is never
2168
+ // replayed and a newer snapshot may supersede it. Every other
2169
+ // failure leaves the outcome unknown and the row pending.
2170
+ if (error instanceof WireRequestError && error.retryable === false) {
2171
+ markSnapshotJournalRejected(this.databasePath(identity), publication.mutationId);
2172
+ }
2173
+ throw pipelineStageError("cloud graph commit", error);
2076
2174
  }
2175
+ if (trace?.journey === "register") {
2176
+ this.stage({
2177
+ primitive: "register",
2178
+ stage: "cloud-graph-commit",
2179
+ status: "completed",
2180
+ ...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
2181
+ durationMs: performance.now() - graphCommitStarted,
2182
+ measurements: {
2183
+ version: Number(frame.version),
2184
+ operationBytes: publication.operationBlob.byteLength,
2185
+ ...legMs,
2186
+ },
2187
+ });
2188
+ }
2189
+ this.cloudState = {
2190
+ resourceId: publication.resourceId,
2191
+ authorityEpoch: Number(frame.authority_epoch),
2192
+ headVersion: Number(frame.version),
2193
+ checksum: publication.snapshotChecksum,
2194
+ deliveryWatermark: Number(frame.delivery_watermark),
2195
+ records: publication.snapshotRecords,
2196
+ };
2197
+ deleteJournalEntry(this.databasePath(identity), publication.mutationId);
2077
2198
  }
2078
2199
  }
2079
2200
  const sha256Hex = (input) => createHash("sha256").update(input).digest("hex");
@@ -2165,12 +2286,99 @@ function assertHealthyWatch(evidence, rootId) {
2165
2286
  * file over a dozen times; the file itself is never deleted by the host, so
2166
2287
  * every later open needs only the connection and its pragmas. */
2167
2288
  const migratedDatabases = new Set();
2168
- function initializeDatabase(file) {
2289
+ const SNAPSHOT_PUBLICATIONS_SCHEMA = `
2290
+ CREATE TABLE IF NOT EXISTS snapshot_publications (
2291
+ sequence INTEGER PRIMARY KEY AUTOINCREMENT,
2292
+ mutation_id TEXT NOT NULL UNIQUE,
2293
+ resource_id TEXT NOT NULL,
2294
+ meta_json TEXT NOT NULL,
2295
+ snapshot_blob BLOB NOT NULL,
2296
+ operation_blob BLOB NOT NULL,
2297
+ created_at INTEGER NOT NULL,
2298
+ status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
2299
+ submitted_at INTEGER
2300
+ );`;
2301
+ /** Converts an old-shape publication journal (one nested publication_json
2302
+ * text per row) into the blob journal, atomically, preserving every row's
2303
+ * meaning. A pending row may be a submitted mutation whose outcome exists
2304
+ * only at the authority — dropping it would violate the retry law that
2305
+ * recovers a lost acknowledgement — so conversion is structural: the nested
2306
+ * snapshot text becomes the snapshot blob verbatim, the stored replacement
2307
+ * becomes the operation blob, and semantic judgment stays where it always
2308
+ * was, with the reader (an unlawful row still blocks registration rather
2309
+ * than being silently replaced).
2310
+ *
2311
+ * The transform runs entirely inside SQLite. A production journal row is a
2312
+ * register-scale graph serialized three times over — the exact text whose
2313
+ * repeated JavaScript materialization blew the 512 MB heap — so no part of
2314
+ * it may ever become a JavaScript value here: json_extract lifts the nested
2315
+ * snapshot text straight into its blob, and the meta and operation rows are
2316
+ * json_object over subtrees, all in engine memory outside the V8 heap.
2317
+ *
2318
+ * Tables older than the three-state journal carry no status or submitted_at;
2319
+ * their rows were written before the pre-submit durability mark existed, so
2320
+ * the only safe reading is "possibly submitted": keep them pending and
2321
+ * marked submitted, which replays the same mutation id — always idempotent —
2322
+ * and never supersedes. A row whose publication_json does not parse fails
2323
+ * the open loudly with the journal intact; nothing is ever deleted on a
2324
+ * conversion failure. */
2325
+ function convertLegacySnapshotJournal(database, columns) {
2326
+ database.transaction(() => {
2327
+ database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
2328
+ database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
2329
+ database.exec(`
2330
+ INSERT INTO snapshot_publications(
2331
+ mutation_id, resource_id, meta_json, snapshot_blob, operation_blob,
2332
+ created_at, status, submitted_at
2333
+ )
2334
+ SELECT
2335
+ mutation_id,
2336
+ resource_id,
2337
+ json_object(
2338
+ 'authorityEpoch', json_extract(publication_json, '$.authorityEpoch'),
2339
+ 'baseVersion', json_extract(publication_json, '$.baseVersion'),
2340
+ 'deliveryWatermark', json_extract(publication_json, '$.deliveryWatermark'),
2341
+ 'snapshotChecksum', json_extract(publication_json, '$.snapshotChecksum'),
2342
+ 'operationKind', 'entity.roots.replace',
2343
+ 'replacementRootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2344
+ 'uploadUuids', coalesce((
2345
+ SELECT json_group_array(json_extract(uploaded.value, '$.uuid'))
2346
+ FROM json_each(publication_json, '$.uploadRecords') AS uploaded
2347
+ ), json_array())
2348
+ ),
2349
+ CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
2350
+ CAST(json_object(
2351
+ 'kind', 'entity.roots.replace',
2352
+ 'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
2353
+ 'records', json(json_extract(publication_json, '$.replacementRecords'))
2354
+ ) AS BLOB),
2355
+ created_at,
2356
+ ${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
2357
+ ${columns.has("submitted_at") ? "submitted_at" : "created_at"}
2358
+ FROM snapshot_publications_legacy ORDER BY sequence
2359
+ `);
2360
+ database.exec("DROP TABLE snapshot_publications_legacy");
2361
+ })();
2362
+ }
2363
+ /** The single opener and schema-migration entry for a user ground database.
2364
+ * Exported so the upgrade path is provable in isolation — the journal
2365
+ * conversion law test opens a legacy file through this exact entry inside a
2366
+ * heap-capped child process. */
2367
+ export function initializeDatabase(file) {
2169
2368
  ensurePrivateDir(dirname(file));
2170
2369
  const database = new Database(file);
2171
2370
  database.pragma("busy_timeout = 5000");
2172
2371
  database.pragma("journal_mode = WAL");
2173
2372
  database.pragma("synchronous = FULL");
2373
+ // The legacy probe runs on every open, before the migration memo: a journal
2374
+ // row may be a submitted publication whose outcome exists only at the
2375
+ // authority, so an old-shape table must convert — never drop — no matter
2376
+ // how this file arrived at this process. One PRAGMA against the in-memory
2377
+ // catalog costs nothing next to the connection open itself.
2378
+ const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
2379
+ if (publicationColumns.has("publication_json")) {
2380
+ convertLegacySnapshotJournal(database, publicationColumns);
2381
+ }
2174
2382
  if (migratedDatabases.has(file))
2175
2383
  return database;
2176
2384
  database.exec(`
@@ -2262,14 +2470,7 @@ function initializeDatabase(file) {
2262
2470
  authority_id TEXT PRIMARY KEY,
2263
2471
  delivery_sequence INTEGER NOT NULL CHECK(delivery_sequence >= 0)
2264
2472
  );
2265
- CREATE TABLE IF NOT EXISTS snapshot_publications (
2266
- sequence INTEGER PRIMARY KEY AUTOINCREMENT,
2267
- mutation_id TEXT NOT NULL UNIQUE,
2268
- resource_id TEXT NOT NULL,
2269
- publication_json TEXT NOT NULL,
2270
- created_at INTEGER NOT NULL,
2271
- status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected'))
2272
- );
2473
+ ${SNAPSHOT_PUBLICATIONS_SCHEMA}
2273
2474
  CREATE TABLE IF NOT EXISTS workspace_add_intents (
2274
2475
  workspace_uuid TEXT PRIMARY KEY,
2275
2476
  destination_path TEXT NOT NULL UNIQUE,
@@ -2305,10 +2506,6 @@ function initializeDatabase(file) {
2305
2506
  if (!inboxColumns.has("resolved_at")) {
2306
2507
  database.exec("ALTER TABLE record_inbox ADD COLUMN resolved_at INTEGER");
2307
2508
  }
2308
- const publicationColumns = new Set(database.prepare("PRAGMA table_info(snapshot_publications)").all().map(({ name }) => name));
2309
- if (!publicationColumns.has("status")) {
2310
- database.exec("ALTER TABLE snapshot_publications ADD COLUMN status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected'))");
2311
- }
2312
2509
  database.exec("DROP INDEX IF EXISTS record_inbox_delivery");
2313
2510
  database.exec("DROP TABLE IF EXISTS mutation_journal");
2314
2511
  migratedDatabases.add(file);
@@ -2350,6 +2547,31 @@ function readGroundRowsForRoots(file, table, rootUUIDs) {
2350
2547
  database.close();
2351
2548
  }
2352
2549
  }
2550
+ /** Exact ground rows for a UUID set, batched under SQLite's parameter bound.
2551
+ * Upload resolves each traveling artifact's source path and recorded size
2552
+ * from these rows — content bytes live in the ground itself, never in a
2553
+ * staged register copy. */
2554
+ function readGroundRowsByUuid(file, uuids) {
2555
+ if (!existsSync(file) || uuids.length === 0)
2556
+ return new Map();
2557
+ const database = initializeDatabase(file);
2558
+ try {
2559
+ const found = new Map();
2560
+ for (let start = 0; start < uuids.length; start += 500) {
2561
+ const batch = uuids.slice(start, start + 500);
2562
+ const rows = database.prepare(`
2563
+ SELECT ${GROUND_ROW_PROJECTION}
2564
+ FROM entities WHERE uuid IN (${batch.map(() => "?").join(", ")})
2565
+ `).all(...batch);
2566
+ for (const row of rows)
2567
+ found.set(row.uuid, row);
2568
+ }
2569
+ return found;
2570
+ }
2571
+ finally {
2572
+ database.close();
2573
+ }
2574
+ }
2353
2575
  /** Type and UUID settle locally even when immutable repository capture must
2354
2576
  * retry. Those rows are identity evidence for the next observation, while
2355
2577
  * their absence from the notebook still means no verified logical result has
@@ -2734,33 +2956,36 @@ function readSnapshotPublications(file) {
2734
2956
  try {
2735
2957
  return database.prepare(`
2736
2958
  SELECT mutation_id AS mutationId, resource_id AS resourceId,
2737
- publication_json AS publicationJson, status
2959
+ meta_json AS metaJson, snapshot_blob AS snapshotBlob,
2960
+ operation_blob AS operationBlob, status,
2961
+ submitted_at AS submittedAt
2738
2962
  FROM snapshot_publications ORDER BY sequence
2739
2963
  `).all().map((row) => {
2740
- const publication = JSON.parse(row.publicationJson);
2741
- if (!Array.isArray(publication.uploadRecords)) {
2742
- throw new Error(`snapshot publication ${row.mutationId} has no exact upload subset`);
2743
- }
2744
- if (!Array.isArray(publication.replacementRootIds)
2745
- || !Array.isArray(publication.replacementRecords)) {
2964
+ const meta = JSON.parse(row.metaJson);
2965
+ // No lawful publication carries zero roots — rootReplacementFromRecords
2966
+ // refuses them at build time, so a rootless row here is damage.
2967
+ if (!Array.isArray(meta.replacementRootIds) || meta.replacementRootIds.length === 0
2968
+ || !Array.isArray(meta.uploadUuids) || meta.operationKind !== "entity.roots.replace") {
2746
2969
  throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
2747
2970
  }
2748
- const deliveryWatermark = Number(publication.deliveryWatermark);
2971
+ const deliveryWatermark = Number(meta.deliveryWatermark);
2749
2972
  if (!Number.isSafeInteger(deliveryWatermark) || deliveryWatermark < 0) {
2750
2973
  throw new Error(`snapshot publication ${row.mutationId} has no delivery watermark`);
2751
2974
  }
2752
2975
  return {
2753
2976
  mutationId: row.mutationId,
2754
2977
  resourceId: row.resourceId,
2755
- authorityEpoch: Number(publication.authorityEpoch),
2756
- baseVersion: Number(publication.baseVersion),
2978
+ authorityEpoch: Number(meta.authorityEpoch),
2979
+ baseVersion: Number(meta.baseVersion),
2757
2980
  deliveryWatermark,
2758
- snapshotChecksum: String(publication.snapshotChecksum),
2759
- snapshotJson: String(publication.snapshotJson),
2760
- uploadRecords: snapshotFromRecords(publication.uploadRecords).records,
2761
- replacementRootIds: publication.replacementRootIds.map(String),
2762
- replacementRecords: snapshotFromRecords(publication.replacementRecords).records,
2981
+ snapshotChecksum: String(meta.snapshotChecksum),
2982
+ operationKind: meta.operationKind,
2983
+ replacementRootIds: meta.replacementRootIds.map(String),
2984
+ uploadUuids: meta.uploadUuids.map(String),
2985
+ snapshotBlob: row.snapshotBlob,
2986
+ operationBlob: row.operationBlob,
2763
2987
  status: row.status === "rejected" ? "rejected" : "pending",
2988
+ submitted: row.submittedAt !== null,
2764
2989
  };
2765
2990
  });
2766
2991
  }
@@ -2904,47 +3129,61 @@ function commitDetectedState(file, input) {
2904
3129
  }
2905
3130
  }
2906
3131
  /** Journal the newest snapshot publication for its resource. A journal row
2907
- * has exactly two meanings: `pending` — the submit outcome is unknown, so
2908
- * the same mutation id must retry until the authority answers (the mutation
2909
- * may already be accepted with only the acknowledgement lost); `rejected` —
2910
- * the authority conclusively refused it, so it is never replayed. Only a
2911
- * rejected row may be superseded — last write wins — and supersession and
2912
- * insertion share one transaction, so a crash leaves exactly one durable
2913
- * intent. Content uploads are content-addressed and idempotent, so a
2914
- * superseded row loses no uploaded work. */
3132
+ * has exactly three meanings: `pending` and never submitted — a local intent
3133
+ * the authority has never seen, superseded freely (last write wins);
3134
+ * `pending` and submitted — the submit outcome is unknown, so the same
3135
+ * mutation id must retry until the authority answers (the mutation may
3136
+ * already be accepted with only the acknowledgement lost) and nothing may
3137
+ * supersede it; `rejected` — the authority conclusively refused it, so it is
3138
+ * never replayed and awaits supersession. Supersession and insertion share
3139
+ * one transaction, so a crash leaves exactly one durable intent. Content
3140
+ * uploads are content-addressed and idempotent, so a superseded row loses no
3141
+ * uploaded work. */
2915
3142
  function supersedeSnapshotJournal(file, entry) {
2916
3143
  const database = initializeDatabase(file);
2917
3144
  try {
2918
3145
  database.transaction(() => {
2919
3146
  const unresolved = database.prepare(`
2920
3147
  SELECT 1 AS present FROM snapshot_publications
2921
- WHERE resource_id = ? AND status = 'pending' LIMIT 1
3148
+ WHERE resource_id = ? AND status = 'pending' AND submitted_at IS NOT NULL LIMIT 1
2922
3149
  `).get(entry.resourceId);
2923
3150
  if (unresolved) {
2924
- throw new Error("a pending publication with an unknown outcome must resolve before supersession");
3151
+ throw new Error("a submitted publication with an unknown outcome must resolve before supersession");
2925
3152
  }
2926
- database.prepare("DELETE FROM snapshot_publications WHERE resource_id = ? AND status = 'rejected'")
3153
+ database.prepare("DELETE FROM snapshot_publications WHERE resource_id = ?")
2927
3154
  .run(entry.resourceId);
2928
3155
  database.prepare(`
2929
3156
  INSERT INTO snapshot_publications(
2930
- mutation_id, resource_id, publication_json, created_at
2931
- ) VALUES (?, ?, ?, ?)
3157
+ mutation_id, resource_id, meta_json, snapshot_blob, operation_blob, created_at
3158
+ ) VALUES (?, ?, ?, ?, ?, ?)
2932
3159
  `).run(entry.mutationId, entry.resourceId, stableJson({
2933
3160
  authorityEpoch: entry.authorityEpoch,
2934
3161
  baseVersion: entry.baseVersion,
2935
3162
  deliveryWatermark: entry.deliveryWatermark,
2936
3163
  snapshotChecksum: entry.snapshotChecksum,
2937
- snapshotJson: entry.snapshotJson,
2938
- uploadRecords: entry.uploadRecords,
3164
+ operationKind: entry.operationKind,
2939
3165
  replacementRootIds: entry.replacementRootIds,
2940
- replacementRecords: entry.replacementRecords,
2941
- }), Date.now());
3166
+ uploadUuids: entry.uploadUuids,
3167
+ }), entry.snapshotBlob, entry.operationBlob, Date.now());
2942
3168
  })();
2943
3169
  }
2944
3170
  finally {
2945
3171
  database.close();
2946
3172
  }
2947
3173
  }
3174
+ /** Durably declare a submit attempt before the first frame leaves. From this
3175
+ * moment the row's outcome may exist only at the authority, so it must retry
3176
+ * the same mutation id and may never be superseded or rebuilt locally. */
3177
+ function markSnapshotJournalSubmitted(file, mutationId) {
3178
+ const database = initializeDatabase(file);
3179
+ try {
3180
+ database.prepare("UPDATE snapshot_publications SET submitted_at = ? WHERE mutation_id = ?")
3181
+ .run(Date.now(), mutationId);
3182
+ }
3183
+ finally {
3184
+ database.close();
3185
+ }
3186
+ }
2948
3187
  /** Settle a journal row as conclusively refused. It is never replayed again;
2949
3188
  * a newer complete snapshot supersedes it on the next publication. */
2950
3189
  function markSnapshotJournalRejected(file, mutationId) {
@@ -2966,6 +3205,32 @@ function conclusivelyRefused(error) {
2966
3205
  return error.retryable === false;
2967
3206
  return error instanceof Error && conclusivelyRefused(error.cause);
2968
3207
  }
3208
+ /** Ground moved between capture and upload: a source no longer yields the
3209
+ * bytes its recorded content head names, so the publication built on that
3210
+ * capture can never complete. It was never submitted, so the lawful recovery
3211
+ * is a fresh walk — capture current ground and publish that truth instead. */
3212
+ class StaleCaptureError extends Error {
3213
+ constructor(detail, options) {
3214
+ super(`captured ground moved before upload: ${detail}`, options);
3215
+ }
3216
+ }
3217
+ function staleCapture(error) {
3218
+ if (error instanceof StaleCaptureError)
3219
+ return true;
3220
+ return error instanceof Error && staleCapture(error.cause);
3221
+ }
3222
+ /** Local read evidence disagreeing with the recorded content head — the
3223
+ * upload law's typed mismatch or the source file being gone entirely. */
3224
+ function sourceMismatch(error) {
3225
+ const code = error?.code;
3226
+ return code === "UPLOAD_SOURCE_MISMATCH" || code === "ENOENT";
3227
+ }
3228
+ /** The walked file moved while its bytes were being captured — ordinary
3229
+ * ground behavior, answered by recapturing what stands there now. */
3230
+ function captureChurn(error) {
3231
+ const code = error?.code;
3232
+ return code === "CAPTURE_SOURCE_CHANGED" || code === "ENOENT";
3233
+ }
2969
3234
  function deleteJournalEntry(file, mutationId) {
2970
3235
  const database = initializeDatabase(file);
2971
3236
  try {
@@ -3436,6 +3701,7 @@ async function reconcileGround(input) {
3436
3701
  const notebookRemovals = new Set();
3437
3702
  const detection = [];
3438
3703
  const transferFailures = [];
3704
+ const largeManifests = new Map();
3439
3705
  const scan = async (parameters) => {
3440
3706
  const counters = {
3441
3707
  entries: 0,
@@ -3513,6 +3779,8 @@ async function reconcileGround(input) {
3513
3779
  : coreSuspicion,
3514
3780
  existingRows: existing.filter((row) => row.rootUUID === coreUUID),
3515
3781
  suspicion: observationFor(userRoot),
3782
+ plansSendRecords: suspicions !== undefined,
3783
+ largeManifests,
3516
3784
  });
3517
3785
  }
3518
3786
  const physicalRoots = registrationOwner ? [registrationOwner] : outerBoundRoots;
@@ -3548,6 +3816,11 @@ async function reconcileGround(input) {
3548
3816
  existingRows: existingWorkspaceRows,
3549
3817
  registeredBoundaries,
3550
3818
  suspicion: observationFor(rootPath),
3819
+ // A register scan publishes one complete snapshot; it never mints Send
3820
+ // records for the same initial contents, so it plans no detection and
3821
+ // stages no cache bytes — upload reads the ground itself.
3822
+ plansSendRecords: suspicions !== undefined && registrationWorkspaceId === undefined,
3823
+ largeManifests,
3551
3824
  });
3552
3825
  }
3553
3826
  const changedUUIDs = new Set([
@@ -3571,10 +3844,11 @@ async function reconcileGround(input) {
3571
3844
  notebookRemovals: [...notebookRemovals],
3572
3845
  publicationRootIds: selectedRoots ? [...selectedRoots] : null,
3573
3846
  transferFailures,
3847
+ largeManifests,
3574
3848
  };
3575
3849
  }
3576
3850
  async function reconcileRoot(input) {
3577
- const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, } = input;
3851
+ const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, plansSendRecords, largeManifests, } = input;
3578
3852
  // Prior rows reach this scan from every binding this root encloses. A row's
3579
3853
  // stored relativePath is relative to the root that last committed it, which
3580
3854
  // is a different frame when an enclosing boundary was just registered
@@ -3614,7 +3888,7 @@ async function reconcileRoot(input) {
3614
3888
  transferable: true,
3615
3889
  transferFailure: null,
3616
3890
  notebookRemovals: [],
3617
- detection: input.suspicion ? planGroundDetection({
3891
+ detection: input.suspicion && plansSendRecords ? planGroundDetection({
3618
3892
  rootId: rootUUID,
3619
3893
  suspicion: input.suspicion,
3620
3894
  before: existingRows.map(portableRecord),
@@ -3742,6 +4016,11 @@ async function reconcileRoot(input) {
3742
4016
  return [...new Set(paths)].sort();
3743
4017
  };
3744
4018
  let visitedEntries = 0;
4019
+ // File captures overlap reads and hashing across a bounded pool while the
4020
+ // directory traversal itself stays ordered and serial. Each task settles
4021
+ // its own entry in place; every task is awaited before entries are read.
4022
+ const captureTasks = [];
4023
+ const captureGate = concurrencyGate(WALK_CAPTURE_CONCURRENCY);
3745
4024
  const visit = async (directory, parentUUID, parentType, base = "", activeRepository = parentType === "repo.git"
3746
4025
  ? {
3747
4026
  root: directory,
@@ -3817,6 +4096,7 @@ async function reconcileRoot(input) {
3817
4096
  let type;
3818
4097
  let payloadVersion = null;
3819
4098
  let linkTarget;
4099
+ let deferredCapture = false;
3820
4100
  let contentVerifiedAtMs = existing?.contentVerifiedAtMs ?? null;
3821
4101
  if (!observe && existing && !stats.isDirectory()) {
3822
4102
  type = existing.type;
@@ -3842,7 +4122,7 @@ async function reconcileRoot(input) {
3842
4122
  linkTarget = readlinkSync(absolutePath);
3843
4123
  const bytes = Buffer.from(linkTarget, "utf8");
3844
4124
  payloadVersion = sha256Hex(bytes);
3845
- if (!activeRepository)
4125
+ if (!activeRepository && plansSendRecords)
3846
4126
  sealCapturedArtifact(cacheDir, {
3847
4127
  entityId: uuid,
3848
4128
  entityType: type,
@@ -3862,9 +4142,10 @@ async function reconcileRoot(input) {
3862
4142
  }
3863
4143
  else {
3864
4144
  evidence && (evidence.contentReads += 1);
3865
- const captured = await captureContentFile(absolutePath, stats, activeRepository ? null : cacheDir);
3866
- type = classifyFile({ binary: captured.binary });
3867
- payloadVersion = captured.contentHash;
4145
+ // Provisional identity; the pooled capture scheduled below settles
4146
+ // type, content head, and fingerprint before entries are read.
4147
+ type = "file.binary";
4148
+ deferredCapture = true;
3868
4149
  }
3869
4150
  }
3870
4151
  else {
@@ -3886,6 +4167,44 @@ async function reconcileRoot(input) {
3886
4167
  ...(linkTarget !== undefined ? { linkTarget } : {}),
3887
4168
  ...statFingerprint(stats),
3888
4169
  });
4170
+ if (deferredCapture) {
4171
+ const entry = entries[entries.length - 1];
4172
+ captureTasks.push(captureGate(async () => {
4173
+ let current = stats;
4174
+ for (let attempt = 1;; attempt++) {
4175
+ try {
4176
+ const captured = await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir);
4177
+ entry.type = classifyFile({ binary: captured.binary });
4178
+ entry.payloadVersion = captured.contentHash;
4179
+ entry.contentVerifiedAtMs = Date.now();
4180
+ Object.assign(entry, statFingerprint(current));
4181
+ if (captured.bytes > CHUNK_BYTES) {
4182
+ largeManifests?.set(captured.contentHash, captured.manifest);
4183
+ }
4184
+ return;
4185
+ }
4186
+ catch (error) {
4187
+ if (!captureChurn(error))
4188
+ throw error;
4189
+ if (attempt >= REGISTER_STALE_CAPTURE_ATTEMPTS) {
4190
+ throw new StaleCaptureError(`${absolutePath} kept changing while being captured`, { cause: error });
4191
+ }
4192
+ // The file moved mid-read. Capture what stands there now —
4193
+ // the walk publishes point-in-time truth, not the first bytes
4194
+ // it happened to observe.
4195
+ try {
4196
+ current = lstatSync(absolutePath);
4197
+ }
4198
+ catch (statError) {
4199
+ throw new StaleCaptureError(`${absolutePath} disappeared during capture`, { cause: statError });
4200
+ }
4201
+ if (!current.isFile()) {
4202
+ throw new StaleCaptureError(`${absolutePath} is no longer a file`, { cause: error });
4203
+ }
4204
+ }
4205
+ }
4206
+ }));
4207
+ }
3889
4208
  if (stats.isDirectory() && !stats.isSymbolicLink()) {
3890
4209
  const childRailChanged = existing !== undefined && existing.type !== type;
3891
4210
  await visit(absolutePath, uuid, type, relativePath, type === "repo.git"
@@ -3899,6 +4218,7 @@ async function reconcileRoot(input) {
3899
4218
  }
3900
4219
  };
3901
4220
  await visit(rootPath, rootUUID, rootType);
4221
+ await Promise.all(captureTasks);
3902
4222
  // When every entry sits at its stored address with its stored type and
3903
4223
  // physical fingerprint (honoring any fixed UUID), path-first claiming makes
3904
4224
  // the multi-pass reconciliation the identity map — resolve directly. Any
@@ -4285,7 +4605,9 @@ async function reconcileRoot(input) {
4285
4605
  const before = existingRows.map(portableRecord);
4286
4606
  const beforeByUuid = new Map(before.map((record) => [record.uuid, record]));
4287
4607
  const entryByUuid = new Map(entries.map((entry) => [entry.uuid, entry]));
4288
- const detected = rows.map((row) => {
4608
+ // Replays exist to mint Send records; a scan that plans none builds none —
4609
+ // and its walk staged no cache bytes for replays to read.
4610
+ const detected = !plansSendRecords ? [] : rows.map((row) => {
4289
4611
  const base = beforeByUuid.get(row.record.uuid) ?? null;
4290
4612
  const entry = entryByUuid.get(row.record.uuid);
4291
4613
  if (row.record.type !== "repo.git" && entry && territory.ownerOf(entry)) {
@@ -4380,7 +4702,7 @@ async function reconcileRoot(input) {
4380
4702
  transferable: true,
4381
4703
  transferFailure: null,
4382
4704
  notebookRemovals,
4383
- detection: input.suspicion ? planGroundDetection({
4705
+ detection: input.suspicion && plansSendRecords ? planGroundDetection({
4384
4706
  rootId: rootUUID,
4385
4707
  suspicion: input.suspicion,
4386
4708
  before: plannedBefore,
@@ -5087,4 +5409,25 @@ async function boundedForEach(items, concurrency, perform) {
5087
5409
  };
5088
5410
  await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
5089
5411
  }
5412
+ /** Admit queued async tasks with at most `limit` in flight. Scheduling never
5413
+ * blocks the caller; each task's settlement is its returned promise. The
5414
+ * admitted task re-checks the bound because a released slot may be claimed
5415
+ * synchronously before a woken waiter resumes. */
5416
+ function concurrencyGate(limit) {
5417
+ let active = 0;
5418
+ const waiting = [];
5419
+ return async (task) => {
5420
+ while (active >= limit) {
5421
+ await new Promise((grant) => waiting.push(grant));
5422
+ }
5423
+ active += 1;
5424
+ try {
5425
+ return await task();
5426
+ }
5427
+ finally {
5428
+ active -= 1;
5429
+ waiting.shift()?.();
5430
+ }
5431
+ };
5432
+ }
5090
5433
  //# sourceMappingURL=user-ground-host.js.map