@amalgm/shell 0.1.105 → 0.1.107

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ import { existsSync, lstatSync, realpathSync, readFileSync, readdirSync, readlin
3
3
  import { open } from "node:fs/promises";
4
4
  import { basename, dirname, extname, isAbsolute, join, relative, resolve, sep } from "node:path";
5
5
  import { buildUserHomeManifest, buildUserManifest, liveMachineStateDir, scopedAmalgmDir, shippedUserHomeDeclaration, } from "@amalgm/core/identity";
6
- import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
6
+ import { CHUNK_BYTES, CONTENT_CONTRACT, INLINE_CARGO_MAX_BYTES, ENTITY_CLOUD_CONTRACT, ENTITY_CLOUD_SCHEMA_VERSION, artifactForRecord, canonicalVersion, checkContentManifest, classifyDirectory, classifyFile, classifyRegisteredRoot, createLocalEntityRecord, createEntityRecordAuthorityPort, convergeUserGround, decodeAcceptedEntityRecord, createUserGroundEnrollmentPolicy, isRepositoryMetadataEntry, indexRepositoryTerritory, downloadAll as downloadAllArtifacts, encodeEntityRecord, EntityApplyRail, EntityRecordRail, membershipHash, parseSnapshot, pathIsSuspect, PRESIGN_BATCH_OBJECTS, privateEntityResourceId, repositoryIdentityHash, rootReplacementFromRecords, sameRecords, snapshotFromRecords, stableJson, travelingRecords, upload as uploadArtifact, userGroundRecords, } from "@amalgm/live";
7
7
  import Database from "better-sqlite3";
8
8
  import { atomicCopy, atomicWrite, ensurePrivateDir, ensureUserDir } from "./filesystem.js";
9
9
  import { emitFilesStage, } from "./files-observability.js";
@@ -25,6 +25,7 @@ import { projectMaterializedGraph } from "./materialized-graph.js";
25
25
  import { NodeWatchHost, } from "./watching/index.js";
26
26
  import { GroundCoordinator } from "./ground-coordination.js";
27
27
  import { submitMutationOperation } from "./paged-mutation-submit.js";
28
+ import { adoptSnapshotDeliveryBaseline } from "./snapshot-delivery-baseline.js";
28
29
  import { WireClient, WireRequestError } from "./wire-client.js";
29
30
  const SMALL_CONTENT_UPLOAD_CONCURRENCY = 16;
30
31
  /** Storage-object lanes for one download set. Tiny objects use the wide
@@ -259,13 +260,31 @@ export class UserGroundHost {
259
260
  const cloud = await this.cloudPort(identity).lookup(resourceId);
260
261
  if (!cloud)
261
262
  throw new Error("this user has no cloud entity registry");
263
+ // The authority's active snapshot is the compact base; Add's active
264
+ // materialization head is that snapshot plus the complete committed
265
+ // tail this machine has now received. Fold the tail before Live
266
+ // selects the workspace so a destination created after mutations is
267
+ // exact immediately, rather than first installing stale ground and
268
+ // racing Apply afterward.
269
+ await this.recordRail?.flush();
270
+ const deliveryWatermark = readAuthorityDeliveryCursor(this.databasePath(identity), resourceId);
271
+ if (deliveryWatermark < cloud.deliveryWatermark) {
272
+ throw new Error("local delivery cursor is behind the active snapshot watermark");
273
+ }
274
+ const snapshot = snapshotFromRecords(recordsAtDeliveryPrefix(cloud.snapshot.records, readAcceptedResults(this.databasePath(identity), resourceId, cloud.deliveryWatermark, deliveryWatermark)));
275
+ const current = {
276
+ ...cloud,
277
+ deliveryWatermark,
278
+ checksum: sha256Hex(stableJson(snapshot)),
279
+ snapshot,
280
+ };
262
281
  this.stage({
263
282
  primitive: "download",
264
283
  stage: "registry-lookup",
265
284
  status: "completed",
266
285
  durationMs: performance.now() - started,
267
286
  });
268
- return cloud;
287
+ return current;
269
288
  },
270
289
  materializeWorkspace: async (input) => {
271
290
  const groundWaitStarted = performance.now();
@@ -298,9 +317,10 @@ export class UserGroundHost {
298
317
  records: input.records,
299
318
  destinationParent: input.destinationParent,
300
319
  cloudHead: input.cloudHead,
320
+ deliveryWatermark: input.deliveryWatermark,
301
321
  readContent: (artifacts) => this.downloadContentSet(privateEntityResourceId(identity.userId, sha256Hex), this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: input.workspace.uuid }),
302
322
  adoptMaterialization: (rootUUID) => {
303
- const settled = this.applyHost?.adoptVerifiedMaterialization(rootUUID) ?? 0;
323
+ const settled = this.adoptCloudDeliveryBaseline(identity, input.deliveryWatermark, rootUUID);
304
324
  this.applyRail?.recordsAvailable();
305
325
  if (settled > 0) {
306
326
  this.stage({
@@ -424,9 +444,10 @@ export class UserGroundHost {
424
444
  records: intent.records,
425
445
  destinationParent: dirname(intent.destinationPath),
426
446
  cloudHead: intent.cloudHead,
447
+ deliveryWatermark: intent.deliveryWatermark,
427
448
  readContent: (artifacts) => this.downloadContentSet(intent.resourceId, this.cacheDir(identity), artifacts.map((artifact) => ({ artifact })), 10, { journey: "add", workspaceId: intent.workspaceId }),
428
449
  adoptMaterialization: (rootUUID) => {
429
- this.applyHost?.adoptVerifiedMaterialization(rootUUID);
450
+ this.adoptCloudDeliveryBaseline(identity, intent.deliveryWatermark, rootUUID);
430
451
  this.applyRail?.recordsAvailable();
431
452
  },
432
453
  onStage: (stage) => this.options.onWorkspaceAddStage?.({
@@ -504,7 +525,8 @@ export class UserGroundHost {
504
525
  // the publish path re-proves seal evidence and repairs any missing
505
526
  // uploads before this answer is lawful.
506
527
  try {
507
- await this.publishMaterializedSnapshot(identity, new Set([workspaceId]), { journey: "register", workspaceId });
528
+ const deliveryWatermark = await this.prepareSnapshotPublication(identity, { journey: "register", workspaceId });
529
+ await this.publishMaterializedSnapshot(identity, new Set([workspaceId]), deliveryWatermark, { journey: "register", workspaceId });
508
530
  }
509
531
  finally {
510
532
  this.finishColdOperation();
@@ -519,6 +541,10 @@ export class UserGroundHost {
519
541
  // Register is a cold declaration, not a filesystem edit. It performs one
520
542
  // complete reconciliation and publishes one graph head; it must not mint
521
543
  // a parallel stream of Detect records for the same initial contents.
544
+ // Freeze the official prefix before taking ground ownership. Apply may
545
+ // need that same ground, so draining it inside the registration scope
546
+ // would make the two rails wait on one another.
547
+ const deliveryWatermark = await this.prepareSnapshotPublication(identity, { journey: "register", workspaceId });
522
548
  let releaseGround = null;
523
549
  try {
524
550
  const groundWaitStarted = performance.now();
@@ -612,7 +638,7 @@ export class UserGroundHost {
612
638
  if (!scanned.publicationRootIds) {
613
639
  throw new Error("cold registration reconciliation did not name its publication roots");
614
640
  }
615
- await this.publishMaterializedSnapshot(identity, new Set(scanned.publicationRootIds), { journey: "register", workspaceId });
641
+ await this.publishMaterializedSnapshot(identity, new Set(scanned.publicationRootIds), deliveryWatermark, { journey: "register", workspaceId });
616
642
  this.watchHost.settle(observations);
617
643
  }
618
644
  catch (error) {
@@ -710,6 +736,20 @@ export class UserGroundHost {
710
736
  if (this.syncing)
711
737
  await this.syncing;
712
738
  }
739
+ /** Install the exact snapshot/tail boundary before Receive or Apply can
740
+ * treat older official history as materialization work. The SQLite cursor,
741
+ * retained Inbox resolution, and Apply baseline all derive from the same N. */
742
+ adoptCloudDeliveryBaseline(identity, deliveryWatermark, rootUUID = null) {
743
+ const database = initializeDatabase(this.databasePath(identity));
744
+ let resolved = 0;
745
+ try {
746
+ resolved = adoptSnapshotDeliveryBaseline(database, privateEntityResourceId(identity.userId, sha256Hex), deliveryWatermark, this.options.now?.().getTime() ?? Date.now());
747
+ }
748
+ finally {
749
+ database.close();
750
+ }
751
+ return resolved + (this.applyHost?.adoptVerifiedMaterialization(rootUUID) ?? 0);
752
+ }
713
753
  async startRecordRail(identity) {
714
754
  if (!this.cloudState)
715
755
  throw new Error("entity Record rail requires cloud authority");
@@ -717,12 +757,13 @@ export class UserGroundHost {
717
757
  throw new Error("entity Record rail already started");
718
758
  await this.wire.open();
719
759
  // One floor, no per-feature gates: this shell and its gateway release
720
- // together, and the gateway ships first. 10 = contract-aware cargo
721
- // closure; an older gateway drops the sealed-evidence frame the upload
722
- // planner depends on, so nothing this shell registers could publish.
723
- if (this.wire.protocolVersion < 10) {
760
+ // together, and the gateway ships first. 11 = snapshot delivery
761
+ // watermarks; an older gateway cannot atomically bind Add's graph to the
762
+ // official entity-record prefix it already includes.
763
+ if (this.wire.protocolVersion < 11) {
724
764
  throw new Error("gateway speaks an older wire protocol than this shell");
725
765
  }
766
+ this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
726
767
  const store = new EntityRecordSqliteStore(initializeDatabase(this.databasePath(identity)));
727
768
  const wireAuthority = createEntityRecordAuthorityPort({
728
769
  request: (frame, acceptedTypes, binaryBody) => binaryBody === undefined
@@ -1760,6 +1801,7 @@ export class UserGroundHost {
1760
1801
  authorityEpoch: first.authorityEpoch,
1761
1802
  headVersion: first.baseVersion,
1762
1803
  checksum: "",
1804
+ deliveryWatermark: first.deliveryWatermark,
1763
1805
  records: snapshotFromRecords(JSON.parse(first.snapshotJson).records || []).records,
1764
1806
  };
1765
1807
  try {
@@ -1778,13 +1820,10 @@ export class UserGroundHost {
1778
1820
  /** Register is the deliberate cold graph-publication boundary. Ordinary
1779
1821
  * Watch detection stops at a durable exact record and never rebuilds this
1780
1822
  * resource snapshot. */
1781
- async publishMaterializedSnapshot(identity, publicationRootIds, trace) {
1782
- // Any durable pending intent resolves before a new snapshot derives: an
1783
- // acknowledged publication advances the head through its original
1784
- // mutation id (the authority answers a known mutation id idempotently),
1785
- // and a conclusive refusal marks the row rejected so the snapshot
1786
- // derived below may supersede it. A transport failure keeps the row
1787
- // pending — its outcome is unknown, so nothing may replace it.
1823
+ async prepareSnapshotPublication(identity, trace) {
1824
+ // An older durable publication resolves before the new boundary derives.
1825
+ // Unknown outcomes remain pending; a conclusive refusal may be replaced
1826
+ // by the newer complete snapshot below.
1788
1827
  try {
1789
1828
  await this.publishPendingSnapshots(identity, trace);
1790
1829
  }
@@ -1792,13 +1831,30 @@ export class UserGroundHost {
1792
1831
  if (!conclusivelyRefused(error))
1793
1832
  throw error;
1794
1833
  }
1834
+ await this.recordRail?.flush();
1795
1835
  const state = this.cloudState;
1796
1836
  if (!state)
1797
1837
  throw new Error("cloud state is unavailable for graph publication");
1838
+ // Freeze N immediately after Receive reaches its explicit boundary. Apply
1839
+ // then settles every Inbox row already known at that boundary. A later
1840
+ // delivery may run concurrently, but the graph still declares the older
1841
+ // N and the authority refuses it if that newer record commits first.
1842
+ const deliveryWatermark = readAuthorityDeliveryCursor(this.databasePath(identity), state.resourceId);
1843
+ if (deliveryWatermark < state.deliveryWatermark) {
1844
+ throw new Error("local delivery cursor is behind the active snapshot watermark");
1845
+ }
1846
+ await this.applyRail?.flush();
1847
+ return deliveryWatermark;
1848
+ }
1849
+ async publishMaterializedSnapshot(identity, publicationRootIds, deliveryWatermark, trace) {
1850
+ const state = this.cloudState;
1851
+ if (!state)
1852
+ throw new Error("cloud state is unavailable for graph publication");
1853
+ const officialRecords = recordsAtDeliveryPrefix(state.records, readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark));
1798
1854
  const localRecords = recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
1799
1855
  .map(portableRecord), readGroundRowsForRoots(this.databasePath(identity), "detection_notebook", publicationRootIds)
1800
1856
  .map(portableRecord));
1801
- const merged = mergeMaterializedRoots(state.records, localRecords);
1857
+ const merged = mergeMaterializedRoots(officialRecords, localRecords);
1802
1858
  const traveling = travelingRecords(merged, sha256Hex);
1803
1859
  const impossibleGitLeaf = traveling.find((record) => (record.type === "file.text" || record.type === "file.binary" || record.type === "link")
1804
1860
  && record.payloadVersion?.startsWith("git:"));
@@ -1826,7 +1882,7 @@ export class UserGroundHost {
1826
1882
  const uploadRecords = cargo
1827
1883
  .filter((entry) => !sealedArtifacts.has(entry.contentHash))
1828
1884
  .map((entry) => entry.record);
1829
- if (checksum === state.checksum) {
1885
+ if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
1830
1886
  // The graph is already committed; only cargo can be missing. Repair
1831
1887
  // uploads need no journal row — a crash simply re-runs the same
1832
1888
  // repair on the next Register.
@@ -1857,6 +1913,7 @@ export class UserGroundHost {
1857
1913
  baseVersion: state.headVersion,
1858
1914
  snapshotChecksum: checksum,
1859
1915
  snapshotJson,
1916
+ deliveryWatermark,
1860
1917
  uploadRecords,
1861
1918
  replacementRootIds: replacement.rootIds,
1862
1919
  replacementRecords: replacement.records,
@@ -1923,6 +1980,7 @@ export class UserGroundHost {
1923
1980
  contract: ENTITY_CLOUD_CONTRACT,
1924
1981
  schemaVersion: ENTITY_CLOUD_SCHEMA_VERSION,
1925
1982
  operationKind: replacement.kind,
1983
+ deliveryWatermark: pending.deliveryWatermark,
1926
1984
  }, replacement);
1927
1985
  }
1928
1986
  catch (error) {
@@ -1959,6 +2017,7 @@ export class UserGroundHost {
1959
2017
  authorityEpoch: Number(frame.authority_epoch),
1960
2018
  headVersion: Number(frame.version),
1961
2019
  checksum: pending.snapshotChecksum,
2020
+ deliveryWatermark: Number(frame.delivery_watermark),
1962
2021
  records: snapshot.records,
1963
2022
  };
1964
2023
  deleteJournalEntry(this.databasePath(identity), pending.mutationId);
@@ -2147,6 +2206,10 @@ function initializeDatabase(file) {
2147
2206
  authority_id TEXT PRIMARY KEY,
2148
2207
  received_through INTEGER NOT NULL CHECK(received_through >= 0)
2149
2208
  );
2209
+ CREATE TABLE IF NOT EXISTS authority_snapshot_baselines (
2210
+ authority_id TEXT PRIMARY KEY,
2211
+ delivery_sequence INTEGER NOT NULL CHECK(delivery_sequence >= 0)
2212
+ );
2150
2213
  CREATE TABLE IF NOT EXISTS snapshot_publications (
2151
2214
  sequence INTEGER PRIMARY KEY AUTOINCREMENT,
2152
2215
  mutation_id TEXT NOT NULL UNIQUE,
@@ -2363,6 +2426,53 @@ function countDescendantRows(file, rootUUID) {
2363
2426
  database.close();
2364
2427
  }
2365
2428
  }
2429
+ /** The one materialized workspace selected by Add, never every unrelated
2430
+ * workspace retained in the machine notebook. Recovery has the root UUID in
2431
+ * hand, so SQLite can walk that exact parent chain before any rows enter JS. */
2432
+ function readGroundClosureRows(file, rootUUID) {
2433
+ if (!existsSync(file))
2434
+ return [];
2435
+ const database = initializeDatabase(file);
2436
+ try {
2437
+ return database.prepare(`
2438
+ WITH RECURSIVE descendants(uuid) AS (
2439
+ SELECT uuid FROM entities WHERE uuid = ?
2440
+ UNION ALL
2441
+ SELECT child.uuid
2442
+ FROM entities AS child
2443
+ JOIN descendants AS parent ON child.parent_uuid = parent.uuid
2444
+ )
2445
+ SELECT ${GROUND_ROW_PROJECTION}
2446
+ FROM entities
2447
+ WHERE uuid IN (SELECT uuid FROM descendants)
2448
+ ORDER BY uuid
2449
+ `).all(rootUUID);
2450
+ }
2451
+ finally {
2452
+ database.close();
2453
+ }
2454
+ }
2455
+ function readGroundRowsByUUIDs(file, table, uuids) {
2456
+ if (!existsSync(file) || uuids.size === 0)
2457
+ return [];
2458
+ const database = initializeDatabase(file);
2459
+ try {
2460
+ const rows = [];
2461
+ const selected = [...uuids];
2462
+ for (let offset = 0; offset < selected.length; offset += 500) {
2463
+ const batch = selected.slice(offset, offset + 500);
2464
+ const placeholders = batch.map(() => "?").join(", ");
2465
+ rows.push(...database.prepare(`
2466
+ SELECT ${GROUND_ROW_PROJECTION} FROM ${table}
2467
+ WHERE uuid IN (${placeholders}) ORDER BY uuid
2468
+ `).all(...batch));
2469
+ }
2470
+ return rows;
2471
+ }
2472
+ finally {
2473
+ database.close();
2474
+ }
2475
+ }
2366
2476
  /** Recovery facts created before Watch could prove registration: a binding
2367
2477
  * whose portable reference is absent from the notebook, or a bound root that
2368
2478
  * has never produced a local entity row. They pull only those exact roots
@@ -2456,9 +2566,13 @@ function parseWorkspaceAddIntent(json) {
2456
2566
  const cloudHead = String(value.cloudHead || "");
2457
2567
  const destinationPath = String(value.destinationPath || "");
2458
2568
  const stagingPath = String(value.stagingPath || "");
2569
+ const deliveryWatermark = Number(value.deliveryWatermark);
2459
2570
  if (!UUID.test(workspaceId) || !resourceId || !/^[0-9a-f]{64}$/i.test(cloudHead)) {
2460
2571
  throw new Error("workspace Add intent identity is invalid");
2461
2572
  }
2573
+ if (!Number.isSafeInteger(deliveryWatermark) || deliveryWatermark < 0) {
2574
+ throw new Error(`workspace Add intent ${workspaceId} has no delivery watermark`);
2575
+ }
2462
2576
  if (!isAbsolute(destinationPath)
2463
2577
  || stagingPath !== workspaceAddStagingPath(destinationPath, workspaceId)) {
2464
2578
  throw new Error(`workspace Add intent ${workspaceId} has invalid placement`);
@@ -2467,6 +2581,7 @@ function parseWorkspaceAddIntent(json) {
2467
2581
  workspaceId,
2468
2582
  resourceId,
2469
2583
  cloudHead: cloudHead.toLowerCase(),
2584
+ deliveryWatermark,
2470
2585
  destinationPath,
2471
2586
  stagingPath,
2472
2587
  records: snapshotFromRecords(Array.isArray(value.records) ? value.records : []).records,
@@ -2578,11 +2693,16 @@ function readSnapshotPublications(file) {
2578
2693
  || !Array.isArray(publication.replacementRecords)) {
2579
2694
  throw new Error(`snapshot publication ${row.mutationId} has no exact root replacement`);
2580
2695
  }
2696
+ const deliveryWatermark = Number(publication.deliveryWatermark);
2697
+ if (!Number.isSafeInteger(deliveryWatermark) || deliveryWatermark < 0) {
2698
+ throw new Error(`snapshot publication ${row.mutationId} has no delivery watermark`);
2699
+ }
2581
2700
  return {
2582
2701
  mutationId: row.mutationId,
2583
2702
  resourceId: row.resourceId,
2584
2703
  authorityEpoch: Number(publication.authorityEpoch),
2585
2704
  baseVersion: Number(publication.baseVersion),
2705
+ deliveryWatermark,
2586
2706
  snapshotChecksum: String(publication.snapshotChecksum),
2587
2707
  snapshotJson: String(publication.snapshotJson),
2588
2708
  uploadRecords: snapshotFromRecords(publication.uploadRecords).records,
@@ -2609,6 +2729,55 @@ function hasRecordsBeyondSnapshot(file) {
2609
2729
  database.close();
2610
2730
  }
2611
2731
  }
2732
+ function readAuthorityDeliveryCursor(file, authorityId) {
2733
+ const database = initializeDatabase(file);
2734
+ try {
2735
+ const row = database.prepare(`
2736
+ SELECT received_through AS receivedThrough
2737
+ FROM authority_delivery_cursors WHERE authority_id = ?
2738
+ `).get(authorityId);
2739
+ return row?.receivedThrough ?? 0;
2740
+ }
2741
+ finally {
2742
+ database.close();
2743
+ }
2744
+ }
2745
+ /** Latest official result for every UUID in the newly received complete
2746
+ * authority prefix. The record carries the whole logical result, so roots
2747
+ * this machine has not materialized still advance honestly in a Register
2748
+ * snapshot without inventing filesystem evidence. */
2749
+ function readAcceptedResults(file, authorityId, afterDelivery, throughDelivery) {
2750
+ if (throughDelivery <= afterDelivery)
2751
+ return [];
2752
+ const database = initializeDatabase(file);
2753
+ try {
2754
+ const latest = new Map();
2755
+ for (const { recordJson } of database.prepare(`
2756
+ SELECT record_json AS recordJson FROM record_inbox
2757
+ WHERE authority_id = ?
2758
+ AND delivery_sequence > ? AND delivery_sequence <= ?
2759
+ ORDER BY delivery_sequence
2760
+ `).iterate(authorityId, afterDelivery, throughDelivery)) {
2761
+ const accepted = decodeAcceptedEntityRecord(recordJson);
2762
+ latest.set(accepted.entityId, accepted.change.result);
2763
+ }
2764
+ return [...latest.values()];
2765
+ }
2766
+ finally {
2767
+ database.close();
2768
+ }
2769
+ }
2770
+ function recordsAtDeliveryPrefix(base, results) {
2771
+ if (results.length === 0)
2772
+ return base;
2773
+ const records = new Map(base.map((record) => [record.uuid, record]));
2774
+ for (const result of results)
2775
+ records.set(result.uuid, result);
2776
+ // Entity channels are independently ordered. Container membership is the
2777
+ // one derived relationship, so derive it once from the final prefix rather
2778
+ // than letting record arrival order choose a container version.
2779
+ return recordsRetainingMissingEvidence([...records.values()], []);
2780
+ }
2612
2781
  /** Record is the SQLite transaction that accepts Detect's exact proposals and
2613
2782
  * advances the last-verified notebook. Watch may settle only after this
2614
2783
  * function returns. The record table is the local-only outbox. */
@@ -2711,6 +2880,7 @@ function supersedeSnapshotJournal(file, entry) {
2711
2880
  `).run(entry.mutationId, entry.resourceId, stableJson({
2712
2881
  authorityEpoch: entry.authorityEpoch,
2713
2882
  baseVersion: entry.baseVersion,
2883
+ deliveryWatermark: entry.deliveryWatermark,
2714
2884
  snapshotChecksum: entry.snapshotChecksum,
2715
2885
  snapshotJson: entry.snapshotJson,
2716
2886
  uploadRecords: entry.uploadRecords,
@@ -2921,8 +3091,17 @@ function persistRows(file, identity, resourceId, rootUUID, rows, previousRows =
2921
3091
  const releaseChangedSlot = database.prepare("DELETE FROM entities WHERE uuid = ?");
2922
3092
  const removeNotebook = database.prepare("DELETE FROM detection_notebook WHERE uuid = ? AND resource_id = ? AND root_uuid = ?");
2923
3093
  const previousByUuid = new Map(previousRows.map((row) => [row.uuid, row]));
2924
- const materializedIds = new Set(database.prepare("SELECT uuid FROM entities").all()
2925
- .map(({ uuid }) => uuid));
3094
+ const materializedIds = new Set();
3095
+ const proposedUUIDs = rows.map(({ record }) => record.uuid);
3096
+ for (let offset = 0; offset < proposedUUIDs.length; offset += 500) {
3097
+ const batch = proposedUUIDs.slice(offset, offset + 500);
3098
+ const placeholders = batch.map(() => "?").join(", ");
3099
+ for (const { uuid } of database.prepare(`
3100
+ SELECT uuid FROM entities WHERE uuid IN (${placeholders})
3101
+ `).all(...batch)) {
3102
+ materializedIds.add(uuid);
3103
+ }
3104
+ }
2926
3105
  const currentUuids = new Set(rows.map((row) => row.record.uuid));
2927
3106
  const rowChanged = (row, old) => {
2928
3107
  return !old
@@ -4504,7 +4683,7 @@ async function materializeRecords(input) {
4504
4683
  * the same cloud entities on every machine. */
4505
4684
  function installCloudWorkspaceReferences(input) {
4506
4685
  const { identity, userRoot, database, bindingDir, resourceId, workspace, references, } = input;
4507
- const localRows = readRows(database);
4686
+ const localRows = readGroundRowsByUUIDs(database, "entities", new Set(references.flatMap(({ parentUUID }) => parentUUID ? [parentUUID] : [])));
4508
4687
  const rendered = [];
4509
4688
  try {
4510
4689
  for (const reference of references) {
@@ -4600,12 +4779,9 @@ function installCloudWorkspaceReferences(input) {
4600
4779
  return references;
4601
4780
  }
4602
4781
  async function installCloudWorkspace(input) {
4603
- const { identity, userRoot, database, cacheDir, bindingDir, resourceId, workspace, reference, references, records, destinationParent, cloudHead, readContent, onStage, } = input;
4604
- const allLocalRows = readRows(database);
4605
- const existingBoundary = allLocalRows.find((row) => row.uuid === workspace.uuid);
4606
- const existing = existingBoundary
4607
- ? descendantRows(allLocalRows, workspace.uuid)
4608
- : [];
4782
+ const { identity, userRoot, database, cacheDir, bindingDir, resourceId, workspace, reference, references, records, destinationParent, cloudHead, deliveryWatermark, readContent, onStage, } = input;
4783
+ const existing = readGroundClosureRows(database, workspace.uuid);
4784
+ const existingBoundary = existing.find((row) => row.uuid === workspace.uuid);
4609
4785
  const pending = readWorkspaceAddIntent(database, workspace.uuid);
4610
4786
  if (existing.length > 0) {
4611
4787
  const root = existing.find((row) => row.uuid === workspace.uuid);
@@ -4654,6 +4830,7 @@ async function installCloudWorkspace(input) {
4654
4830
  workspace: activeSelection.workspace,
4655
4831
  references: activeReferences,
4656
4832
  });
4833
+ input.adoptMaterialization(workspace.uuid);
4657
4834
  return {
4658
4835
  path: root.absolutePath,
4659
4836
  rows: existing.length,
@@ -4669,10 +4846,14 @@ async function installCloudWorkspace(input) {
4669
4846
  // the old binding stayed bound and watched over ground with no rows, and
4670
4847
  // the next Detect pass there would mint a second identity for every file.
4671
4848
  const selectedUUIDs = new Set(records.map((record) => record.uuid));
4672
- const enclosedElsewhere = allLocalRows.filter((row) => row.uuid !== workspace.uuid && selectedUUIDs.has(row.uuid));
4849
+ const selectedMaterializations = readGroundRowsByUUIDs(database, "entities", selectedUUIDs);
4850
+ const enclosedElsewhere = selectedMaterializations.filter((row) => row.uuid !== workspace.uuid && selectedUUIDs.has(row.uuid));
4673
4851
  if (enclosedElsewhere.length > 0) {
4674
- const roots = [...new Set(enclosedElsewhere.map((row) => row.rootUUID))]
4675
- .map((rootUUID) => allLocalRows.find((row) => row.uuid === rootUUID)?.absolutePath ?? rootUUID);
4852
+ const rootUUIDs = new Set(enclosedElsewhere.map((row) => row.rootUUID));
4853
+ const materializedRoots = new Map(readGroundRowsByUUIDs(database, "entities", rootUUIDs)
4854
+ .map((row) => [row.uuid, row]));
4855
+ const roots = [...rootUUIDs]
4856
+ .map((rootUUID) => materializedRoots.get(rootUUID)?.absolutePath ?? rootUUID);
4676
4857
  throw new Error(`files add: workspace ${workspace.uuid} encloses ${enclosedElsewhere.length} entities already `
4677
4858
  + `materialized on this machine under ${roots.join(", ")}; remove that materialization first`);
4678
4859
  }
@@ -4707,6 +4888,7 @@ async function installCloudWorkspace(input) {
4707
4888
  workspaceId: workspace.uuid,
4708
4889
  resourceId,
4709
4890
  cloudHead,
4891
+ deliveryWatermark,
4710
4892
  destinationPath: destination,
4711
4893
  stagingPath,
4712
4894
  records: snapshotFromRecords(records).records,
@@ -4785,6 +4967,7 @@ function readyFromFrame(frame) {
4785
4967
  const cloud = {
4786
4968
  resourceId: String(frame.resource_id || ""),
4787
4969
  version: Number(frame.head_version),
4970
+ deliveryWatermark: Number(frame.delivery_watermark),
4788
4971
  checksum: String(frame.snapshot_checksum || ""),
4789
4972
  snapshot: parseSnapshot(bytes),
4790
4973
  };
@@ -4795,6 +4978,7 @@ function readyFromFrame(frame) {
4795
4978
  authorityEpoch: Number(frame.authority_epoch),
4796
4979
  headVersion: cloud.version,
4797
4980
  checksum: cloud.checksum,
4981
+ deliveryWatermark: cloud.deliveryWatermark,
4798
4982
  records: cloud.snapshot.records,
4799
4983
  },
4800
4984
  };