@amalgm/shell 0.1.133 → 0.1.135

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,8 @@ import { EntityApplyHost, } from "./entity-apply-host.js";
11
11
  import { EntityRecordSqliteStore } from "./entity-record-store.js";
12
12
  import { observeEntityRecordPorts } from "./entity-record-observability.js";
13
13
  import { indexRowsByOutermostRoot } from "./ground-root-index.js";
14
- import { ContentCacheDownloadSet, captureContentFile, hasSealedContent, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
14
+ import { ContentCacheDownloadSet, captureContentFile, contentBlockCollector, hasSealedContent, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
15
+ import { RegisterUploadFusion } from "./register-upload-fusion.js";
15
16
  import { StagedContentSet, addStagingEvidenceDir, clearAddStagingEvidence, } from "./workspace-add-staging.js";
16
17
  import { DirectContentPlane } from "./content-direct.js";
17
18
  import { encodeContentWireBytes } from "./content-wire-codec.js";
@@ -37,8 +38,8 @@ const CARGO_SEGMENT_LOAD_CONCURRENCY = 8;
37
38
  /** Storage-object lanes for one download set. Tiny objects use the wide
38
39
  * request window, while the separate byte budget narrows large chunks so
39
40
  * request concurrency can never silently become an unbounded memory claim. */
40
- /** Admission is sized to the refs batch law so one same-turn wave coalesces
41
- * into one refs frame and the span fetcher merges each pack's members into a
41
+ /** Admission is sized to the refs batch law so one planned wave resolves in
42
+ * one refs frame and the span planner merges each pack's members into a
42
43
  * few broad ranged reads. Memory is bounded by the byte laws below, never by
43
44
  * this count. */
44
45
  const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = CONTENT_REFS_BATCH_OBJECTS;
@@ -676,20 +677,82 @@ export class UserGroundHost {
676
677
  this.stage({
677
678
  primitive: "register", stage: "entity-registration", status: "started", workspaceId,
678
679
  });
679
- const scanned = await reconcileGround({
680
- identity,
681
- userRoot: this.userRoot(identity),
682
- database: this.databasePath(identity),
683
- cacheDir: this.cacheDir(identity),
684
- onScan: (evidence) => this.reportDetectScan(evidence),
685
- measuring: this.measuring(),
686
- registrationWorkspaceId: workspaceId,
687
- suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
680
+ // The walk's own read is also the upload: smalls ride cargo
681
+ // segments, larges stream blocks and seal last. The flush settles
682
+ // every fused effect before publication plans, so the upload
683
+ // stage's seal inventory — the guarantor for anything fusion
684
+ // could not ship — already sees these seals.
685
+ const fusionResourceId = this.cloudState.resourceId;
686
+ const fusionLegs = new TransferLegs();
687
+ const fusion = new RegisterUploadFusion({
688
+ // Fusion ships whole small artifacts into packs; skipping one
689
+ // requires it to be both sealed and packed, so sealed-but-
690
+ // standalone smalls re-ship here and placement converges on
691
+ // packs without a separate migration.
692
+ sealedContentHashes: async (hashes) => {
693
+ const inventory = await this.sealInventory(fusionResourceId, hashes);
694
+ return new Set([...inventory.sealed].filter((hash) => inventory.packed.has(hash)));
695
+ },
696
+ sendCargo: async (artifacts, payload) => {
697
+ const { sealed } = await this.putCargo(fusionResourceId, artifacts, payload, fusionLegs);
698
+ return sealed;
699
+ },
700
+ putBlocks: async (chunks, bytes) => {
701
+ await this.putContentBlocks(fusionResourceId, chunks, bytes, fusionLegs);
702
+ },
703
+ sealManifest: async (manifest) => {
704
+ await this.putContentManifest(fusionResourceId, manifest, fusionLegs);
705
+ },
688
706
  });
707
+ const scanned = await (async () => {
708
+ try {
709
+ return await reconcileGround({
710
+ identity,
711
+ userRoot: this.userRoot(identity),
712
+ database: this.databasePath(identity),
713
+ cacheDir: this.cacheDir(identity),
714
+ onScan: (evidence) => this.reportDetectScan(evidence),
715
+ measuring: this.measuring(),
716
+ registrationWorkspaceId: workspaceId,
717
+ suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
718
+ fusion,
719
+ });
720
+ }
721
+ finally {
722
+ // Even an abandoned walk settles its in-flight cargo, and a
723
+ // stale-capture re-walk then skips whatever already sealed.
724
+ const fused = await fusion.flush();
725
+ this.stage({
726
+ primitive: "register",
727
+ stage: "fused-upload",
728
+ status: "completed",
729
+ workspaceId,
730
+ measurements: {
731
+ smallArtifacts: fused.smallArtifacts,
732
+ smallBytes: fused.smallBytes,
733
+ cargoSegments: fused.cargoSegments,
734
+ sealedSkipped: fused.sealedSkipped,
735
+ largeArtifacts: fused.largeArtifacts,
736
+ largeBytes: fused.largeBytes,
737
+ ...(fused.disabledReason !== null ? { disabled: fused.disabledReason } : {}),
738
+ ...fusionLegs.report(),
739
+ },
740
+ });
741
+ }
742
+ })();
689
743
  const retry = scanned.detection.find((plan) => plan.kind === "retry");
690
744
  if (retry?.kind === "retry") {
691
745
  throw new Error(retry.reasons.join("; "));
692
746
  }
747
+ // The classic upload re-proved every source byte against its
748
+ // captured hash when it read it. The fused rail already shipped
749
+ // those bytes, so the equivalent proof is the fingerprint law at
750
+ // stat cost: any fused source that moved after capture makes this
751
+ // attempt stale, and the re-walk publishes what stands there now.
752
+ const movedGround = await fusion.movedGround();
753
+ if (movedGround !== null) {
754
+ throw new StaleCaptureError(`${movedGround} changed after its fused upload`);
755
+ }
693
756
  const registrationDuration = performance.now() - registrationStarted;
694
757
  const commitStarted = performance.now();
695
758
  this.stage({
@@ -867,10 +930,11 @@ export class UserGroundHost {
867
930
  throw new Error("entity Record rail already started");
868
931
  await this.wire.open();
869
932
  // One floor, no per-feature gates: this shell and its gateway release
870
- // together, and the gateway ships first. 14 = immutable content packs
871
- // with a refs frame answering locations; an older gateway still speaks
872
- // presign and cannot resolve a packed object at all.
873
- if (this.wire.protocolVersion < 14) {
933
+ // together, and the gateway ships first. 16 = placement answers on the
934
+ // seal inventory; an older gateway cannot say which sealed artifacts are
935
+ // packed, and this shell plans pack-span downloads and repacks against
936
+ // exactly that answer.
937
+ if (this.wire.protocolVersion < 16) {
874
938
  throw new Error("gateway speaks an older wire protocol than this shell");
875
939
  }
876
940
  this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
@@ -1413,6 +1477,40 @@ export class UserGroundHost {
1413
1477
  wireBytes: encoded.bytes.byteLength,
1414
1478
  };
1415
1479
  }
1480
+ /** Store one wave of provisional content blocks, addressed by their own
1481
+ * digests and bound to no artifact. Resolving means every block is
1482
+ * durable at the authority. Answers the wire bytes actually sent. */
1483
+ async putContentBlocks(resourceId, blocks, payload, legs) {
1484
+ const encoded = await legs.time("encodeMs", () => encodeContentWireBytes(payload));
1485
+ const frame = await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
1486
+ type: "private.entity-content.put-blocks",
1487
+ resource_id: resourceId,
1488
+ blocks,
1489
+ bytes: payload.byteLength,
1490
+ wire_bytes: encoded.bytes.byteLength,
1491
+ ...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
1492
+ }, encoded.bytes, ["private.entity-content.stored-blocks"])));
1493
+ legs.add("gatewayVerifyMs", Number(frame.verify_ms));
1494
+ legs.add("gatewayStoreMs", Number(frame.store_ms));
1495
+ return encoded.bytes.byteLength;
1496
+ }
1497
+ /** Seal one artifact's manifest. The authority re-proves every named
1498
+ * block durable before anything becomes visible, so this frame is only
1499
+ * lawful after the complete file went over the wire. */
1500
+ async putContentManifest(resourceId, manifest, legs) {
1501
+ const bytes = Buffer.from(stableJson(manifest), "utf8");
1502
+ const frame = await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
1503
+ type: "private.entity-content.put",
1504
+ resource_id: resourceId,
1505
+ content_hash: manifest.contentHash,
1506
+ sha256: sha256Hex(bytes),
1507
+ bytes: bytes.length,
1508
+ }, bytes, ["private.entity-content.stored"])));
1509
+ legs.add("gatewayVerifyMs", Number(frame.verify_ms));
1510
+ legs.add("gatewayStoreMs", Number(frame.store_ms));
1511
+ legs.add("gatewaySealMs", Number(frame.seal_ms));
1512
+ return bytes.length;
1513
+ }
1416
1514
  async uploadSnapshotContent(identity, resourceId, records, trace, manifestHints) {
1417
1515
  const cache = this.cacheDir(identity);
1418
1516
  const byHash = new Map();
@@ -1448,11 +1546,16 @@ export class UserGroundHost {
1448
1546
  // The authority's seal inventory is the only permission to skip an
1449
1547
  // upload — asked before any source is touched, so sealed artifacts cost
1450
1548
  // zero local reads and a retried publication whose upload already sealed
1451
- // completely never needs its captured ground to still exist.
1452
- const alreadySealed = await legs.time("planMs", () => this.sealedContentHashes(resourceId, [...byHash.keys()]));
1549
+ // completely never needs its captured ground to still exist. Placement
1550
+ // rides the same answer: a sealed-but-unpacked small artifact is
1551
+ // correctness-complete but stored inefficiently, so this guarantor also
1552
+ // re-ships it into a pack when its source is still at hand (axiom 44).
1553
+ const inventory = await legs.time("planMs", () => this.sealInventory(resourceId, [...byHash.keys()]));
1453
1554
  const unsealed = [...byHash.values()]
1454
- .filter(({ artifact }) => !alreadySealed.has(artifact.contentHash));
1455
- const rows = legs.timeSync("planMs", () => readGroundRowsByUuid(this.databasePath(identity), unsealed.map(({ uuid }) => uuid)));
1555
+ .filter(({ artifact }) => !inventory.sealed.has(artifact.contentHash));
1556
+ const unpacked = [...byHash.values()].filter(({ artifact }) => inventory.sealed.has(artifact.contentHash)
1557
+ && !inventory.packed.has(artifact.contentHash));
1558
+ const rows = legs.timeSync("planMs", () => readGroundRowsByUuid(this.databasePath(identity), [...unsealed, ...unpacked].map(({ uuid }) => uuid)));
1456
1559
  // Each artifact uploads from the strongest evidence available: a cache
1457
1560
  // copy when another journey pinned one, otherwise the ground entity
1458
1561
  // itself. Register stages nothing, so the ground path is the common
@@ -1499,6 +1602,42 @@ export class UserGroundHost {
1499
1602
  bytes,
1500
1603
  };
1501
1604
  }));
1605
+ // Repack candidates are sealed already, so their sources are optional:
1606
+ // anything missing, oversized, or unreadable skips silently — a frozen
1607
+ // workspace simply keeps standalone placement until its next actual
1608
+ // publication. What readers see never changes either way.
1609
+ const repackSources = legs.timeSync("planMs", () => unpacked.flatMap(({ artifact, uuid }) => {
1610
+ const cached = contentCacheFile(cache, artifact.contentHash);
1611
+ const fromCache = existsSync(cached);
1612
+ const row = rows.get(uuid);
1613
+ if (!fromCache && (artifact.kind === "git" || !row))
1614
+ return [];
1615
+ const file = fromCache ? cached : row.absolutePath;
1616
+ let linkTarget;
1617
+ let bytes;
1618
+ try {
1619
+ if (!fromCache && artifact.kind === "link") {
1620
+ linkTarget = Buffer.from(readlinkSync(file), "utf8");
1621
+ bytes = linkTarget.byteLength;
1622
+ }
1623
+ else {
1624
+ bytes = statSync(file).size;
1625
+ }
1626
+ }
1627
+ catch {
1628
+ return [];
1629
+ }
1630
+ if (bytes > CHUNK_BYTES)
1631
+ return [];
1632
+ return [{
1633
+ artifact,
1634
+ file,
1635
+ fromCache,
1636
+ ...(linkTarget ? { linkTarget } : {}),
1637
+ bytes,
1638
+ repack: true,
1639
+ }];
1640
+ }));
1502
1641
  const stale = (source, error) => {
1503
1642
  if (source.fromCache) {
1504
1643
  // A cache copy that no longer proves its hash is torn evidence:
@@ -1518,8 +1657,11 @@ export class UserGroundHost {
1518
1657
  const small = sources.filter((source) => source.bytes <= CHUNK_BYTES);
1519
1658
  const large = sources.filter((source) => source.bytes > CHUNK_BYTES);
1520
1659
  let cargoSegments = 0;
1660
+ let repackedArtifacts = 0;
1661
+ let repackedBytes = 0;
1521
1662
  const sendCargoSegment = async (segment) => {
1522
- const loaded = new Array(segment.items.length);
1663
+ const loaded = new Array(segment.items.length).fill(null);
1664
+ const dropped = [];
1523
1665
  await boundedForEach(segment.items.map((item, index) => ({ source: item.source, index })), CARGO_SEGMENT_LOAD_CONCURRENCY, async ({ source, index }) => {
1524
1666
  try {
1525
1667
  const bytes = source.linkTarget ?? await legs.time("loadMs", () => readFile(source.file));
@@ -1530,42 +1672,50 @@ export class UserGroundHost {
1530
1672
  };
1531
1673
  }
1532
1674
  catch (error) {
1675
+ // A repack source that churned or vanished under this read is
1676
+ // dropped, never stale: the artifact stays sealed standalone,
1677
+ // and it is answered as sealed because it truthfully is.
1678
+ if (source.repack) {
1679
+ dropped.push(source.artifact.contentHash);
1680
+ return;
1681
+ }
1533
1682
  if (sourceMismatch(error))
1534
1683
  stale(source, error);
1535
1684
  throw error;
1536
1685
  }
1537
1686
  });
1538
- const { sealed, wireBytes } = await this.putCargo(resourceId, loaded.map((item, index) => ({
1539
- content_hash: segment.items[index].contentHash,
1540
- bytes: item.manifest.bytes,
1541
- chunks: item.manifest.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
1542
- })), Buffer.concat(loaded.map((item) => item.bytes)), legs);
1543
- for (const item of loaded)
1544
- uploadedChunks += item.manifest.chunks.length;
1687
+ const shipping = segment.items.flatMap((item, index) => {
1688
+ const load = loaded[index];
1689
+ return load === null ? [] : [{ item, load }];
1690
+ });
1691
+ if (shipping.length === 0)
1692
+ return dropped;
1693
+ const { sealed, wireBytes } = await this.putCargo(resourceId, shipping.map(({ item, load }) => ({
1694
+ content_hash: item.contentHash,
1695
+ bytes: load.manifest.bytes,
1696
+ chunks: load.manifest.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
1697
+ })), Buffer.concat(shipping.map(({ load }) => load.bytes)), legs);
1698
+ for (const { item, load } of shipping) {
1699
+ uploadedChunks += load.manifest.chunks.length;
1700
+ if (item.source.repack) {
1701
+ repackedArtifacts += 1;
1702
+ repackedBytes += load.manifest.bytes;
1703
+ }
1704
+ }
1545
1705
  uploadedBytes += wireBytes;
1546
1706
  cargoSegments += 1;
1547
- return sealed;
1707
+ return [...sealed, ...dropped];
1548
1708
  };
1549
1709
  const transfer = async (source) => {
1550
1710
  const { artifact, file, manifest: knownManifest, bytes: fileBytes } = source;
1551
1711
  const sourceFile = { handle: null };
1552
1712
  let manifestPresent = false;
1553
- const putChunk = async (manifest, index, bytes) => {
1713
+ const chunkBlocks = (manifest, indexes) => indexes.map((index) => {
1554
1714
  const chunk = manifest.chunks[index];
1555
1715
  if (!chunk)
1556
1716
  throw new Error(`content manifest has no chunk ${index}`);
1557
- await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
1558
- type: "private.entity-content.put",
1559
- resource_id: resourceId,
1560
- content_hash: artifact.contentHash,
1561
- kind: "chunk",
1562
- part_index: index,
1563
- part_count: manifest.chunks.length,
1564
- sha256: chunk.sha256,
1565
- bytes: chunk.bytes,
1566
- }, bytes, ["private.entity-content.stored"])));
1567
- uploadedBytes += bytes.byteLength;
1568
- };
1717
+ return { sha256: chunk.sha256, bytes: chunk.bytes };
1718
+ });
1569
1719
  try {
1570
1720
  const receipt = await uploadArtifact(artifact, {
1571
1721
  bytes: fileBytes,
@@ -1595,40 +1745,16 @@ export class UserGroundHost {
1595
1745
  }
1596
1746
  return frame.missing;
1597
1747
  },
1598
- putChunk: async (_candidate, manifest, index, bytes) => putChunk(manifest, index, bytes),
1748
+ putChunk: async (_candidate, manifest, index, bytes) => {
1749
+ uploadedBytes += await this.putContentBlocks(resourceId, chunkBlocks(manifest, [index]), bytes, legs);
1750
+ },
1599
1751
  putChunkBatch: async (_candidate, manifest, indexes, bytes) => {
1600
- const encoded = await legs.time("encodeMs", () => encodeContentWireBytes(bytes));
1601
- await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
1602
- type: "private.entity-content.put-batch",
1603
- resource_id: resourceId,
1604
- content_hash: artifact.contentHash,
1605
- part_count: manifest.chunks.length,
1606
- chunks: indexes.map((index) => ({
1607
- part_index: index,
1608
- sha256: manifest.chunks[index]?.sha256,
1609
- bytes: manifest.chunks[index]?.bytes,
1610
- })),
1611
- bytes: bytes.byteLength,
1612
- wire_bytes: encoded.bytes.byteLength,
1613
- ...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
1614
- }, encoded.bytes, ["private.entity-content.stored-batch"])));
1615
- uploadedBytes += encoded.bytes.byteLength;
1752
+ uploadedBytes += await this.putContentBlocks(resourceId, chunkBlocks(manifest, indexes), bytes, legs);
1616
1753
  },
1617
1754
  putManifest: async (_candidate, manifest) => {
1618
1755
  if (manifestPresent)
1619
1756
  return;
1620
- const bytes = Buffer.from(stableJson(manifest), "utf8");
1621
- await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
1622
- type: "private.entity-content.put",
1623
- resource_id: resourceId,
1624
- content_hash: artifact.contentHash,
1625
- kind: "manifest",
1626
- part_index: manifest.chunks.length,
1627
- part_count: manifest.chunks.length,
1628
- sha256: sha256Hex(bytes),
1629
- bytes: bytes.length,
1630
- }, bytes, ["private.entity-content.stored"])));
1631
- uploadedBytes += bytes.byteLength;
1757
+ uploadedBytes += await this.putContentManifest(resourceId, manifest, legs);
1632
1758
  },
1633
1759
  });
1634
1760
  uploadedChunks += receipt.uploadedChunks;
@@ -1645,7 +1771,7 @@ export class UserGroundHost {
1645
1771
  }
1646
1772
  };
1647
1773
  try {
1648
- await streamCargo(small.map((source) => ({
1774
+ await streamCargo([...small, ...repackSources].map((source) => ({
1649
1775
  contentHash: source.artifact.contentHash,
1650
1776
  bytes: source.bytes,
1651
1777
  source,
@@ -1672,6 +1798,12 @@ export class UserGroundHost {
1672
1798
  cargoBytes: small.reduce((total, source) => total + source.bytes, 0),
1673
1799
  largeArtifacts: large.length,
1674
1800
  largeBytes: large.reduce((total, source) => total + source.bytes, 0),
1801
+ // Placement work: sealed-but-standalone smalls re-shipped into
1802
+ // packs by this guarantor. Candidates counts what the inventory
1803
+ // named; repacked counts what actually shipped this pass.
1804
+ repackCandidates: unpacked.length,
1805
+ repackedArtifacts,
1806
+ repackedBytes,
1675
1807
  uploadedBytes,
1676
1808
  loadedBytes,
1677
1809
  uploadedChunks,
@@ -1701,10 +1833,11 @@ export class UserGroundHost {
1701
1833
  }
1702
1834
  }
1703
1835
  /** Download a set of immutable artifacts straight from content storage.
1704
- * The set is the primitive: every storage object the set still needs —
1705
- * manifests and blocks alike — shares one wide lane bound, so no artifact
1706
- * serializes behind another's plan or seal. Each object is still its own
1707
- * storage request. A lone artifact is a set of one. */
1836
+ * The set is the primitive and the plan is physical: the engine resolves
1837
+ * the whole want-list in two planned waves — manifests grouped by the
1838
+ * pack that holds them, then blocks the same way — so all members of one
1839
+ * pack collapse into a few broad ranged reads while standalone objects
1840
+ * stay one read each. A lone artifact is a set of one. */
1708
1841
  async downloadContentSet(resourceId, cacheDir, wants, retryAttempts = 10, trace, staged) {
1709
1842
  const unique = new Map(wants.map((want) => [want.artifact.contentHash, want]));
1710
1843
  const sole = unique.size === 1 ? [...unique.values()][0]?.artifact : undefined;
@@ -1745,56 +1878,75 @@ export class UserGroundHost {
1745
1878
  throw new Error("download cache has no content manifest");
1746
1879
  return found;
1747
1880
  };
1748
- // Every storage object — the manifest and each block — is fetched
1749
- // straight from content storage over a presigned URL and opened locally;
1750
- // packed objects arrive as merged byte ranges of their immutable pack.
1751
- // The wire carries only the key handoff and refs requests. A retry asks
1752
- // for a fresh ref every attempt.
1753
- const fetchObject = async (object) => {
1754
- requests += 1;
1755
- const bytes = await legs.time("fetchMs", () => retryContent(() => this.direct.fetchObject(resourceId, object), true, retryAttempts));
1756
- downloadedBytes += bytes.byteLength;
1757
- return bytes;
1881
+ // An Add stages ordinary files directly at their hidden final
1882
+ // destination — except content this machine already holds sealed,
1883
+ // which stays on the cache rail so a warm re-Add remains a local
1884
+ // clone instead of a fresh download.
1885
+ const route = (artifact, manifest) => {
1886
+ const stagedPaths = stagedSet && staged?.destinations.get(artifact.contentHash);
1887
+ if (stagedPaths && !hasSealedContent(cacheDir, artifact.contentHash, manifest.bytes)) {
1888
+ stagedArtifacts += 1;
1889
+ targets.set(artifact.contentHash, stagedSet.target(artifact, manifest, stagedPaths));
1890
+ }
1891
+ else {
1892
+ targets.set(artifact.contentHash, downloads.target(artifact, manifest));
1893
+ }
1758
1894
  };
1759
1895
  try {
1760
- const receipts = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
1896
+ // Every storage object — manifests and blocks alike — is fetched
1897
+ // straight from content storage over a presigned URL and opened
1898
+ // locally; the engine plans whole waves against these primitives, so
1899
+ // the wire carries only the key handoff and batched refs requests. A
1900
+ // retried read refreshes its URL but never its geometry: placement is
1901
+ // immutable, and a moved answer fails loudly instead of retrying.
1902
+ const { receipts, shape } = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
1761
1903
  sha256Hex: timedSha256Hex,
1762
- getManifest: async (artifact) => {
1904
+ localManifest: (artifact) => legs.timeSync("planMs", () => {
1763
1905
  // The record's plan or this machine's own manifest cache already
1764
1906
  // proves chunk identity; downloaded bytes are still hash-verified,
1765
1907
  // so a stale local manifest can only fail loudly, never corrupt.
1766
- const local = legs.timeSync("planMs", () => unique.get(artifact.contentHash)?.manifest
1767
- ?? readCachedManifest(cacheDir, artifact.contentHash));
1768
- const manifest = local ?? await (async () => {
1769
- const bytes = await fetchObject({ kind: "manifest", sha256: artifact.contentHash });
1770
- const checked = checkContentManifest(JSON.parse(bytes.toString("utf8")), artifact.contentHash);
1771
- if (!checked.ok)
1772
- throw new Error(checked.error);
1773
- return checked.value;
1774
- })();
1775
- // An Add stages ordinary files directly at their hidden final
1776
- // destination — except content this machine already holds sealed,
1777
- // which stays on the cache rail so a warm re-Add remains a local
1778
- // clone instead of a fresh download.
1779
- legs.timeSync("planMs", () => {
1780
- const stagedPaths = stagedSet && staged?.destinations.get(artifact.contentHash);
1781
- if (stagedPaths && !hasSealedContent(cacheDir, artifact.contentHash, manifest.bytes)) {
1782
- stagedArtifacts += 1;
1783
- targets.set(artifact.contentHash, stagedSet.target(artifact, manifest, stagedPaths));
1784
- }
1785
- else {
1786
- targets.set(artifact.contentHash, downloads.target(artifact, manifest));
1908
+ const local = unique.get(artifact.contentHash)?.manifest
1909
+ ?? readCachedManifest(cacheDir, artifact.contentHash);
1910
+ if (local)
1911
+ route(artifact, local);
1912
+ return local ?? null;
1913
+ }),
1914
+ requestRefs: async (objects) => legs.time("fetchMs", () => retryContent(() => this.direct.requestRefs(resourceId, objects), true, retryAttempts)),
1915
+ fetchSpan: async (url, span, refresh) => legs.time("fetchMs", async () => {
1916
+ let attempt = 0;
1917
+ return retryContent(async () => {
1918
+ let signed = url;
1919
+ if (attempt > 0) {
1920
+ try {
1921
+ signed = await refresh();
1922
+ }
1923
+ catch (error) {
1924
+ // A moved placement (or a refused re-resolution) is a
1925
+ // planning fact, not a transient fetch fault — surface it
1926
+ // without burning the remaining attempts.
1927
+ throw new WireRequestError({
1928
+ type: "shared.error",
1929
+ code: "entity_content_moved",
1930
+ message: error instanceof Error ? error.message : String(error),
1931
+ status: 409,
1932
+ retryable: false,
1933
+ });
1934
+ }
1787
1935
  }
1788
- });
1789
- return manifest;
1936
+ attempt += 1;
1937
+ return this.direct.fetchSpan(signed, span);
1938
+ }, true, retryAttempts);
1939
+ }),
1940
+ openEnvelope: async (_object, sealed) => {
1941
+ requests += 1;
1942
+ const plain = await legs.time("fetchMs", () => this.direct.openEnvelope(resourceId, sealed));
1943
+ downloadedBytes += plain.byteLength;
1944
+ return plain;
1790
1945
  },
1791
- hasChunk: async (artifact, _manifest, index) => legs.time("reuseMs", () => target(artifact).hasChunk(index)),
1792
- getChunk: async (_artifact, manifest, index) => {
1793
- const chunk = manifest.chunks[index];
1794
- if (!chunk)
1795
- throw new Error(`content manifest has no chunk ${index}`);
1796
- return fetchObject({ kind: "block", sha256: chunk.sha256 });
1946
+ manifestProven: async (artifact, manifest) => {
1947
+ legs.timeSync("planMs", () => route(artifact, manifest));
1797
1948
  },
1949
+ hasChunk: async (artifact, _manifest, index) => legs.time("reuseMs", () => target(artifact).hasChunk(index)),
1798
1950
  putChunk: async (artifact, _manifest, index, bytes) => legs.time("writeMs", () => target(artifact).putChunk(index, bytes)),
1799
1951
  sealArtifact: async (artifact) => {
1800
1952
  const sealed = await legs.time("sealMs", () => target(artifact).seal());
@@ -1835,6 +1987,10 @@ export class UserGroundHost {
1835
1987
  // nothing in flight. The staged rail decomposes its own write and
1836
1988
  // seal internals below; `stagedArtifacts` counts its placements.
1837
1989
  requests,
1990
+ // The engine's own geometry: how manifests were known (local vs
1991
+ // fetched, packed vs standalone) and how many planned pack spans
1992
+ // carried the packed objects.
1993
+ ...shape,
1838
1994
  ...physicalPlaneDelta(planeBefore, this.direct.metrics()),
1839
1995
  ...legs.report(),
1840
1996
  ...(stagedSet ? stagedSet.report() : {}),
@@ -2415,27 +2571,35 @@ export class UserGroundHost {
2415
2571
  });
2416
2572
  }
2417
2573
  /** Which of these heads does the authority hold sealed under the current
2418
- * content contract? Batched so a whole workspace costs a handful of
2419
- * frames; the answer is the only permission to skip an upload. */
2420
- async sealedContentHashes(resourceId, hashes) {
2574
+ * content contract — and which of the sealed ones are packed, their whole
2575
+ * closure inside immutable packs? Batched so a whole workspace costs a
2576
+ * handful of frames. `sealed` is the only permission to skip an upload
2577
+ * for correctness; `packed` is placement knowledge only. The packed
2578
+ * answer is a protocol 16 capability and is required: a missing field
2579
+ * means an older gateway, and this machine refuses to guess placement. */
2580
+ async sealInventory(resourceId, hashes) {
2421
2581
  const batches = [];
2422
2582
  for (let start = 0; start < hashes.length; start += CONTENT_REFS_BATCH_OBJECTS) {
2423
2583
  batches.push(hashes.slice(start, start + CONTENT_REFS_BATCH_OBJECTS));
2424
2584
  }
2425
2585
  const sealed = new Set();
2586
+ const packed = new Set();
2426
2587
  const answers = await Promise.all(batches.map((batch) => retryContent(() => this.wire.request({
2427
2588
  type: "private.entity-content.sealed",
2428
2589
  resource_id: resourceId,
2429
2590
  content_hashes: batch,
2430
2591
  }, ["private.entity-content.sealed-result"]))));
2431
2592
  for (const frame of answers) {
2432
- if (!Array.isArray(frame.sealed)) {
2433
- throw new Error("content authority answered an invalid seal inventory");
2593
+ if (!Array.isArray(frame.sealed) || !Array.isArray(frame.packed)) {
2594
+ throw new Error("content authority answered a seal inventory without "
2595
+ + "placement; protocol 16 requires the gateway to deploy before this machine");
2434
2596
  }
2435
2597
  for (const hash of frame.sealed)
2436
2598
  sealed.add(String(hash));
2599
+ for (const hash of frame.packed)
2600
+ packed.add(String(hash));
2437
2601
  }
2438
- return sealed;
2602
+ return { sealed, packed };
2439
2603
  }
2440
2604
  async publishPendingSnapshots(identity, trace, manifestHints) {
2441
2605
  for (const pending of readSnapshotPublications(this.databasePath(identity))) {
@@ -2456,7 +2620,7 @@ export class UserGroundHost {
2456
2620
  return;
2457
2621
  const started = performance.now();
2458
2622
  const legs = new TransferLegs();
2459
- const sealed = await legs.time("planMs", () => this.sealedContentHashes(resourceId, pages.map((page) => page.sha256)));
2623
+ const { sealed } = await legs.time("planMs", () => this.sealInventory(resourceId, pages.map((page) => page.sha256)));
2460
2624
  const unsealed = pages
2461
2625
  .filter((page) => !sealed.has(page.sha256))
2462
2626
  .map((page) => ({ contentHash: page.sha256, bytes: page.bytes.byteLength, page }));
@@ -4102,7 +4266,7 @@ function materializedWorkspaceBindings(bindingDir) {
4102
4266
  return bindings.sort((left, right) => left.workspaceId.localeCompare(right.workspaceId));
4103
4267
  }
4104
4268
  async function reconcileGround(input) {
4105
- const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, measuring = false, } = input;
4269
+ const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, fusion, onScan, measuring = false, } = input;
4106
4270
  const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
4107
4271
  const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
4108
4272
  const boundRootEntries = materializedWorkspaceBindings(bindingDir);
@@ -4328,9 +4492,11 @@ async function reconcileGround(input) {
4328
4492
  suspicion: observationFor(rootPath),
4329
4493
  // A register scan publishes one complete snapshot; it never mints Send
4330
4494
  // records for the same initial contents, so it plans no detection and
4331
- // stages no cache bytes — upload reads the ground itself.
4495
+ // stages no cache bytes — the fused rail ships content from the walk's
4496
+ // own read, and upload re-reads only what fusion could not seal.
4332
4497
  plansSendRecords: suspicions !== undefined && registrationWorkspaceId === undefined,
4333
4498
  largeManifests,
4499
+ fusion,
4334
4500
  });
4335
4501
  }
4336
4502
  const changedUUIDs = new Set([
@@ -4358,7 +4524,7 @@ async function reconcileGround(input) {
4358
4524
  };
4359
4525
  }
4360
4526
  async function reconcileRoot(input) {
4361
- const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, } = input;
4527
+ const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, fusion, } = input;
4362
4528
  // Prior rows reach this scan from every binding this root encloses. A row's
4363
4529
  // stored relativePath is relative to the root that last committed it, which
4364
4530
  // is a different frame when an enclosing boundary was just registered
@@ -4711,9 +4877,20 @@ async function reconcileRoot(input) {
4711
4877
  captureTasks.push(captureGate(async () => {
4712
4878
  const taskStarted = measureEvidence ? performance.now() : 0;
4713
4879
  let current = stats;
4880
+ // Repository leaves ship on the Git-native rail; everything else
4881
+ // may fuse its capture read with upload. Fusion is opportunistic:
4882
+ // once disabled, captures continue exactly as the classic path
4883
+ // and the upload stage ships what the walk could not.
4884
+ const fused = activeRepository ? undefined : fusion;
4714
4885
  for (let attempt = 1;; attempt++) {
4715
4886
  try {
4716
- const captured = await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir);
4887
+ const collect = fused?.enabled && current.size <= CHUNK_BYTES
4888
+ ? contentBlockCollector(current.size)
4889
+ : null;
4890
+ const captured = fused && current.size > CHUNK_BYTES
4891
+ ? (await fused.captureLarge(absolutePath, current))
4892
+ ?? await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir)
4893
+ : await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir, collect);
4717
4894
  if (measureEvidence && evidence) {
4718
4895
  evidence.captureBusyMs += performance.now() - taskStarted;
4719
4896
  evidence.captureBytes += captured.bytes;
@@ -4726,6 +4903,12 @@ async function reconcileRoot(input) {
4726
4903
  if (captured.bytes > CHUNK_BYTES) {
4727
4904
  largeManifests?.set(captured.contentHash, captured.manifest);
4728
4905
  }
4906
+ // The capture proved these exact bytes against the fingerprint
4907
+ // law; hand them to the cargo rail. Waiting here is window
4908
+ // backpressure, bounding memory by the capture pool's width.
4909
+ if (collect && fused) {
4910
+ await fused.addSmall(absolutePath, current, captured, collect.collected);
4911
+ }
4729
4912
  return;
4730
4913
  }
4731
4914
  catch (error) {