@amalgm/shell 0.1.133 → 0.1.135
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/content-cache-host.d.ts +27 -3
- package/dist/content-cache-host.js +82 -7
- package/dist/content-cache-host.js.map +1 -1
- package/dist/content-direct.d.ts +21 -13
- package/dist/content-direct.js +70 -68
- package/dist/content-direct.js.map +1 -1
- package/dist/register-upload-fusion.d.ts +83 -0
- package/dist/register-upload-fusion.js +169 -0
- package/dist/register-upload-fusion.js.map +1 -0
- package/dist/user-ground-host.d.ts +20 -7
- package/dist/user-ground-host.js +311 -128
- package/dist/user-ground-host.js.map +1 -1
- package/dist/wire-client.js +1 -1
- package/npm-shrinkwrap.json +6 -6
- package/package.json +2 -2
package/dist/user-ground-host.js
CHANGED
|
@@ -11,7 +11,8 @@ import { EntityApplyHost, } from "./entity-apply-host.js";
|
|
|
11
11
|
import { EntityRecordSqliteStore } from "./entity-record-store.js";
|
|
12
12
|
import { observeEntityRecordPorts } from "./entity-record-observability.js";
|
|
13
13
|
import { indexRowsByOutermostRoot } from "./ground-root-index.js";
|
|
14
|
-
import { ContentCacheDownloadSet, captureContentFile, hasSealedContent, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
|
|
14
|
+
import { ContentCacheDownloadSet, captureContentFile, contentBlockCollector, hasSealedContent, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
|
|
15
|
+
import { RegisterUploadFusion } from "./register-upload-fusion.js";
|
|
15
16
|
import { StagedContentSet, addStagingEvidenceDir, clearAddStagingEvidence, } from "./workspace-add-staging.js";
|
|
16
17
|
import { DirectContentPlane } from "./content-direct.js";
|
|
17
18
|
import { encodeContentWireBytes } from "./content-wire-codec.js";
|
|
@@ -37,8 +38,8 @@ const CARGO_SEGMENT_LOAD_CONCURRENCY = 8;
|
|
|
37
38
|
/** Storage-object lanes for one download set. Tiny objects use the wide
|
|
38
39
|
* request window, while the separate byte budget narrows large chunks so
|
|
39
40
|
* request concurrency can never silently become an unbounded memory claim. */
|
|
40
|
-
/** Admission is sized to the refs batch law so one
|
|
41
|
-
*
|
|
41
|
+
/** Admission is sized to the refs batch law so one planned wave resolves in
|
|
42
|
+
* one refs frame and the span planner merges each pack's members into a
|
|
42
43
|
* few broad ranged reads. Memory is bounded by the byte laws below, never by
|
|
43
44
|
* this count. */
|
|
44
45
|
const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = CONTENT_REFS_BATCH_OBJECTS;
|
|
@@ -676,20 +677,82 @@ export class UserGroundHost {
|
|
|
676
677
|
this.stage({
|
|
677
678
|
primitive: "register", stage: "entity-registration", status: "started", workspaceId,
|
|
678
679
|
});
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
680
|
+
// The walk's own read is also the upload: smalls ride cargo
|
|
681
|
+
// segments, larges stream blocks and seal last. The flush settles
|
|
682
|
+
// every fused effect before publication plans, so the upload
|
|
683
|
+
// stage's seal inventory — the guarantor for anything fusion
|
|
684
|
+
// could not ship — already sees these seals.
|
|
685
|
+
const fusionResourceId = this.cloudState.resourceId;
|
|
686
|
+
const fusionLegs = new TransferLegs();
|
|
687
|
+
const fusion = new RegisterUploadFusion({
|
|
688
|
+
// Fusion ships whole small artifacts into packs; skipping one
|
|
689
|
+
// requires it to be both sealed and packed, so sealed-but-
|
|
690
|
+
// standalone smalls re-ship here and placement converges on
|
|
691
|
+
// packs without a separate migration.
|
|
692
|
+
sealedContentHashes: async (hashes) => {
|
|
693
|
+
const inventory = await this.sealInventory(fusionResourceId, hashes);
|
|
694
|
+
return new Set([...inventory.sealed].filter((hash) => inventory.packed.has(hash)));
|
|
695
|
+
},
|
|
696
|
+
sendCargo: async (artifacts, payload) => {
|
|
697
|
+
const { sealed } = await this.putCargo(fusionResourceId, artifacts, payload, fusionLegs);
|
|
698
|
+
return sealed;
|
|
699
|
+
},
|
|
700
|
+
putBlocks: async (chunks, bytes) => {
|
|
701
|
+
await this.putContentBlocks(fusionResourceId, chunks, bytes, fusionLegs);
|
|
702
|
+
},
|
|
703
|
+
sealManifest: async (manifest) => {
|
|
704
|
+
await this.putContentManifest(fusionResourceId, manifest, fusionLegs);
|
|
705
|
+
},
|
|
688
706
|
});
|
|
707
|
+
const scanned = await (async () => {
|
|
708
|
+
try {
|
|
709
|
+
return await reconcileGround({
|
|
710
|
+
identity,
|
|
711
|
+
userRoot: this.userRoot(identity),
|
|
712
|
+
database: this.databasePath(identity),
|
|
713
|
+
cacheDir: this.cacheDir(identity),
|
|
714
|
+
onScan: (evidence) => this.reportDetectScan(evidence),
|
|
715
|
+
measuring: this.measuring(),
|
|
716
|
+
registrationWorkspaceId: workspaceId,
|
|
717
|
+
suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
|
|
718
|
+
fusion,
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
finally {
|
|
722
|
+
// Even an abandoned walk settles its in-flight cargo, and a
|
|
723
|
+
// stale-capture re-walk then skips whatever already sealed.
|
|
724
|
+
const fused = await fusion.flush();
|
|
725
|
+
this.stage({
|
|
726
|
+
primitive: "register",
|
|
727
|
+
stage: "fused-upload",
|
|
728
|
+
status: "completed",
|
|
729
|
+
workspaceId,
|
|
730
|
+
measurements: {
|
|
731
|
+
smallArtifacts: fused.smallArtifacts,
|
|
732
|
+
smallBytes: fused.smallBytes,
|
|
733
|
+
cargoSegments: fused.cargoSegments,
|
|
734
|
+
sealedSkipped: fused.sealedSkipped,
|
|
735
|
+
largeArtifacts: fused.largeArtifacts,
|
|
736
|
+
largeBytes: fused.largeBytes,
|
|
737
|
+
...(fused.disabledReason !== null ? { disabled: fused.disabledReason } : {}),
|
|
738
|
+
...fusionLegs.report(),
|
|
739
|
+
},
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
})();
|
|
689
743
|
const retry = scanned.detection.find((plan) => plan.kind === "retry");
|
|
690
744
|
if (retry?.kind === "retry") {
|
|
691
745
|
throw new Error(retry.reasons.join("; "));
|
|
692
746
|
}
|
|
747
|
+
// The classic upload re-proved every source byte against its
|
|
748
|
+
// captured hash when it read it. The fused rail already shipped
|
|
749
|
+
// those bytes, so the equivalent proof is the fingerprint law at
|
|
750
|
+
// stat cost: any fused source that moved after capture makes this
|
|
751
|
+
// attempt stale, and the re-walk publishes what stands there now.
|
|
752
|
+
const movedGround = await fusion.movedGround();
|
|
753
|
+
if (movedGround !== null) {
|
|
754
|
+
throw new StaleCaptureError(`${movedGround} changed after its fused upload`);
|
|
755
|
+
}
|
|
693
756
|
const registrationDuration = performance.now() - registrationStarted;
|
|
694
757
|
const commitStarted = performance.now();
|
|
695
758
|
this.stage({
|
|
@@ -867,10 +930,11 @@ export class UserGroundHost {
|
|
|
867
930
|
throw new Error("entity Record rail already started");
|
|
868
931
|
await this.wire.open();
|
|
869
932
|
// One floor, no per-feature gates: this shell and its gateway release
|
|
870
|
-
// together, and the gateway ships first.
|
|
871
|
-
//
|
|
872
|
-
//
|
|
873
|
-
|
|
933
|
+
// together, and the gateway ships first. 16 = placement answers on the
|
|
934
|
+
// seal inventory; an older gateway cannot say which sealed artifacts are
|
|
935
|
+
// packed, and this shell plans pack-span downloads and repacks against
|
|
936
|
+
// exactly that answer.
|
|
937
|
+
if (this.wire.protocolVersion < 16) {
|
|
874
938
|
throw new Error("gateway speaks an older wire protocol than this shell");
|
|
875
939
|
}
|
|
876
940
|
this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
|
|
@@ -1413,6 +1477,40 @@ export class UserGroundHost {
|
|
|
1413
1477
|
wireBytes: encoded.bytes.byteLength,
|
|
1414
1478
|
};
|
|
1415
1479
|
}
|
|
1480
|
+
/** Store one wave of provisional content blocks, addressed by their own
|
|
1481
|
+
* digests and bound to no artifact. Resolving means every block is
|
|
1482
|
+
* durable at the authority. Answers the wire bytes actually sent. */
|
|
1483
|
+
async putContentBlocks(resourceId, blocks, payload, legs) {
|
|
1484
|
+
const encoded = await legs.time("encodeMs", () => encodeContentWireBytes(payload));
|
|
1485
|
+
const frame = await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1486
|
+
type: "private.entity-content.put-blocks",
|
|
1487
|
+
resource_id: resourceId,
|
|
1488
|
+
blocks,
|
|
1489
|
+
bytes: payload.byteLength,
|
|
1490
|
+
wire_bytes: encoded.bytes.byteLength,
|
|
1491
|
+
...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
|
|
1492
|
+
}, encoded.bytes, ["private.entity-content.stored-blocks"])));
|
|
1493
|
+
legs.add("gatewayVerifyMs", Number(frame.verify_ms));
|
|
1494
|
+
legs.add("gatewayStoreMs", Number(frame.store_ms));
|
|
1495
|
+
return encoded.bytes.byteLength;
|
|
1496
|
+
}
|
|
1497
|
+
/** Seal one artifact's manifest. The authority re-proves every named
|
|
1498
|
+
* block durable before anything becomes visible, so this frame is only
|
|
1499
|
+
* lawful after the complete file went over the wire. */
|
|
1500
|
+
async putContentManifest(resourceId, manifest, legs) {
|
|
1501
|
+
const bytes = Buffer.from(stableJson(manifest), "utf8");
|
|
1502
|
+
const frame = await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1503
|
+
type: "private.entity-content.put",
|
|
1504
|
+
resource_id: resourceId,
|
|
1505
|
+
content_hash: manifest.contentHash,
|
|
1506
|
+
sha256: sha256Hex(bytes),
|
|
1507
|
+
bytes: bytes.length,
|
|
1508
|
+
}, bytes, ["private.entity-content.stored"])));
|
|
1509
|
+
legs.add("gatewayVerifyMs", Number(frame.verify_ms));
|
|
1510
|
+
legs.add("gatewayStoreMs", Number(frame.store_ms));
|
|
1511
|
+
legs.add("gatewaySealMs", Number(frame.seal_ms));
|
|
1512
|
+
return bytes.length;
|
|
1513
|
+
}
|
|
1416
1514
|
async uploadSnapshotContent(identity, resourceId, records, trace, manifestHints) {
|
|
1417
1515
|
const cache = this.cacheDir(identity);
|
|
1418
1516
|
const byHash = new Map();
|
|
@@ -1448,11 +1546,16 @@ export class UserGroundHost {
|
|
|
1448
1546
|
// The authority's seal inventory is the only permission to skip an
|
|
1449
1547
|
// upload — asked before any source is touched, so sealed artifacts cost
|
|
1450
1548
|
// zero local reads and a retried publication whose upload already sealed
|
|
1451
|
-
// completely never needs its captured ground to still exist.
|
|
1452
|
-
|
|
1549
|
+
// completely never needs its captured ground to still exist. Placement
|
|
1550
|
+
// rides the same answer: a sealed-but-unpacked small artifact is
|
|
1551
|
+
// correctness-complete but stored inefficiently, so this guarantor also
|
|
1552
|
+
// re-ships it into a pack when its source is still at hand (axiom 44).
|
|
1553
|
+
const inventory = await legs.time("planMs", () => this.sealInventory(resourceId, [...byHash.keys()]));
|
|
1453
1554
|
const unsealed = [...byHash.values()]
|
|
1454
|
-
.filter(({ artifact }) => !
|
|
1455
|
-
const
|
|
1555
|
+
.filter(({ artifact }) => !inventory.sealed.has(artifact.contentHash));
|
|
1556
|
+
const unpacked = [...byHash.values()].filter(({ artifact }) => inventory.sealed.has(artifact.contentHash)
|
|
1557
|
+
&& !inventory.packed.has(artifact.contentHash));
|
|
1558
|
+
const rows = legs.timeSync("planMs", () => readGroundRowsByUuid(this.databasePath(identity), [...unsealed, ...unpacked].map(({ uuid }) => uuid)));
|
|
1456
1559
|
// Each artifact uploads from the strongest evidence available: a cache
|
|
1457
1560
|
// copy when another journey pinned one, otherwise the ground entity
|
|
1458
1561
|
// itself. Register stages nothing, so the ground path is the common
|
|
@@ -1499,6 +1602,42 @@ export class UserGroundHost {
|
|
|
1499
1602
|
bytes,
|
|
1500
1603
|
};
|
|
1501
1604
|
}));
|
|
1605
|
+
// Repack candidates are sealed already, so their sources are optional:
|
|
1606
|
+
// anything missing, oversized, or unreadable skips silently — a frozen
|
|
1607
|
+
// workspace simply keeps standalone placement until its next actual
|
|
1608
|
+
// publication. What readers see never changes either way.
|
|
1609
|
+
const repackSources = legs.timeSync("planMs", () => unpacked.flatMap(({ artifact, uuid }) => {
|
|
1610
|
+
const cached = contentCacheFile(cache, artifact.contentHash);
|
|
1611
|
+
const fromCache = existsSync(cached);
|
|
1612
|
+
const row = rows.get(uuid);
|
|
1613
|
+
if (!fromCache && (artifact.kind === "git" || !row))
|
|
1614
|
+
return [];
|
|
1615
|
+
const file = fromCache ? cached : row.absolutePath;
|
|
1616
|
+
let linkTarget;
|
|
1617
|
+
let bytes;
|
|
1618
|
+
try {
|
|
1619
|
+
if (!fromCache && artifact.kind === "link") {
|
|
1620
|
+
linkTarget = Buffer.from(readlinkSync(file), "utf8");
|
|
1621
|
+
bytes = linkTarget.byteLength;
|
|
1622
|
+
}
|
|
1623
|
+
else {
|
|
1624
|
+
bytes = statSync(file).size;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
catch {
|
|
1628
|
+
return [];
|
|
1629
|
+
}
|
|
1630
|
+
if (bytes > CHUNK_BYTES)
|
|
1631
|
+
return [];
|
|
1632
|
+
return [{
|
|
1633
|
+
artifact,
|
|
1634
|
+
file,
|
|
1635
|
+
fromCache,
|
|
1636
|
+
...(linkTarget ? { linkTarget } : {}),
|
|
1637
|
+
bytes,
|
|
1638
|
+
repack: true,
|
|
1639
|
+
}];
|
|
1640
|
+
}));
|
|
1502
1641
|
const stale = (source, error) => {
|
|
1503
1642
|
if (source.fromCache) {
|
|
1504
1643
|
// A cache copy that no longer proves its hash is torn evidence:
|
|
@@ -1518,8 +1657,11 @@ export class UserGroundHost {
|
|
|
1518
1657
|
const small = sources.filter((source) => source.bytes <= CHUNK_BYTES);
|
|
1519
1658
|
const large = sources.filter((source) => source.bytes > CHUNK_BYTES);
|
|
1520
1659
|
let cargoSegments = 0;
|
|
1660
|
+
let repackedArtifacts = 0;
|
|
1661
|
+
let repackedBytes = 0;
|
|
1521
1662
|
const sendCargoSegment = async (segment) => {
|
|
1522
|
-
const loaded = new Array(segment.items.length);
|
|
1663
|
+
const loaded = new Array(segment.items.length).fill(null);
|
|
1664
|
+
const dropped = [];
|
|
1523
1665
|
await boundedForEach(segment.items.map((item, index) => ({ source: item.source, index })), CARGO_SEGMENT_LOAD_CONCURRENCY, async ({ source, index }) => {
|
|
1524
1666
|
try {
|
|
1525
1667
|
const bytes = source.linkTarget ?? await legs.time("loadMs", () => readFile(source.file));
|
|
@@ -1530,42 +1672,50 @@ export class UserGroundHost {
|
|
|
1530
1672
|
};
|
|
1531
1673
|
}
|
|
1532
1674
|
catch (error) {
|
|
1675
|
+
// A repack source that churned or vanished under this read is
|
|
1676
|
+
// dropped, never stale: the artifact stays sealed standalone,
|
|
1677
|
+
// and it is answered as sealed because it truthfully is.
|
|
1678
|
+
if (source.repack) {
|
|
1679
|
+
dropped.push(source.artifact.contentHash);
|
|
1680
|
+
return;
|
|
1681
|
+
}
|
|
1533
1682
|
if (sourceMismatch(error))
|
|
1534
1683
|
stale(source, error);
|
|
1535
1684
|
throw error;
|
|
1536
1685
|
}
|
|
1537
1686
|
});
|
|
1538
|
-
const
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1687
|
+
const shipping = segment.items.flatMap((item, index) => {
|
|
1688
|
+
const load = loaded[index];
|
|
1689
|
+
return load === null ? [] : [{ item, load }];
|
|
1690
|
+
});
|
|
1691
|
+
if (shipping.length === 0)
|
|
1692
|
+
return dropped;
|
|
1693
|
+
const { sealed, wireBytes } = await this.putCargo(resourceId, shipping.map(({ item, load }) => ({
|
|
1694
|
+
content_hash: item.contentHash,
|
|
1695
|
+
bytes: load.manifest.bytes,
|
|
1696
|
+
chunks: load.manifest.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
|
|
1697
|
+
})), Buffer.concat(shipping.map(({ load }) => load.bytes)), legs);
|
|
1698
|
+
for (const { item, load } of shipping) {
|
|
1699
|
+
uploadedChunks += load.manifest.chunks.length;
|
|
1700
|
+
if (item.source.repack) {
|
|
1701
|
+
repackedArtifacts += 1;
|
|
1702
|
+
repackedBytes += load.manifest.bytes;
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1545
1705
|
uploadedBytes += wireBytes;
|
|
1546
1706
|
cargoSegments += 1;
|
|
1547
|
-
return sealed;
|
|
1707
|
+
return [...sealed, ...dropped];
|
|
1548
1708
|
};
|
|
1549
1709
|
const transfer = async (source) => {
|
|
1550
1710
|
const { artifact, file, manifest: knownManifest, bytes: fileBytes } = source;
|
|
1551
1711
|
const sourceFile = { handle: null };
|
|
1552
1712
|
let manifestPresent = false;
|
|
1553
|
-
const
|
|
1713
|
+
const chunkBlocks = (manifest, indexes) => indexes.map((index) => {
|
|
1554
1714
|
const chunk = manifest.chunks[index];
|
|
1555
1715
|
if (!chunk)
|
|
1556
1716
|
throw new Error(`content manifest has no chunk ${index}`);
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
resource_id: resourceId,
|
|
1560
|
-
content_hash: artifact.contentHash,
|
|
1561
|
-
kind: "chunk",
|
|
1562
|
-
part_index: index,
|
|
1563
|
-
part_count: manifest.chunks.length,
|
|
1564
|
-
sha256: chunk.sha256,
|
|
1565
|
-
bytes: chunk.bytes,
|
|
1566
|
-
}, bytes, ["private.entity-content.stored"])));
|
|
1567
|
-
uploadedBytes += bytes.byteLength;
|
|
1568
|
-
};
|
|
1717
|
+
return { sha256: chunk.sha256, bytes: chunk.bytes };
|
|
1718
|
+
});
|
|
1569
1719
|
try {
|
|
1570
1720
|
const receipt = await uploadArtifact(artifact, {
|
|
1571
1721
|
bytes: fileBytes,
|
|
@@ -1595,40 +1745,16 @@ export class UserGroundHost {
|
|
|
1595
1745
|
}
|
|
1596
1746
|
return frame.missing;
|
|
1597
1747
|
},
|
|
1598
|
-
putChunk: async (_candidate, manifest, index, bytes) =>
|
|
1748
|
+
putChunk: async (_candidate, manifest, index, bytes) => {
|
|
1749
|
+
uploadedBytes += await this.putContentBlocks(resourceId, chunkBlocks(manifest, [index]), bytes, legs);
|
|
1750
|
+
},
|
|
1599
1751
|
putChunkBatch: async (_candidate, manifest, indexes, bytes) => {
|
|
1600
|
-
|
|
1601
|
-
await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1602
|
-
type: "private.entity-content.put-batch",
|
|
1603
|
-
resource_id: resourceId,
|
|
1604
|
-
content_hash: artifact.contentHash,
|
|
1605
|
-
part_count: manifest.chunks.length,
|
|
1606
|
-
chunks: indexes.map((index) => ({
|
|
1607
|
-
part_index: index,
|
|
1608
|
-
sha256: manifest.chunks[index]?.sha256,
|
|
1609
|
-
bytes: manifest.chunks[index]?.bytes,
|
|
1610
|
-
})),
|
|
1611
|
-
bytes: bytes.byteLength,
|
|
1612
|
-
wire_bytes: encoded.bytes.byteLength,
|
|
1613
|
-
...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
|
|
1614
|
-
}, encoded.bytes, ["private.entity-content.stored-batch"])));
|
|
1615
|
-
uploadedBytes += encoded.bytes.byteLength;
|
|
1752
|
+
uploadedBytes += await this.putContentBlocks(resourceId, chunkBlocks(manifest, indexes), bytes, legs);
|
|
1616
1753
|
},
|
|
1617
1754
|
putManifest: async (_candidate, manifest) => {
|
|
1618
1755
|
if (manifestPresent)
|
|
1619
1756
|
return;
|
|
1620
|
-
|
|
1621
|
-
await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1622
|
-
type: "private.entity-content.put",
|
|
1623
|
-
resource_id: resourceId,
|
|
1624
|
-
content_hash: artifact.contentHash,
|
|
1625
|
-
kind: "manifest",
|
|
1626
|
-
part_index: manifest.chunks.length,
|
|
1627
|
-
part_count: manifest.chunks.length,
|
|
1628
|
-
sha256: sha256Hex(bytes),
|
|
1629
|
-
bytes: bytes.length,
|
|
1630
|
-
}, bytes, ["private.entity-content.stored"])));
|
|
1631
|
-
uploadedBytes += bytes.byteLength;
|
|
1757
|
+
uploadedBytes += await this.putContentManifest(resourceId, manifest, legs);
|
|
1632
1758
|
},
|
|
1633
1759
|
});
|
|
1634
1760
|
uploadedChunks += receipt.uploadedChunks;
|
|
@@ -1645,7 +1771,7 @@ export class UserGroundHost {
|
|
|
1645
1771
|
}
|
|
1646
1772
|
};
|
|
1647
1773
|
try {
|
|
1648
|
-
await streamCargo(small.map((source) => ({
|
|
1774
|
+
await streamCargo([...small, ...repackSources].map((source) => ({
|
|
1649
1775
|
contentHash: source.artifact.contentHash,
|
|
1650
1776
|
bytes: source.bytes,
|
|
1651
1777
|
source,
|
|
@@ -1672,6 +1798,12 @@ export class UserGroundHost {
|
|
|
1672
1798
|
cargoBytes: small.reduce((total, source) => total + source.bytes, 0),
|
|
1673
1799
|
largeArtifacts: large.length,
|
|
1674
1800
|
largeBytes: large.reduce((total, source) => total + source.bytes, 0),
|
|
1801
|
+
// Placement work: sealed-but-standalone smalls re-shipped into
|
|
1802
|
+
// packs by this guarantor. Candidates counts what the inventory
|
|
1803
|
+
// named; repacked counts what actually shipped this pass.
|
|
1804
|
+
repackCandidates: unpacked.length,
|
|
1805
|
+
repackedArtifacts,
|
|
1806
|
+
repackedBytes,
|
|
1675
1807
|
uploadedBytes,
|
|
1676
1808
|
loadedBytes,
|
|
1677
1809
|
uploadedChunks,
|
|
@@ -1701,10 +1833,11 @@ export class UserGroundHost {
|
|
|
1701
1833
|
}
|
|
1702
1834
|
}
|
|
1703
1835
|
/** Download a set of immutable artifacts straight from content storage.
|
|
1704
|
-
* The set is the primitive
|
|
1705
|
-
*
|
|
1706
|
-
*
|
|
1707
|
-
*
|
|
1836
|
+
* The set is the primitive and the plan is physical: the engine resolves
|
|
1837
|
+
* the whole want-list in two planned waves — manifests grouped by the
|
|
1838
|
+
* pack that holds them, then blocks the same way — so all members of one
|
|
1839
|
+
* pack collapse into a few broad ranged reads while standalone objects
|
|
1840
|
+
* stay one read each. A lone artifact is a set of one. */
|
|
1708
1841
|
async downloadContentSet(resourceId, cacheDir, wants, retryAttempts = 10, trace, staged) {
|
|
1709
1842
|
const unique = new Map(wants.map((want) => [want.artifact.contentHash, want]));
|
|
1710
1843
|
const sole = unique.size === 1 ? [...unique.values()][0]?.artifact : undefined;
|
|
@@ -1745,56 +1878,75 @@ export class UserGroundHost {
|
|
|
1745
1878
|
throw new Error("download cache has no content manifest");
|
|
1746
1879
|
return found;
|
|
1747
1880
|
};
|
|
1748
|
-
//
|
|
1749
|
-
//
|
|
1750
|
-
//
|
|
1751
|
-
//
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1881
|
+
// An Add stages ordinary files directly at their hidden final
|
|
1882
|
+
// destination — except content this machine already holds sealed,
|
|
1883
|
+
// which stays on the cache rail so a warm re-Add remains a local
|
|
1884
|
+
// clone instead of a fresh download.
|
|
1885
|
+
const route = (artifact, manifest) => {
|
|
1886
|
+
const stagedPaths = stagedSet && staged?.destinations.get(artifact.contentHash);
|
|
1887
|
+
if (stagedPaths && !hasSealedContent(cacheDir, artifact.contentHash, manifest.bytes)) {
|
|
1888
|
+
stagedArtifacts += 1;
|
|
1889
|
+
targets.set(artifact.contentHash, stagedSet.target(artifact, manifest, stagedPaths));
|
|
1890
|
+
}
|
|
1891
|
+
else {
|
|
1892
|
+
targets.set(artifact.contentHash, downloads.target(artifact, manifest));
|
|
1893
|
+
}
|
|
1758
1894
|
};
|
|
1759
1895
|
try {
|
|
1760
|
-
|
|
1896
|
+
// Every storage object — manifests and blocks alike — is fetched
|
|
1897
|
+
// straight from content storage over a presigned URL and opened
|
|
1898
|
+
// locally; the engine plans whole waves against these primitives, so
|
|
1899
|
+
// the wire carries only the key handoff and batched refs requests. A
|
|
1900
|
+
// retried read refreshes its URL but never its geometry: placement is
|
|
1901
|
+
// immutable, and a moved answer fails loudly instead of retrying.
|
|
1902
|
+
const { receipts, shape } = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
|
|
1761
1903
|
sha256Hex: timedSha256Hex,
|
|
1762
|
-
|
|
1904
|
+
localManifest: (artifact) => legs.timeSync("planMs", () => {
|
|
1763
1905
|
// The record's plan or this machine's own manifest cache already
|
|
1764
1906
|
// proves chunk identity; downloaded bytes are still hash-verified,
|
|
1765
1907
|
// so a stale local manifest can only fail loudly, never corrupt.
|
|
1766
|
-
const local =
|
|
1767
|
-
?? readCachedManifest(cacheDir, artifact.contentHash)
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1908
|
+
const local = unique.get(artifact.contentHash)?.manifest
|
|
1909
|
+
?? readCachedManifest(cacheDir, artifact.contentHash);
|
|
1910
|
+
if (local)
|
|
1911
|
+
route(artifact, local);
|
|
1912
|
+
return local ?? null;
|
|
1913
|
+
}),
|
|
1914
|
+
requestRefs: async (objects) => legs.time("fetchMs", () => retryContent(() => this.direct.requestRefs(resourceId, objects), true, retryAttempts)),
|
|
1915
|
+
fetchSpan: async (url, span, refresh) => legs.time("fetchMs", async () => {
|
|
1916
|
+
let attempt = 0;
|
|
1917
|
+
return retryContent(async () => {
|
|
1918
|
+
let signed = url;
|
|
1919
|
+
if (attempt > 0) {
|
|
1920
|
+
try {
|
|
1921
|
+
signed = await refresh();
|
|
1922
|
+
}
|
|
1923
|
+
catch (error) {
|
|
1924
|
+
// A moved placement (or a refused re-resolution) is a
|
|
1925
|
+
// planning fact, not a transient fetch fault — surface it
|
|
1926
|
+
// without burning the remaining attempts.
|
|
1927
|
+
throw new WireRequestError({
|
|
1928
|
+
type: "shared.error",
|
|
1929
|
+
code: "entity_content_moved",
|
|
1930
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1931
|
+
status: 409,
|
|
1932
|
+
retryable: false,
|
|
1933
|
+
});
|
|
1934
|
+
}
|
|
1787
1935
|
}
|
|
1788
|
-
|
|
1789
|
-
|
|
1936
|
+
attempt += 1;
|
|
1937
|
+
return this.direct.fetchSpan(signed, span);
|
|
1938
|
+
}, true, retryAttempts);
|
|
1939
|
+
}),
|
|
1940
|
+
openEnvelope: async (_object, sealed) => {
|
|
1941
|
+
requests += 1;
|
|
1942
|
+
const plain = await legs.time("fetchMs", () => this.direct.openEnvelope(resourceId, sealed));
|
|
1943
|
+
downloadedBytes += plain.byteLength;
|
|
1944
|
+
return plain;
|
|
1790
1945
|
},
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
const chunk = manifest.chunks[index];
|
|
1794
|
-
if (!chunk)
|
|
1795
|
-
throw new Error(`content manifest has no chunk ${index}`);
|
|
1796
|
-
return fetchObject({ kind: "block", sha256: chunk.sha256 });
|
|
1946
|
+
manifestProven: async (artifact, manifest) => {
|
|
1947
|
+
legs.timeSync("planMs", () => route(artifact, manifest));
|
|
1797
1948
|
},
|
|
1949
|
+
hasChunk: async (artifact, _manifest, index) => legs.time("reuseMs", () => target(artifact).hasChunk(index)),
|
|
1798
1950
|
putChunk: async (artifact, _manifest, index, bytes) => legs.time("writeMs", () => target(artifact).putChunk(index, bytes)),
|
|
1799
1951
|
sealArtifact: async (artifact) => {
|
|
1800
1952
|
const sealed = await legs.time("sealMs", () => target(artifact).seal());
|
|
@@ -1835,6 +1987,10 @@ export class UserGroundHost {
|
|
|
1835
1987
|
// nothing in flight. The staged rail decomposes its own write and
|
|
1836
1988
|
// seal internals below; `stagedArtifacts` counts its placements.
|
|
1837
1989
|
requests,
|
|
1990
|
+
// The engine's own geometry: how manifests were known (local vs
|
|
1991
|
+
// fetched, packed vs standalone) and how many planned pack spans
|
|
1992
|
+
// carried the packed objects.
|
|
1993
|
+
...shape,
|
|
1838
1994
|
...physicalPlaneDelta(planeBefore, this.direct.metrics()),
|
|
1839
1995
|
...legs.report(),
|
|
1840
1996
|
...(stagedSet ? stagedSet.report() : {}),
|
|
@@ -2415,27 +2571,35 @@ export class UserGroundHost {
|
|
|
2415
2571
|
});
|
|
2416
2572
|
}
|
|
2417
2573
|
/** Which of these heads does the authority hold sealed under the current
|
|
2418
|
-
* content contract
|
|
2419
|
-
*
|
|
2420
|
-
|
|
2574
|
+
* content contract — and which of the sealed ones are packed, their whole
|
|
2575
|
+
* closure inside immutable packs? Batched so a whole workspace costs a
|
|
2576
|
+
* handful of frames. `sealed` is the only permission to skip an upload
|
|
2577
|
+
* for correctness; `packed` is placement knowledge only. The packed
|
|
2578
|
+
* answer is a protocol 16 capability and is required: a missing field
|
|
2579
|
+
* means an older gateway, and this machine refuses to guess placement. */
|
|
2580
|
+
async sealInventory(resourceId, hashes) {
|
|
2421
2581
|
const batches = [];
|
|
2422
2582
|
for (let start = 0; start < hashes.length; start += CONTENT_REFS_BATCH_OBJECTS) {
|
|
2423
2583
|
batches.push(hashes.slice(start, start + CONTENT_REFS_BATCH_OBJECTS));
|
|
2424
2584
|
}
|
|
2425
2585
|
const sealed = new Set();
|
|
2586
|
+
const packed = new Set();
|
|
2426
2587
|
const answers = await Promise.all(batches.map((batch) => retryContent(() => this.wire.request({
|
|
2427
2588
|
type: "private.entity-content.sealed",
|
|
2428
2589
|
resource_id: resourceId,
|
|
2429
2590
|
content_hashes: batch,
|
|
2430
2591
|
}, ["private.entity-content.sealed-result"]))));
|
|
2431
2592
|
for (const frame of answers) {
|
|
2432
|
-
if (!Array.isArray(frame.sealed)) {
|
|
2433
|
-
throw new Error("content authority answered
|
|
2593
|
+
if (!Array.isArray(frame.sealed) || !Array.isArray(frame.packed)) {
|
|
2594
|
+
throw new Error("content authority answered a seal inventory without "
|
|
2595
|
+
+ "placement; protocol 16 requires the gateway to deploy before this machine");
|
|
2434
2596
|
}
|
|
2435
2597
|
for (const hash of frame.sealed)
|
|
2436
2598
|
sealed.add(String(hash));
|
|
2599
|
+
for (const hash of frame.packed)
|
|
2600
|
+
packed.add(String(hash));
|
|
2437
2601
|
}
|
|
2438
|
-
return sealed;
|
|
2602
|
+
return { sealed, packed };
|
|
2439
2603
|
}
|
|
2440
2604
|
async publishPendingSnapshots(identity, trace, manifestHints) {
|
|
2441
2605
|
for (const pending of readSnapshotPublications(this.databasePath(identity))) {
|
|
@@ -2456,7 +2620,7 @@ export class UserGroundHost {
|
|
|
2456
2620
|
return;
|
|
2457
2621
|
const started = performance.now();
|
|
2458
2622
|
const legs = new TransferLegs();
|
|
2459
|
-
const sealed = await legs.time("planMs", () => this.
|
|
2623
|
+
const { sealed } = await legs.time("planMs", () => this.sealInventory(resourceId, pages.map((page) => page.sha256)));
|
|
2460
2624
|
const unsealed = pages
|
|
2461
2625
|
.filter((page) => !sealed.has(page.sha256))
|
|
2462
2626
|
.map((page) => ({ contentHash: page.sha256, bytes: page.bytes.byteLength, page }));
|
|
@@ -4102,7 +4266,7 @@ function materializedWorkspaceBindings(bindingDir) {
|
|
|
4102
4266
|
return bindings.sort((left, right) => left.workspaceId.localeCompare(right.workspaceId));
|
|
4103
4267
|
}
|
|
4104
4268
|
async function reconcileGround(input) {
|
|
4105
|
-
const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, measuring = false, } = input;
|
|
4269
|
+
const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, fusion, onScan, measuring = false, } = input;
|
|
4106
4270
|
const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
|
|
4107
4271
|
const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
|
|
4108
4272
|
const boundRootEntries = materializedWorkspaceBindings(bindingDir);
|
|
@@ -4328,9 +4492,11 @@ async function reconcileGround(input) {
|
|
|
4328
4492
|
suspicion: observationFor(rootPath),
|
|
4329
4493
|
// A register scan publishes one complete snapshot; it never mints Send
|
|
4330
4494
|
// records for the same initial contents, so it plans no detection and
|
|
4331
|
-
// stages no cache bytes —
|
|
4495
|
+
// stages no cache bytes — the fused rail ships content from the walk's
|
|
4496
|
+
// own read, and upload re-reads only what fusion could not seal.
|
|
4332
4497
|
plansSendRecords: suspicions !== undefined && registrationWorkspaceId === undefined,
|
|
4333
4498
|
largeManifests,
|
|
4499
|
+
fusion,
|
|
4334
4500
|
});
|
|
4335
4501
|
}
|
|
4336
4502
|
const changedUUIDs = new Set([
|
|
@@ -4358,7 +4524,7 @@ async function reconcileGround(input) {
|
|
|
4358
4524
|
};
|
|
4359
4525
|
}
|
|
4360
4526
|
async function reconcileRoot(input) {
|
|
4361
|
-
const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, } = input;
|
|
4527
|
+
const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, fusion, } = input;
|
|
4362
4528
|
// Prior rows reach this scan from every binding this root encloses. A row's
|
|
4363
4529
|
// stored relativePath is relative to the root that last committed it, which
|
|
4364
4530
|
// is a different frame when an enclosing boundary was just registered
|
|
@@ -4711,9 +4877,20 @@ async function reconcileRoot(input) {
|
|
|
4711
4877
|
captureTasks.push(captureGate(async () => {
|
|
4712
4878
|
const taskStarted = measureEvidence ? performance.now() : 0;
|
|
4713
4879
|
let current = stats;
|
|
4880
|
+
// Repository leaves ship on the Git-native rail; everything else
|
|
4881
|
+
// may fuse its capture read with upload. Fusion is opportunistic:
|
|
4882
|
+
// once disabled, captures continue exactly as the classic path
|
|
4883
|
+
// and the upload stage ships what the walk could not.
|
|
4884
|
+
const fused = activeRepository ? undefined : fusion;
|
|
4714
4885
|
for (let attempt = 1;; attempt++) {
|
|
4715
4886
|
try {
|
|
4716
|
-
const
|
|
4887
|
+
const collect = fused?.enabled && current.size <= CHUNK_BYTES
|
|
4888
|
+
? contentBlockCollector(current.size)
|
|
4889
|
+
: null;
|
|
4890
|
+
const captured = fused && current.size > CHUNK_BYTES
|
|
4891
|
+
? (await fused.captureLarge(absolutePath, current))
|
|
4892
|
+
?? await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir)
|
|
4893
|
+
: await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir, collect);
|
|
4717
4894
|
if (measureEvidence && evidence) {
|
|
4718
4895
|
evidence.captureBusyMs += performance.now() - taskStarted;
|
|
4719
4896
|
evidence.captureBytes += captured.bytes;
|
|
@@ -4726,6 +4903,12 @@ async function reconcileRoot(input) {
|
|
|
4726
4903
|
if (captured.bytes > CHUNK_BYTES) {
|
|
4727
4904
|
largeManifests?.set(captured.contentHash, captured.manifest);
|
|
4728
4905
|
}
|
|
4906
|
+
// The capture proved these exact bytes against the fingerprint
|
|
4907
|
+
// law; hand them to the cargo rail. Waiting here is window
|
|
4908
|
+
// backpressure, bounding memory by the capture pool's width.
|
|
4909
|
+
if (collect && fused) {
|
|
4910
|
+
await fused.addSmall(absolutePath, current, captured, collect.collected);
|
|
4911
|
+
}
|
|
4729
4912
|
return;
|
|
4730
4913
|
}
|
|
4731
4914
|
catch (error) {
|