@amalgm/shell 0.1.134 → 0.1.136
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bounded-for-each.d.ts +3 -0
- package/dist/bounded-for-each.js +26 -0
- package/dist/bounded-for-each.js.map +1 -0
- package/dist/content-direct.d.ts +21 -13
- package/dist/content-direct.js +70 -68
- package/dist/content-direct.js.map +1 -1
- package/dist/filesystem.d.ts +3 -0
- package/dist/filesystem.js +11 -0
- package/dist/filesystem.js.map +1 -1
- package/dist/transfer-instrumentation.d.ts +18 -0
- package/dist/transfer-instrumentation.js +70 -0
- package/dist/transfer-instrumentation.js.map +1 -1
- package/dist/user-ground-host.d.ts +12 -7
- package/dist/user-ground-host.js +221 -100
- package/dist/user-ground-host.js.map +1 -1
- package/dist/wire-client.js +1 -1
- package/dist/workspace-add-staging.d.ts +98 -34
- package/dist/workspace-add-staging.js +255 -149
- package/dist/workspace-add-staging.js.map +1 -1
- package/npm-shrinkwrap.json +6 -6
- package/package.json +2 -2
package/dist/user-ground-host.js
CHANGED
|
@@ -14,6 +14,7 @@ import { indexRowsByOutermostRoot } from "./ground-root-index.js";
|
|
|
14
14
|
import { ContentCacheDownloadSet, captureContentFile, contentBlockCollector, hasSealedContent, hashContentFile, pruneSettledContentCache, sealCapturedArtifact, } from "./content-cache-host.js";
|
|
15
15
|
import { RegisterUploadFusion } from "./register-upload-fusion.js";
|
|
16
16
|
import { StagedContentSet, addStagingEvidenceDir, clearAddStagingEvidence, } from "./workspace-add-staging.js";
|
|
17
|
+
import { boundedForEach } from "./bounded-for-each.js";
|
|
17
18
|
import { DirectContentPlane } from "./content-direct.js";
|
|
18
19
|
import { encodeContentWireBytes } from "./content-wire-codec.js";
|
|
19
20
|
import { TransferLegs } from "./transfer-instrumentation.js";
|
|
@@ -38,8 +39,8 @@ const CARGO_SEGMENT_LOAD_CONCURRENCY = 8;
|
|
|
38
39
|
/** Storage-object lanes for one download set. Tiny objects use the wide
|
|
39
40
|
* request window, while the separate byte budget narrows large chunks so
|
|
40
41
|
* request concurrency can never silently become an unbounded memory claim. */
|
|
41
|
-
/** Admission is sized to the refs batch law so one
|
|
42
|
-
*
|
|
42
|
+
/** Admission is sized to the refs batch law so one planned wave resolves in
|
|
43
|
+
* one refs frame and the span planner merges each pack's members into a
|
|
43
44
|
* few broad ranged reads. Memory is bounded by the byte laws below, never by
|
|
44
45
|
* this count. */
|
|
45
46
|
const DIRECT_OBJECT_DOWNLOAD_CONCURRENCY = CONTENT_REFS_BATCH_OBJECTS;
|
|
@@ -400,7 +401,7 @@ export class UserGroundHost {
|
|
|
400
401
|
});
|
|
401
402
|
}
|
|
402
403
|
},
|
|
403
|
-
onStage: async (stage) => {
|
|
404
|
+
onStage: async (stage, measurements) => {
|
|
404
405
|
const now = performance.now();
|
|
405
406
|
this.stage({
|
|
406
407
|
primitive: stage === "watching" ? "watch" : "download",
|
|
@@ -408,6 +409,7 @@ export class UserGroundHost {
|
|
|
408
409
|
status: "completed",
|
|
409
410
|
workspaceId: input.workspace.uuid,
|
|
410
411
|
durationMs: now - milestoneAt,
|
|
412
|
+
...(measurements ? { measurements } : {}),
|
|
411
413
|
});
|
|
412
414
|
milestoneAt = now;
|
|
413
415
|
await this.options.onWorkspaceAddStage?.({
|
|
@@ -685,7 +687,14 @@ export class UserGroundHost {
|
|
|
685
687
|
const fusionResourceId = this.cloudState.resourceId;
|
|
686
688
|
const fusionLegs = new TransferLegs();
|
|
687
689
|
const fusion = new RegisterUploadFusion({
|
|
688
|
-
|
|
690
|
+
// Fusion ships whole small artifacts into packs; skipping one
|
|
691
|
+
// requires it to be both sealed and packed, so sealed-but-
|
|
692
|
+
// standalone smalls re-ship here and placement converges on
|
|
693
|
+
// packs without a separate migration.
|
|
694
|
+
sealedContentHashes: async (hashes) => {
|
|
695
|
+
const inventory = await this.sealInventory(fusionResourceId, hashes);
|
|
696
|
+
return new Set([...inventory.sealed].filter((hash) => inventory.packed.has(hash)));
|
|
697
|
+
},
|
|
689
698
|
sendCargo: async (artifacts, payload) => {
|
|
690
699
|
const { sealed } = await this.putCargo(fusionResourceId, artifacts, payload, fusionLegs);
|
|
691
700
|
return sealed;
|
|
@@ -923,10 +932,11 @@ export class UserGroundHost {
|
|
|
923
932
|
throw new Error("entity Record rail already started");
|
|
924
933
|
await this.wire.open();
|
|
925
934
|
// One floor, no per-feature gates: this shell and its gateway release
|
|
926
|
-
// together, and the gateway ships first.
|
|
927
|
-
//
|
|
928
|
-
//
|
|
929
|
-
|
|
935
|
+
// together, and the gateway ships first. 16 = placement answers on the
|
|
936
|
+
// seal inventory; an older gateway cannot say which sealed artifacts are
|
|
937
|
+
// packed, and this shell plans pack-span downloads and repacks against
|
|
938
|
+
// exactly that answer.
|
|
939
|
+
if (this.wire.protocolVersion < 16) {
|
|
930
940
|
throw new Error("gateway speaks an older wire protocol than this shell");
|
|
931
941
|
}
|
|
932
942
|
this.adoptCloudDeliveryBaseline(identity, this.cloudState.deliveryWatermark);
|
|
@@ -1538,11 +1548,16 @@ export class UserGroundHost {
|
|
|
1538
1548
|
// The authority's seal inventory is the only permission to skip an
|
|
1539
1549
|
// upload — asked before any source is touched, so sealed artifacts cost
|
|
1540
1550
|
// zero local reads and a retried publication whose upload already sealed
|
|
1541
|
-
// completely never needs its captured ground to still exist.
|
|
1542
|
-
|
|
1551
|
+
// completely never needs its captured ground to still exist. Placement
|
|
1552
|
+
// rides the same answer: a sealed-but-unpacked small artifact is
|
|
1553
|
+
// correctness-complete but stored inefficiently, so this guarantor also
|
|
1554
|
+
// re-ships it into a pack when its source is still at hand (axiom 44).
|
|
1555
|
+
const inventory = await legs.time("planMs", () => this.sealInventory(resourceId, [...byHash.keys()]));
|
|
1543
1556
|
const unsealed = [...byHash.values()]
|
|
1544
|
-
.filter(({ artifact }) => !
|
|
1545
|
-
const
|
|
1557
|
+
.filter(({ artifact }) => !inventory.sealed.has(artifact.contentHash));
|
|
1558
|
+
const unpacked = [...byHash.values()].filter(({ artifact }) => inventory.sealed.has(artifact.contentHash)
|
|
1559
|
+
&& !inventory.packed.has(artifact.contentHash));
|
|
1560
|
+
const rows = legs.timeSync("planMs", () => readGroundRowsByUuid(this.databasePath(identity), [...unsealed, ...unpacked].map(({ uuid }) => uuid)));
|
|
1546
1561
|
// Each artifact uploads from the strongest evidence available: a cache
|
|
1547
1562
|
// copy when another journey pinned one, otherwise the ground entity
|
|
1548
1563
|
// itself. Register stages nothing, so the ground path is the common
|
|
@@ -1589,6 +1604,42 @@ export class UserGroundHost {
|
|
|
1589
1604
|
bytes,
|
|
1590
1605
|
};
|
|
1591
1606
|
}));
|
|
1607
|
+
// Repack candidates are sealed already, so their sources are optional:
|
|
1608
|
+
// anything missing, oversized, or unreadable skips silently — a frozen
|
|
1609
|
+
// workspace simply keeps standalone placement until its next actual
|
|
1610
|
+
// publication. What readers see never changes either way.
|
|
1611
|
+
const repackSources = legs.timeSync("planMs", () => unpacked.flatMap(({ artifact, uuid }) => {
|
|
1612
|
+
const cached = contentCacheFile(cache, artifact.contentHash);
|
|
1613
|
+
const fromCache = existsSync(cached);
|
|
1614
|
+
const row = rows.get(uuid);
|
|
1615
|
+
if (!fromCache && (artifact.kind === "git" || !row))
|
|
1616
|
+
return [];
|
|
1617
|
+
const file = fromCache ? cached : row.absolutePath;
|
|
1618
|
+
let linkTarget;
|
|
1619
|
+
let bytes;
|
|
1620
|
+
try {
|
|
1621
|
+
if (!fromCache && artifact.kind === "link") {
|
|
1622
|
+
linkTarget = Buffer.from(readlinkSync(file), "utf8");
|
|
1623
|
+
bytes = linkTarget.byteLength;
|
|
1624
|
+
}
|
|
1625
|
+
else {
|
|
1626
|
+
bytes = statSync(file).size;
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
catch {
|
|
1630
|
+
return [];
|
|
1631
|
+
}
|
|
1632
|
+
if (bytes > CHUNK_BYTES)
|
|
1633
|
+
return [];
|
|
1634
|
+
return [{
|
|
1635
|
+
artifact,
|
|
1636
|
+
file,
|
|
1637
|
+
fromCache,
|
|
1638
|
+
...(linkTarget ? { linkTarget } : {}),
|
|
1639
|
+
bytes,
|
|
1640
|
+
repack: true,
|
|
1641
|
+
}];
|
|
1642
|
+
}));
|
|
1592
1643
|
const stale = (source, error) => {
|
|
1593
1644
|
if (source.fromCache) {
|
|
1594
1645
|
// A cache copy that no longer proves its hash is torn evidence:
|
|
@@ -1608,8 +1659,11 @@ export class UserGroundHost {
|
|
|
1608
1659
|
const small = sources.filter((source) => source.bytes <= CHUNK_BYTES);
|
|
1609
1660
|
const large = sources.filter((source) => source.bytes > CHUNK_BYTES);
|
|
1610
1661
|
let cargoSegments = 0;
|
|
1662
|
+
let repackedArtifacts = 0;
|
|
1663
|
+
let repackedBytes = 0;
|
|
1611
1664
|
const sendCargoSegment = async (segment) => {
|
|
1612
|
-
const loaded = new Array(segment.items.length);
|
|
1665
|
+
const loaded = new Array(segment.items.length).fill(null);
|
|
1666
|
+
const dropped = [];
|
|
1613
1667
|
await boundedForEach(segment.items.map((item, index) => ({ source: item.source, index })), CARGO_SEGMENT_LOAD_CONCURRENCY, async ({ source, index }) => {
|
|
1614
1668
|
try {
|
|
1615
1669
|
const bytes = source.linkTarget ?? await legs.time("loadMs", () => readFile(source.file));
|
|
@@ -1620,21 +1674,39 @@ export class UserGroundHost {
|
|
|
1620
1674
|
};
|
|
1621
1675
|
}
|
|
1622
1676
|
catch (error) {
|
|
1677
|
+
// A repack source that churned or vanished under this read is
|
|
1678
|
+
// dropped, never stale: the artifact stays sealed standalone,
|
|
1679
|
+
// and it is answered as sealed because it truthfully is.
|
|
1680
|
+
if (source.repack) {
|
|
1681
|
+
dropped.push(source.artifact.contentHash);
|
|
1682
|
+
return;
|
|
1683
|
+
}
|
|
1623
1684
|
if (sourceMismatch(error))
|
|
1624
1685
|
stale(source, error);
|
|
1625
1686
|
throw error;
|
|
1626
1687
|
}
|
|
1627
1688
|
});
|
|
1628
|
-
const
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1689
|
+
const shipping = segment.items.flatMap((item, index) => {
|
|
1690
|
+
const load = loaded[index];
|
|
1691
|
+
return load === null ? [] : [{ item, load }];
|
|
1692
|
+
});
|
|
1693
|
+
if (shipping.length === 0)
|
|
1694
|
+
return dropped;
|
|
1695
|
+
const { sealed, wireBytes } = await this.putCargo(resourceId, shipping.map(({ item, load }) => ({
|
|
1696
|
+
content_hash: item.contentHash,
|
|
1697
|
+
bytes: load.manifest.bytes,
|
|
1698
|
+
chunks: load.manifest.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
|
|
1699
|
+
})), Buffer.concat(shipping.map(({ load }) => load.bytes)), legs);
|
|
1700
|
+
for (const { item, load } of shipping) {
|
|
1701
|
+
uploadedChunks += load.manifest.chunks.length;
|
|
1702
|
+
if (item.source.repack) {
|
|
1703
|
+
repackedArtifacts += 1;
|
|
1704
|
+
repackedBytes += load.manifest.bytes;
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1635
1707
|
uploadedBytes += wireBytes;
|
|
1636
1708
|
cargoSegments += 1;
|
|
1637
|
-
return sealed;
|
|
1709
|
+
return [...sealed, ...dropped];
|
|
1638
1710
|
};
|
|
1639
1711
|
const transfer = async (source) => {
|
|
1640
1712
|
const { artifact, file, manifest: knownManifest, bytes: fileBytes } = source;
|
|
@@ -1701,7 +1773,7 @@ export class UserGroundHost {
|
|
|
1701
1773
|
}
|
|
1702
1774
|
};
|
|
1703
1775
|
try {
|
|
1704
|
-
await streamCargo(small.map((source) => ({
|
|
1776
|
+
await streamCargo([...small, ...repackSources].map((source) => ({
|
|
1705
1777
|
contentHash: source.artifact.contentHash,
|
|
1706
1778
|
bytes: source.bytes,
|
|
1707
1779
|
source,
|
|
@@ -1728,6 +1800,12 @@ export class UserGroundHost {
|
|
|
1728
1800
|
cargoBytes: small.reduce((total, source) => total + source.bytes, 0),
|
|
1729
1801
|
largeArtifacts: large.length,
|
|
1730
1802
|
largeBytes: large.reduce((total, source) => total + source.bytes, 0),
|
|
1803
|
+
// Placement work: sealed-but-standalone smalls re-shipped into
|
|
1804
|
+
// packs by this guarantor. Candidates counts what the inventory
|
|
1805
|
+
// named; repacked counts what actually shipped this pass.
|
|
1806
|
+
repackCandidates: unpacked.length,
|
|
1807
|
+
repackedArtifacts,
|
|
1808
|
+
repackedBytes,
|
|
1731
1809
|
uploadedBytes,
|
|
1732
1810
|
loadedBytes,
|
|
1733
1811
|
uploadedChunks,
|
|
@@ -1757,10 +1835,11 @@ export class UserGroundHost {
|
|
|
1757
1835
|
}
|
|
1758
1836
|
}
|
|
1759
1837
|
/** Download a set of immutable artifacts straight from content storage.
|
|
1760
|
-
* The set is the primitive
|
|
1761
|
-
*
|
|
1762
|
-
*
|
|
1763
|
-
*
|
|
1838
|
+
* The set is the primitive and the plan is physical: the engine resolves
|
|
1839
|
+
* the whole want-list in two planned waves — manifests grouped by the
|
|
1840
|
+
* pack that holds them, then blocks the same way — so all members of one
|
|
1841
|
+
* pack collapse into a few broad ranged reads while standalone objects
|
|
1842
|
+
* stay one read each. A lone artifact is a set of one. */
|
|
1764
1843
|
async downloadContentSet(resourceId, cacheDir, wants, retryAttempts = 10, trace, staged) {
|
|
1765
1844
|
const unique = new Map(wants.map((want) => [want.artifact.contentHash, want]));
|
|
1766
1845
|
const sole = unique.size === 1 ? [...unique.values()][0]?.artifact : undefined;
|
|
@@ -1793,7 +1872,7 @@ export class UserGroundHost {
|
|
|
1793
1872
|
},
|
|
1794
1873
|
});
|
|
1795
1874
|
const downloads = new ContentCacheDownloadSet(cacheDir);
|
|
1796
|
-
|
|
1875
|
+
let stagedSet = null;
|
|
1797
1876
|
const targets = new Map();
|
|
1798
1877
|
const target = (artifact) => {
|
|
1799
1878
|
const found = targets.get(artifact.contentHash);
|
|
@@ -1801,56 +1880,81 @@ export class UserGroundHost {
|
|
|
1801
1880
|
throw new Error("download cache has no content manifest");
|
|
1802
1881
|
return found;
|
|
1803
1882
|
};
|
|
1804
|
-
//
|
|
1805
|
-
//
|
|
1806
|
-
//
|
|
1807
|
-
//
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1883
|
+
// An Add stages ordinary files directly at their hidden final
|
|
1884
|
+
// destination — except content this machine already holds sealed,
|
|
1885
|
+
// which stays on the cache rail so a warm re-Add remains a local
|
|
1886
|
+
// clone instead of a fresh download.
|
|
1887
|
+
const route = (artifact, manifest) => {
|
|
1888
|
+
const set = stagedSet;
|
|
1889
|
+
const stagedPaths = set && staged?.destinations.get(artifact.contentHash);
|
|
1890
|
+
if (set && stagedPaths && !hasSealedContent(cacheDir, artifact.contentHash, manifest.bytes)) {
|
|
1891
|
+
stagedArtifacts += 1;
|
|
1892
|
+
targets.set(artifact.contentHash, set.target(artifact, manifest, stagedPaths));
|
|
1893
|
+
}
|
|
1894
|
+
else {
|
|
1895
|
+
targets.set(artifact.contentHash, downloads.target(artifact, manifest));
|
|
1896
|
+
}
|
|
1814
1897
|
};
|
|
1815
1898
|
try {
|
|
1816
|
-
|
|
1899
|
+
// The staged rail's directory skeleton is the set's first act, so no
|
|
1900
|
+
// chunk's store path includes a mkdir.
|
|
1901
|
+
stagedSet = staged
|
|
1902
|
+
? await StagedContentSet.open(staged.evidenceDir, staged.destinations)
|
|
1903
|
+
: null;
|
|
1904
|
+
// Every storage object — manifests and blocks alike — is fetched
|
|
1905
|
+
// straight from content storage over a presigned URL and opened
|
|
1906
|
+
// locally; the engine plans whole waves against these primitives, so
|
|
1907
|
+
// the wire carries only the key handoff and batched refs requests. A
|
|
1908
|
+
// retried read refreshes its URL but never its geometry: placement is
|
|
1909
|
+
// immutable, and a moved answer fails loudly instead of retrying.
|
|
1910
|
+
const { receipts, shape } = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
|
|
1817
1911
|
sha256Hex: timedSha256Hex,
|
|
1818
|
-
|
|
1912
|
+
localManifest: (artifact) => legs.timeSync("planMs", () => {
|
|
1819
1913
|
// The record's plan or this machine's own manifest cache already
|
|
1820
1914
|
// proves chunk identity; downloaded bytes are still hash-verified,
|
|
1821
1915
|
// so a stale local manifest can only fail loudly, never corrupt.
|
|
1822
|
-
const local =
|
|
1823
|
-
?? readCachedManifest(cacheDir, artifact.contentHash)
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1916
|
+
const local = unique.get(artifact.contentHash)?.manifest
|
|
1917
|
+
?? readCachedManifest(cacheDir, artifact.contentHash);
|
|
1918
|
+
if (local)
|
|
1919
|
+
route(artifact, local);
|
|
1920
|
+
return local ?? null;
|
|
1921
|
+
}),
|
|
1922
|
+
requestRefs: async (objects) => legs.time("fetchMs", () => retryContent(() => this.direct.requestRefs(resourceId, objects), true, retryAttempts)),
|
|
1923
|
+
fetchSpan: async (url, span, refresh) => legs.time("fetchMs", async () => {
|
|
1924
|
+
let attempt = 0;
|
|
1925
|
+
return retryContent(async () => {
|
|
1926
|
+
let signed = url;
|
|
1927
|
+
if (attempt > 0) {
|
|
1928
|
+
try {
|
|
1929
|
+
signed = await refresh();
|
|
1930
|
+
}
|
|
1931
|
+
catch (error) {
|
|
1932
|
+
// A moved placement (or a refused re-resolution) is a
|
|
1933
|
+
// planning fact, not a transient fetch fault — surface it
|
|
1934
|
+
// without burning the remaining attempts.
|
|
1935
|
+
throw new WireRequestError({
|
|
1936
|
+
type: "shared.error",
|
|
1937
|
+
code: "entity_content_moved",
|
|
1938
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1939
|
+
status: 409,
|
|
1940
|
+
retryable: false,
|
|
1941
|
+
});
|
|
1942
|
+
}
|
|
1843
1943
|
}
|
|
1844
|
-
|
|
1845
|
-
|
|
1944
|
+
attempt += 1;
|
|
1945
|
+
return this.direct.fetchSpan(signed, span);
|
|
1946
|
+
}, true, retryAttempts);
|
|
1947
|
+
}),
|
|
1948
|
+
openEnvelope: async (_object, sealed) => {
|
|
1949
|
+
requests += 1;
|
|
1950
|
+
const plain = await legs.time("fetchMs", () => this.direct.openEnvelope(resourceId, sealed));
|
|
1951
|
+
downloadedBytes += plain.byteLength;
|
|
1952
|
+
return plain;
|
|
1846
1953
|
},
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
const chunk = manifest.chunks[index];
|
|
1850
|
-
if (!chunk)
|
|
1851
|
-
throw new Error(`content manifest has no chunk ${index}`);
|
|
1852
|
-
return fetchObject({ kind: "block", sha256: chunk.sha256 });
|
|
1954
|
+
manifestProven: async (artifact, manifest) => {
|
|
1955
|
+
legs.timeSync("planMs", () => route(artifact, manifest));
|
|
1853
1956
|
},
|
|
1957
|
+
hasChunk: async (artifact, _manifest, index) => legs.time("reuseMs", () => target(artifact).hasChunk(index)),
|
|
1854
1958
|
putChunk: async (artifact, _manifest, index, bytes) => legs.time("writeMs", () => target(artifact).putChunk(index, bytes)),
|
|
1855
1959
|
sealArtifact: async (artifact) => {
|
|
1856
1960
|
const sealed = await legs.time("sealMs", () => target(artifact).seal());
|
|
@@ -1887,10 +1991,14 @@ export class UserGroundHost {
|
|
|
1887
1991
|
// plan (manifest cache + routing), fetch (storage waits, whose
|
|
1888
1992
|
// decrypt share the plane delta names), hash (chunk verification),
|
|
1889
1993
|
// reuse (retained-chunk re-proof), write (chunk placement), seal
|
|
1890
|
-
// (whole-file proof +
|
|
1891
|
-
// nothing in flight. The staged rail decomposes its own
|
|
1892
|
-
//
|
|
1994
|
+
// (whole-file proof assertion + duplicates), and idle — wall time
|
|
1995
|
+
// with nothing in flight. The staged rail decomposes its own
|
|
1996
|
+
// placement legs below; `stagedArtifacts` counts its placements.
|
|
1893
1997
|
requests,
|
|
1998
|
+
// The engine's own geometry: how manifests were known (local vs
|
|
1999
|
+
// fetched, packed vs standalone) and how many planned pack spans
|
|
2000
|
+
// carried the packed objects.
|
|
2001
|
+
...shape,
|
|
1894
2002
|
...physicalPlaneDelta(planeBefore, this.direct.metrics()),
|
|
1895
2003
|
...legs.report(),
|
|
1896
2004
|
...(stagedSet ? stagedSet.report() : {}),
|
|
@@ -2471,27 +2579,35 @@ export class UserGroundHost {
|
|
|
2471
2579
|
});
|
|
2472
2580
|
}
|
|
2473
2581
|
/** Which of these heads does the authority hold sealed under the current
|
|
2474
|
-
* content contract
|
|
2475
|
-
*
|
|
2476
|
-
|
|
2582
|
+
* content contract — and which of the sealed ones are packed, their whole
|
|
2583
|
+
* closure inside immutable packs? Batched so a whole workspace costs a
|
|
2584
|
+
* handful of frames. `sealed` is the only permission to skip an upload
|
|
2585
|
+
* for correctness; `packed` is placement knowledge only. The packed
|
|
2586
|
+
* answer is a protocol 16 capability and is required: a missing field
|
|
2587
|
+
* means an older gateway, and this machine refuses to guess placement. */
|
|
2588
|
+
async sealInventory(resourceId, hashes) {
|
|
2477
2589
|
const batches = [];
|
|
2478
2590
|
for (let start = 0; start < hashes.length; start += CONTENT_REFS_BATCH_OBJECTS) {
|
|
2479
2591
|
batches.push(hashes.slice(start, start + CONTENT_REFS_BATCH_OBJECTS));
|
|
2480
2592
|
}
|
|
2481
2593
|
const sealed = new Set();
|
|
2594
|
+
const packed = new Set();
|
|
2482
2595
|
const answers = await Promise.all(batches.map((batch) => retryContent(() => this.wire.request({
|
|
2483
2596
|
type: "private.entity-content.sealed",
|
|
2484
2597
|
resource_id: resourceId,
|
|
2485
2598
|
content_hashes: batch,
|
|
2486
2599
|
}, ["private.entity-content.sealed-result"]))));
|
|
2487
2600
|
for (const frame of answers) {
|
|
2488
|
-
if (!Array.isArray(frame.sealed)) {
|
|
2489
|
-
throw new Error("content authority answered
|
|
2601
|
+
if (!Array.isArray(frame.sealed) || !Array.isArray(frame.packed)) {
|
|
2602
|
+
throw new Error("content authority answered a seal inventory without "
|
|
2603
|
+
+ "placement; protocol 16 requires the gateway to deploy before this machine");
|
|
2490
2604
|
}
|
|
2491
2605
|
for (const hash of frame.sealed)
|
|
2492
2606
|
sealed.add(String(hash));
|
|
2607
|
+
for (const hash of frame.packed)
|
|
2608
|
+
packed.add(String(hash));
|
|
2493
2609
|
}
|
|
2494
|
-
return sealed;
|
|
2610
|
+
return { sealed, packed };
|
|
2495
2611
|
}
|
|
2496
2612
|
async publishPendingSnapshots(identity, trace, manifestHints) {
|
|
2497
2613
|
for (const pending of readSnapshotPublications(this.databasePath(identity))) {
|
|
@@ -2512,7 +2628,7 @@ export class UserGroundHost {
|
|
|
2512
2628
|
return;
|
|
2513
2629
|
const started = performance.now();
|
|
2514
2630
|
const legs = new TransferLegs();
|
|
2515
|
-
const sealed = await legs.time("planMs", () => this.
|
|
2631
|
+
const { sealed } = await legs.time("planMs", () => this.sealInventory(resourceId, pages.map((page) => page.sha256)));
|
|
2516
2632
|
const unsealed = pages
|
|
2517
2633
|
.filter((page) => !sealed.has(page.sha256))
|
|
2518
2634
|
.map((page) => ({ contentHash: page.sha256, bytes: page.bytes.byteLength, page }));
|
|
@@ -5618,9 +5734,11 @@ async function materializeRecords(input) {
|
|
|
5618
5734
|
for (const list of stagedDestinations.values())
|
|
5619
5735
|
list.sort();
|
|
5620
5736
|
}
|
|
5737
|
+
const contentStarted = performance.now();
|
|
5621
5738
|
const fileContent = await readContent([...wanted.values()], staged && stagedDestinations
|
|
5622
5739
|
? { evidenceDir: staged.evidenceDir, destinations: stagedDestinations }
|
|
5623
5740
|
: undefined);
|
|
5741
|
+
const chainStarted = performance.now();
|
|
5624
5742
|
const repositoryChains = new Map();
|
|
5625
5743
|
const walking = repositoryHeads.map(({ record, artifact }) => {
|
|
5626
5744
|
const local = fileContent.get(artifact.contentHash);
|
|
@@ -5665,11 +5783,15 @@ async function materializeRecords(input) {
|
|
|
5665
5783
|
walk.parent = inspectRepositoryTransportFile(parent.file).parentTransportVersion;
|
|
5666
5784
|
}
|
|
5667
5785
|
}
|
|
5786
|
+
const walkStarted = performance.now();
|
|
5787
|
+
let repositoriesMs = 0;
|
|
5668
5788
|
const materializeRepository = async (record, destination) => {
|
|
5669
5789
|
const chain = repositoryChains.get(record.uuid);
|
|
5670
5790
|
if (!chain)
|
|
5671
5791
|
throw new Error(`cloud repository ${record.uuid} transport was not downloaded`);
|
|
5792
|
+
const started = performance.now();
|
|
5672
5793
|
const applied = await applyRepositoryFiles(destination, chain);
|
|
5794
|
+
repositoriesMs += performance.now() - started;
|
|
5673
5795
|
const repository = { record, state: applied };
|
|
5674
5796
|
repositories.push(repository);
|
|
5675
5797
|
repositoriesByUuid.set(record.uuid, repository);
|
|
@@ -5823,6 +5945,7 @@ async function materializeRecords(input) {
|
|
|
5823
5945
|
throw new Error(`materialization for ${record.type} requires its dedicated adapter`);
|
|
5824
5946
|
}
|
|
5825
5947
|
}
|
|
5948
|
+
const rowsStarted = performance.now();
|
|
5826
5949
|
const rows = records.map((record) => {
|
|
5827
5950
|
const relativePath = paths.get(record.uuid);
|
|
5828
5951
|
const absolutePath = relativePath ? join(rootPath, ...relativePath.split("/")) : rootPath;
|
|
@@ -5836,7 +5959,18 @@ async function materializeRecords(input) {
|
|
|
5836
5959
|
...statFingerprint(stats),
|
|
5837
5960
|
};
|
|
5838
5961
|
});
|
|
5839
|
-
|
|
5962
|
+
const finished = performance.now();
|
|
5963
|
+
return {
|
|
5964
|
+
rows,
|
|
5965
|
+
repositories,
|
|
5966
|
+
timing: {
|
|
5967
|
+
contentMs: Math.round(chainStarted - contentStarted),
|
|
5968
|
+
transportChainMs: Math.round(walkStarted - chainStarted),
|
|
5969
|
+
repositoriesMs: Math.round(repositoriesMs),
|
|
5970
|
+
groundWalkMs: Math.round(rowsStarted - walkStarted - repositoriesMs),
|
|
5971
|
+
rowsMs: Math.round(finished - rowsStarted),
|
|
5972
|
+
},
|
|
5973
|
+
};
|
|
5840
5974
|
}
|
|
5841
5975
|
/** Refresh the always-materialized portable reference inventory selected by
|
|
5842
5976
|
* Live. Paths and bindings are local; UUIDs, parents, names, and versions are
|
|
@@ -6085,6 +6219,7 @@ async function installCloudWorkspace(input) {
|
|
|
6085
6219
|
staged: { evidenceDir: addStagingEvidenceDir(database, intent.workspaceId) },
|
|
6086
6220
|
readContent,
|
|
6087
6221
|
});
|
|
6222
|
+
const graphStarted = performance.now();
|
|
6088
6223
|
const rows = projectMaterializedGraph(materialized.rows, materialized.repositories.map((repository) => ({
|
|
6089
6224
|
record: repository.record,
|
|
6090
6225
|
identity: repository.state.identity,
|
|
@@ -6094,7 +6229,13 @@ async function installCloudWorkspace(input) {
|
|
|
6094
6229
|
? join(intent.destinationPath, ...row.relativePath.split("/"))
|
|
6095
6230
|
: intent.destinationPath,
|
|
6096
6231
|
}));
|
|
6097
|
-
|
|
6232
|
+
// The milestone's duration spans the whole materialization; its legs say
|
|
6233
|
+
// where that wall time went after the content set itself.
|
|
6234
|
+
await onStage?.("materialization-verified", {
|
|
6235
|
+
...materialized.timing,
|
|
6236
|
+
graphMs: Math.round(performance.now() - graphStarted),
|
|
6237
|
+
repositories: materialized.repositories.length,
|
|
6238
|
+
});
|
|
6098
6239
|
persistRows(database, identity, intent.resourceId, intent.workspaceId, rows);
|
|
6099
6240
|
input.adoptMaterialization(intent.workspaceId);
|
|
6100
6241
|
await onStage?.("rows-committed");
|
|
@@ -6184,26 +6325,6 @@ async function retryContent(operation, eventualNotFound = false, attempts = 10)
|
|
|
6184
6325
|
}
|
|
6185
6326
|
throw last;
|
|
6186
6327
|
}
|
|
6187
|
-
async function boundedForEach(items, concurrency, perform) {
|
|
6188
|
-
let next = 0;
|
|
6189
|
-
let stopped = false;
|
|
6190
|
-
const worker = async () => {
|
|
6191
|
-
while (!stopped) {
|
|
6192
|
-
const index = next;
|
|
6193
|
-
next += 1;
|
|
6194
|
-
if (index >= items.length)
|
|
6195
|
-
return;
|
|
6196
|
-
try {
|
|
6197
|
-
await perform(items[index]);
|
|
6198
|
-
}
|
|
6199
|
-
catch (error) {
|
|
6200
|
-
stopped = true;
|
|
6201
|
-
throw error;
|
|
6202
|
-
}
|
|
6203
|
-
}
|
|
6204
|
-
};
|
|
6205
|
-
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, () => worker()));
|
|
6206
|
-
}
|
|
6207
6328
|
/** Admit queued async tasks with at most `limit` in flight. Scheduling never
|
|
6208
6329
|
* blocks the caller; each task's settlement is its returned promise. The
|
|
6209
6330
|
* admitted task re-checks the bound because a released slot may be claimed
|