@amalgm/shell 0.1.131 → 0.1.133
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/transfer-instrumentation.d.ts +28 -0
- package/dist/transfer-instrumentation.js +68 -0
- package/dist/transfer-instrumentation.js.map +1 -0
- package/dist/user-ground-host.d.ts +3 -1
- package/dist/user-ground-host.js +85 -53
- package/dist/user-ground-host.js.map +1 -1
- package/dist/workspace-add-staging.d.ts +18 -1
- package/dist/workspace-add-staging.js +35 -2
- package/dist/workspace-add-staging.js.map +1 -1
- package/npm-shrinkwrap.json +6 -6
- package/package.json +2 -2
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-leg accounting for one content transfer.
|
|
3
|
+
*
|
|
4
|
+
* Every instrumented operation lands its wall time in exactly one named leg.
|
|
5
|
+
* Legs sum busy time across concurrent operations, so under a wide window
|
|
6
|
+
* their total may exceed the transfer's wall clock; `idleMs` is the opposite
|
|
7
|
+
* fact — wall time in which zero instrumented operations were in flight — so
|
|
8
|
+
* a transfer's ledger reads as legs (where work went) plus idle (where
|
|
9
|
+
* nothing went). Externally measured durations (a gateway acknowledgement's
|
|
10
|
+
* own timings) fold in without in-flight tracking: they decompose a timed
|
|
11
|
+
* leg, they do not extend it.
|
|
12
|
+
*/
|
|
13
|
+
export declare class TransferLegs {
|
|
14
|
+
private readonly totals;
|
|
15
|
+
private inFlight;
|
|
16
|
+
private quietSince;
|
|
17
|
+
private idleMs;
|
|
18
|
+
/** Fold an externally measured duration into a leg. Absent or non-positive
|
|
19
|
+
* values contribute nothing, so wire frames may omit their timings. */
|
|
20
|
+
add(leg: string, ms: number): void;
|
|
21
|
+
time<T>(leg: string, work: () => Promise<T>): Promise<T>;
|
|
22
|
+
timeSync<T>(leg: string, work: () => T): T;
|
|
23
|
+
/** Rounded per-leg totals plus `idleMs` through this moment, ready to
|
|
24
|
+
* spread into stage measurements. */
|
|
25
|
+
report(): Record<string, number>;
|
|
26
|
+
private enter;
|
|
27
|
+
private exit;
|
|
28
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-leg accounting for one content transfer.
|
|
3
|
+
*
|
|
4
|
+
* Every instrumented operation lands its wall time in exactly one named leg.
|
|
5
|
+
* Legs sum busy time across concurrent operations, so under a wide window
|
|
6
|
+
* their total may exceed the transfer's wall clock; `idleMs` is the opposite
|
|
7
|
+
* fact — wall time in which zero instrumented operations were in flight — so
|
|
8
|
+
* a transfer's ledger reads as legs (where work went) plus idle (where
|
|
9
|
+
* nothing went). Externally measured durations (a gateway acknowledgement's
|
|
10
|
+
* own timings) fold in without in-flight tracking: they decompose a timed
|
|
11
|
+
* leg, they do not extend it.
|
|
12
|
+
*/
|
|
13
|
+
export class TransferLegs {
|
|
14
|
+
totals = new Map();
|
|
15
|
+
inFlight = 0;
|
|
16
|
+
quietSince = performance.now();
|
|
17
|
+
idleMs = 0;
|
|
18
|
+
/** Fold an externally measured duration into a leg. Absent or non-positive
|
|
19
|
+
* values contribute nothing, so wire frames may omit their timings. */
|
|
20
|
+
add(leg, ms) {
|
|
21
|
+
if (!Number.isFinite(ms) || ms <= 0)
|
|
22
|
+
return;
|
|
23
|
+
this.totals.set(leg, (this.totals.get(leg) ?? 0) + ms);
|
|
24
|
+
}
|
|
25
|
+
async time(leg, work) {
|
|
26
|
+
const startedAt = this.enter();
|
|
27
|
+
try {
|
|
28
|
+
return await work();
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
this.exit(leg, startedAt);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
timeSync(leg, work) {
|
|
35
|
+
const startedAt = this.enter();
|
|
36
|
+
try {
|
|
37
|
+
return work();
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
this.exit(leg, startedAt);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/** Rounded per-leg totals plus `idleMs` through this moment, ready to
|
|
44
|
+
* spread into stage measurements. */
|
|
45
|
+
report() {
|
|
46
|
+
const now = performance.now();
|
|
47
|
+
const trailingIdle = this.inFlight === 0 ? now - this.quietSince : 0;
|
|
48
|
+
return {
|
|
49
|
+
...Object.fromEntries([...this.totals].map(([leg, ms]) => [leg, Math.round(ms)])),
|
|
50
|
+
idleMs: Math.round(this.idleMs + trailingIdle),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
enter() {
|
|
54
|
+
const now = performance.now();
|
|
55
|
+
if (this.inFlight === 0)
|
|
56
|
+
this.idleMs += now - this.quietSince;
|
|
57
|
+
this.inFlight += 1;
|
|
58
|
+
return now;
|
|
59
|
+
}
|
|
60
|
+
exit(leg, startedAt) {
|
|
61
|
+
const now = performance.now();
|
|
62
|
+
this.add(leg, now - startedAt);
|
|
63
|
+
this.inFlight -= 1;
|
|
64
|
+
if (this.inFlight === 0)
|
|
65
|
+
this.quietSince = now;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=transfer-instrumentation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transfer-instrumentation.js","sourceRoot":"","sources":["../src/transfer-instrumentation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,YAAY;IACN,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,QAAQ,GAAG,CAAC,CAAC;IACb,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAC/B,MAAM,GAAG,CAAC,CAAC;IAEnB;2EACuE;IACvE,GAAG,CAAC,GAAW,EAAE,EAAU;QACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;YAAE,OAAO;QAC5C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,IAAI,CAAI,GAAW,EAAE,IAAsB;QAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,EAAE,CAAC;QACtB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,QAAQ,CAAI,GAAW,EAAE,IAAa;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;QAChB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED;yCACqC;IACrC,MAAM;QACJ,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC9B,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QACrE,OAAO;YACL,GAAG,MAAM,CAAC,WAAW,CACnB,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAC3D;YACD,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC;SAC/C,CAAC;IACJ,CAAC;IAEO,KAAK;QACX,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;YAAE,IAAI,CAAC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9D,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnB,OAAO,GAAG,CAAC;IACb,CAAC;IAEO,IAAI,CAAC,GAAW,EAAE,SAAiB;QACzC,MAAM,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,SAAS,CAAC,CAAC;QAC/B,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnB,IAAI,IAAI,CAAC,QAAQ,KAAK,CAAC;YAAE,IAAI,CAAC,UAAU,GAAG,GAAG,CAAC;IACjD,CAAC;CACF"}
|
|
@@ -221,7 +221,9 @@ export declare class UserGroundHost {
|
|
|
221
221
|
/** One self-contained cargo segment on the wire: declared artifacts in
|
|
222
222
|
* header order, their bytes concatenated behind them, one verified round
|
|
223
223
|
* trip. Returns the authority's sealed hashes for the stream law to
|
|
224
|
-
* prove coverage.
|
|
224
|
+
* prove coverage. The acknowledgement carries the gateway's own leg
|
|
225
|
+
* timings — verification, storage, sealing — which fold into the ledger
|
|
226
|
+
* as a decomposition of this round trip's `wireMs`. */
|
|
225
227
|
private putCargo;
|
|
226
228
|
private uploadSnapshotContent;
|
|
227
229
|
/** Download a set of immutable artifacts straight from content storage.
|
package/dist/user-ground-host.js
CHANGED
|
@@ -15,6 +15,7 @@ import { ContentCacheDownloadSet, captureContentFile, hasSealedContent, hashCont
|
|
|
15
15
|
import { StagedContentSet, addStagingEvidenceDir, clearAddStagingEvidence, } from "./workspace-add-staging.js";
|
|
16
16
|
import { DirectContentPlane } from "./content-direct.js";
|
|
17
17
|
import { encodeContentWireBytes } from "./content-wire-codec.js";
|
|
18
|
+
import { TransferLegs } from "./transfer-instrumentation.js";
|
|
18
19
|
import { exactTextReplay, planGroundDetection, reconcileGroundUUIDs, } from "./detection/portable.js";
|
|
19
20
|
import { AcceptedInboxResultIndex } from "./detection/accepted-result.js";
|
|
20
21
|
import { sameNotebookBasisRow } from "./detection/notebook-basis.js";
|
|
@@ -1391,17 +1392,22 @@ export class UserGroundHost {
|
|
|
1391
1392
|
/** One self-contained cargo segment on the wire: declared artifacts in
|
|
1392
1393
|
* header order, their bytes concatenated behind them, one verified round
|
|
1393
1394
|
* trip. Returns the authority's sealed hashes for the stream law to
|
|
1394
|
-
* prove coverage.
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1395
|
+
* prove coverage. The acknowledgement carries the gateway's own leg
|
|
1396
|
+
* timings — verification, storage, sealing — which fold into the ledger
|
|
1397
|
+
* as a decomposition of this round trip's `wireMs`. */
|
|
1398
|
+
async putCargo(resourceId, artifacts, payload, legs) {
|
|
1399
|
+
const encoded = await legs.time("encodeMs", () => encodeContentWireBytes(payload));
|
|
1400
|
+
const frame = await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1398
1401
|
type: "private.entity-content.put-cargo",
|
|
1399
1402
|
resource_id: resourceId,
|
|
1400
1403
|
artifacts,
|
|
1401
1404
|
bytes: payload.byteLength,
|
|
1402
1405
|
wire_bytes: encoded.bytes.byteLength,
|
|
1403
1406
|
...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
|
|
1404
|
-
}, encoded.bytes, ["private.entity-content.stored-cargo"]));
|
|
1407
|
+
}, encoded.bytes, ["private.entity-content.stored-cargo"])));
|
|
1408
|
+
legs.add("gatewayVerifyMs", Number(frame.verify_ms));
|
|
1409
|
+
legs.add("gatewayStoreMs", Number(frame.store_ms));
|
|
1410
|
+
legs.add("gatewaySealMs", Number(frame.seal_ms));
|
|
1405
1411
|
return {
|
|
1406
1412
|
sealed: Array.isArray(frame.sealed) ? frame.sealed.map(String) : [],
|
|
1407
1413
|
wireBytes: encoded.bytes.byteLength,
|
|
@@ -1417,9 +1423,12 @@ export class UserGroundHost {
|
|
|
1417
1423
|
}
|
|
1418
1424
|
}
|
|
1419
1425
|
const started = performance.now();
|
|
1426
|
+
const legs = new TransferLegs();
|
|
1427
|
+
const timedSha256Hex = (input) => legs.timeSync("hashMs", () => sha256Hex(input));
|
|
1420
1428
|
let uploadedChunks = 0;
|
|
1421
1429
|
let reusedChunks = 0;
|
|
1422
1430
|
let uploadedBytes = 0;
|
|
1431
|
+
let loadedBytes = 0;
|
|
1423
1432
|
let inventoryRequests = 0;
|
|
1424
1433
|
const identifiers = {
|
|
1425
1434
|
...(trace?.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
@@ -1440,16 +1449,16 @@ export class UserGroundHost {
|
|
|
1440
1449
|
// upload — asked before any source is touched, so sealed artifacts cost
|
|
1441
1450
|
// zero local reads and a retried publication whose upload already sealed
|
|
1442
1451
|
// completely never needs its captured ground to still exist.
|
|
1443
|
-
const alreadySealed = await this.sealedContentHashes(resourceId, [...byHash.keys()]);
|
|
1452
|
+
const alreadySealed = await legs.time("planMs", () => this.sealedContentHashes(resourceId, [...byHash.keys()]));
|
|
1444
1453
|
const unsealed = [...byHash.values()]
|
|
1445
1454
|
.filter(({ artifact }) => !alreadySealed.has(artifact.contentHash));
|
|
1446
|
-
const rows = readGroundRowsByUuid(this.databasePath(identity), unsealed.map(({ uuid }) => uuid));
|
|
1455
|
+
const rows = legs.timeSync("planMs", () => readGroundRowsByUuid(this.databasePath(identity), unsealed.map(({ uuid }) => uuid)));
|
|
1447
1456
|
// Each artifact uploads from the strongest evidence available: a cache
|
|
1448
1457
|
// copy when another journey pinned one, otherwise the ground entity
|
|
1449
1458
|
// itself. Register stages nothing, so the ground path is the common
|
|
1450
1459
|
// case — and every read is re-proven against the captured hash, turning
|
|
1451
1460
|
// ground churn into a stale capture instead of a wrong upload.
|
|
1452
|
-
const sources = unsealed.map(({ artifact, uuid }) => {
|
|
1461
|
+
const sources = legs.timeSync("planMs", () => unsealed.map(({ artifact, uuid }) => {
|
|
1453
1462
|
const cached = contentCacheFile(cache, artifact.contentHash);
|
|
1454
1463
|
const fromCache = existsSync(cached);
|
|
1455
1464
|
const row = rows.get(uuid);
|
|
@@ -1489,7 +1498,7 @@ export class UserGroundHost {
|
|
|
1489
1498
|
...(declared && declared.bytes === bytes ? { manifest: declared } : {}),
|
|
1490
1499
|
bytes,
|
|
1491
1500
|
};
|
|
1492
|
-
});
|
|
1501
|
+
}));
|
|
1493
1502
|
const stale = (source, error) => {
|
|
1494
1503
|
if (source.fromCache) {
|
|
1495
1504
|
// A cache copy that no longer proves its hash is torn evidence:
|
|
@@ -1513,10 +1522,11 @@ export class UserGroundHost {
|
|
|
1513
1522
|
const loaded = new Array(segment.items.length);
|
|
1514
1523
|
await boundedForEach(segment.items.map((item, index) => ({ source: item.source, index })), CARGO_SEGMENT_LOAD_CONCURRENCY, async ({ source, index }) => {
|
|
1515
1524
|
try {
|
|
1516
|
-
const bytes = source.linkTarget ?? await readFile(source.file);
|
|
1525
|
+
const bytes = source.linkTarget ?? await legs.time("loadMs", () => readFile(source.file));
|
|
1526
|
+
loadedBytes += bytes.byteLength;
|
|
1517
1527
|
loaded[index] = {
|
|
1518
1528
|
bytes,
|
|
1519
|
-
manifest: deriveUploadManifest(source.artifact, bytes,
|
|
1529
|
+
manifest: deriveUploadManifest(source.artifact, bytes, timedSha256Hex),
|
|
1520
1530
|
};
|
|
1521
1531
|
}
|
|
1522
1532
|
catch (error) {
|
|
@@ -1529,7 +1539,7 @@ export class UserGroundHost {
|
|
|
1529
1539
|
content_hash: segment.items[index].contentHash,
|
|
1530
1540
|
bytes: item.manifest.bytes,
|
|
1531
1541
|
chunks: item.manifest.chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
|
|
1532
|
-
})), Buffer.concat(loaded.map((item) => item.bytes)));
|
|
1542
|
+
})), Buffer.concat(loaded.map((item) => item.bytes)), legs);
|
|
1533
1543
|
for (const item of loaded)
|
|
1534
1544
|
uploadedChunks += item.manifest.chunks.length;
|
|
1535
1545
|
uploadedBytes += wireBytes;
|
|
@@ -1544,7 +1554,7 @@ export class UserGroundHost {
|
|
|
1544
1554
|
const chunk = manifest.chunks[index];
|
|
1545
1555
|
if (!chunk)
|
|
1546
1556
|
throw new Error(`content manifest has no chunk ${index}`);
|
|
1547
|
-
await retryContent(() => this.wire.requestBinary({
|
|
1557
|
+
await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1548
1558
|
type: "private.entity-content.put",
|
|
1549
1559
|
resource_id: resourceId,
|
|
1550
1560
|
content_hash: artifact.contentHash,
|
|
@@ -1553,31 +1563,32 @@ export class UserGroundHost {
|
|
|
1553
1563
|
part_count: manifest.chunks.length,
|
|
1554
1564
|
sha256: chunk.sha256,
|
|
1555
1565
|
bytes: chunk.bytes,
|
|
1556
|
-
}, bytes, ["private.entity-content.stored"]));
|
|
1566
|
+
}, bytes, ["private.entity-content.stored"])));
|
|
1557
1567
|
uploadedBytes += bytes.byteLength;
|
|
1558
1568
|
};
|
|
1559
1569
|
try {
|
|
1560
1570
|
const receipt = await uploadArtifact(artifact, {
|
|
1561
1571
|
bytes: fileBytes,
|
|
1562
1572
|
...(knownManifest ? { manifest: knownManifest, manifestVerified: true } : {}),
|
|
1563
|
-
verify: async (contentHash) => await hashContentFile(file) === contentHash,
|
|
1573
|
+
verify: async (contentHash) => await legs.time("verifyMs", () => hashContentFile(file)) === contentHash,
|
|
1564
1574
|
read: async (offset, bytes) => {
|
|
1565
1575
|
sourceFile.handle ??= open(file, "r");
|
|
1566
1576
|
const handle = await sourceFile.handle;
|
|
1567
1577
|
const buffer = Buffer.allocUnsafe(bytes);
|
|
1568
|
-
const { bytesRead } = await handle.read(buffer, 0, bytes, offset);
|
|
1578
|
+
const { bytesRead } = await legs.time("loadMs", () => handle.read(buffer, 0, bytes, offset));
|
|
1579
|
+
loadedBytes += bytesRead;
|
|
1569
1580
|
return buffer.subarray(0, bytesRead);
|
|
1570
1581
|
},
|
|
1571
1582
|
}, {
|
|
1572
|
-
sha256Hex,
|
|
1583
|
+
sha256Hex: timedSha256Hex,
|
|
1573
1584
|
missingChunks: async (_candidate, manifest) => {
|
|
1574
1585
|
inventoryRequests += 1;
|
|
1575
|
-
const frame = await retryContent(() => this.wire.request({
|
|
1586
|
+
const frame = await legs.time("inventoryMs", () => retryContent(() => this.wire.request({
|
|
1576
1587
|
type: "private.entity-content.inventory",
|
|
1577
1588
|
resource_id: resourceId,
|
|
1578
1589
|
content_hash: artifact.contentHash,
|
|
1579
1590
|
chunks: manifest.chunks,
|
|
1580
|
-
}, ["private.entity-content.inventory-result"]));
|
|
1591
|
+
}, ["private.entity-content.inventory-result"])));
|
|
1581
1592
|
manifestPresent = frame.complete === true;
|
|
1582
1593
|
if (!Array.isArray(frame.missing)) {
|
|
1583
1594
|
throw new Error("content authority returned an invalid inventory");
|
|
@@ -1586,8 +1597,8 @@ export class UserGroundHost {
|
|
|
1586
1597
|
},
|
|
1587
1598
|
putChunk: async (_candidate, manifest, index, bytes) => putChunk(manifest, index, bytes),
|
|
1588
1599
|
putChunkBatch: async (_candidate, manifest, indexes, bytes) => {
|
|
1589
|
-
const encoded = await encodeContentWireBytes(bytes);
|
|
1590
|
-
await retryContent(() => this.wire.requestBinary({
|
|
1600
|
+
const encoded = await legs.time("encodeMs", () => encodeContentWireBytes(bytes));
|
|
1601
|
+
await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1591
1602
|
type: "private.entity-content.put-batch",
|
|
1592
1603
|
resource_id: resourceId,
|
|
1593
1604
|
content_hash: artifact.contentHash,
|
|
@@ -1600,14 +1611,14 @@ export class UserGroundHost {
|
|
|
1600
1611
|
bytes: bytes.byteLength,
|
|
1601
1612
|
wire_bytes: encoded.bytes.byteLength,
|
|
1602
1613
|
...(encoded.encoding ? { content_encoding: encoded.encoding } : {}),
|
|
1603
|
-
}, encoded.bytes, ["private.entity-content.stored-batch"]));
|
|
1614
|
+
}, encoded.bytes, ["private.entity-content.stored-batch"])));
|
|
1604
1615
|
uploadedBytes += encoded.bytes.byteLength;
|
|
1605
1616
|
},
|
|
1606
1617
|
putManifest: async (_candidate, manifest) => {
|
|
1607
1618
|
if (manifestPresent)
|
|
1608
1619
|
return;
|
|
1609
1620
|
const bytes = Buffer.from(stableJson(manifest), "utf8");
|
|
1610
|
-
await retryContent(() => this.wire.requestBinary({
|
|
1621
|
+
await legs.time("wireMs", () => retryContent(() => this.wire.requestBinary({
|
|
1611
1622
|
type: "private.entity-content.put",
|
|
1612
1623
|
resource_id: resourceId,
|
|
1613
1624
|
content_hash: artifact.contentHash,
|
|
@@ -1616,7 +1627,7 @@ export class UserGroundHost {
|
|
|
1616
1627
|
part_count: manifest.chunks.length,
|
|
1617
1628
|
sha256: sha256Hex(bytes),
|
|
1618
1629
|
bytes: bytes.length,
|
|
1619
|
-
}, bytes, ["private.entity-content.stored"]));
|
|
1630
|
+
}, bytes, ["private.entity-content.stored"])));
|
|
1620
1631
|
uploadedBytes += bytes.byteLength;
|
|
1621
1632
|
},
|
|
1622
1633
|
});
|
|
@@ -1653,11 +1664,27 @@ export class UserGroundHost {
|
|
|
1653
1664
|
artifacts: byHash.size,
|
|
1654
1665
|
reusedArtifacts: byHash.size - sources.length,
|
|
1655
1666
|
declaredBytes: sources.reduce((total, source) => total + source.bytes, 0),
|
|
1667
|
+
// The same declared source bytes split by rail: whole small
|
|
1668
|
+
// artifacts ride shared cargo segments, larger ones ride
|
|
1669
|
+
// per-artifact chunk transfer. Any change to either rail's shape
|
|
1670
|
+
// is sized from a production trace by these four numbers.
|
|
1671
|
+
cargoArtifacts: small.length,
|
|
1672
|
+
cargoBytes: small.reduce((total, source) => total + source.bytes, 0),
|
|
1673
|
+
largeArtifacts: large.length,
|
|
1674
|
+
largeBytes: large.reduce((total, source) => total + source.bytes, 0),
|
|
1656
1675
|
uploadedBytes,
|
|
1676
|
+
loadedBytes,
|
|
1657
1677
|
uploadedChunks,
|
|
1658
1678
|
reusedChunks,
|
|
1659
1679
|
inventoryRequests,
|
|
1660
1680
|
cargoSegments,
|
|
1681
|
+
// Where this upload's time went, leg by leg: plan (seal inventory
|
|
1682
|
+
// + ground rows + source stats), load (source reads), hash, verify
|
|
1683
|
+
// (fused re-read + hash of a hinted manifest's source), encode
|
|
1684
|
+
// (compression), wire (data round trips, whose gateway share the
|
|
1685
|
+
// acknowledgement timings decompose), inventory round trips, and
|
|
1686
|
+
// idle — wall time with nothing in flight.
|
|
1687
|
+
...legs.report(),
|
|
1661
1688
|
},
|
|
1662
1689
|
});
|
|
1663
1690
|
}
|
|
@@ -1668,7 +1695,7 @@ export class UserGroundHost {
|
|
|
1668
1695
|
status: "failed",
|
|
1669
1696
|
...identifiers,
|
|
1670
1697
|
durationMs: performance.now() - started,
|
|
1671
|
-
measurements: { journey: trace?.journey ?? "bootstrap" },
|
|
1698
|
+
measurements: { journey: trace?.journey ?? "bootstrap", ...legs.report() },
|
|
1672
1699
|
});
|
|
1673
1700
|
throw error;
|
|
1674
1701
|
}
|
|
@@ -1683,12 +1710,13 @@ export class UserGroundHost {
|
|
|
1683
1710
|
const sole = unique.size === 1 ? [...unique.values()][0]?.artifact : undefined;
|
|
1684
1711
|
const started = performance.now();
|
|
1685
1712
|
const planeBefore = this.direct.metrics();
|
|
1713
|
+
const legs = new TransferLegs();
|
|
1714
|
+
const timedSha256Hex = (input) => legs.timeSync("hashMs", () => sha256Hex(input));
|
|
1686
1715
|
let declaredBytes = 0;
|
|
1687
1716
|
let downloadedBytes = 0;
|
|
1688
1717
|
let downloadedChunks = 0;
|
|
1689
1718
|
let reusedChunks = 0;
|
|
1690
1719
|
let requests = 0;
|
|
1691
|
-
let cacheWriteMs = 0;
|
|
1692
1720
|
let stagedArtifacts = 0;
|
|
1693
1721
|
const entityId = trace?.entityId ?? sole?.entityId;
|
|
1694
1722
|
const identifiers = {
|
|
@@ -1724,19 +1752,19 @@ export class UserGroundHost {
|
|
|
1724
1752
|
// for a fresh ref every attempt.
|
|
1725
1753
|
const fetchObject = async (object) => {
|
|
1726
1754
|
requests += 1;
|
|
1727
|
-
const bytes = await retryContent(() => this.direct.fetchObject(resourceId, object), true, retryAttempts);
|
|
1755
|
+
const bytes = await legs.time("fetchMs", () => retryContent(() => this.direct.fetchObject(resourceId, object), true, retryAttempts));
|
|
1728
1756
|
downloadedBytes += bytes.byteLength;
|
|
1729
1757
|
return bytes;
|
|
1730
1758
|
};
|
|
1731
1759
|
try {
|
|
1732
1760
|
const receipts = await downloadAllArtifacts([...unique.values()].map((want) => want.artifact), {
|
|
1733
|
-
sha256Hex,
|
|
1761
|
+
sha256Hex: timedSha256Hex,
|
|
1734
1762
|
getManifest: async (artifact) => {
|
|
1735
1763
|
// The record's plan or this machine's own manifest cache already
|
|
1736
1764
|
// proves chunk identity; downloaded bytes are still hash-verified,
|
|
1737
1765
|
// so a stale local manifest can only fail loudly, never corrupt.
|
|
1738
|
-
const local = unique.get(artifact.contentHash)?.manifest
|
|
1739
|
-
?? readCachedManifest(cacheDir, artifact.contentHash);
|
|
1766
|
+
const local = legs.timeSync("planMs", () => unique.get(artifact.contentHash)?.manifest
|
|
1767
|
+
?? readCachedManifest(cacheDir, artifact.contentHash));
|
|
1740
1768
|
const manifest = local ?? await (async () => {
|
|
1741
1769
|
const bytes = await fetchObject({ kind: "manifest", sha256: artifact.contentHash });
|
|
1742
1770
|
const checked = checkContentManifest(JSON.parse(bytes.toString("utf8")), artifact.contentHash);
|
|
@@ -1748,32 +1776,28 @@ export class UserGroundHost {
|
|
|
1748
1776
|
// destination — except content this machine already holds sealed,
|
|
1749
1777
|
// which stays on the cache rail so a warm re-Add remains a local
|
|
1750
1778
|
// clone instead of a fresh download.
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1779
|
+
legs.timeSync("planMs", () => {
|
|
1780
|
+
const stagedPaths = stagedSet && staged?.destinations.get(artifact.contentHash);
|
|
1781
|
+
if (stagedPaths && !hasSealedContent(cacheDir, artifact.contentHash, manifest.bytes)) {
|
|
1782
|
+
stagedArtifacts += 1;
|
|
1783
|
+
targets.set(artifact.contentHash, stagedSet.target(artifact, manifest, stagedPaths));
|
|
1784
|
+
}
|
|
1785
|
+
else {
|
|
1786
|
+
targets.set(artifact.contentHash, downloads.target(artifact, manifest));
|
|
1787
|
+
}
|
|
1788
|
+
});
|
|
1759
1789
|
return manifest;
|
|
1760
1790
|
},
|
|
1761
|
-
hasChunk: async (artifact, _manifest, index) => target(artifact).hasChunk(index),
|
|
1791
|
+
hasChunk: async (artifact, _manifest, index) => legs.time("reuseMs", () => target(artifact).hasChunk(index)),
|
|
1762
1792
|
getChunk: async (_artifact, manifest, index) => {
|
|
1763
1793
|
const chunk = manifest.chunks[index];
|
|
1764
1794
|
if (!chunk)
|
|
1765
1795
|
throw new Error(`content manifest has no chunk ${index}`);
|
|
1766
1796
|
return fetchObject({ kind: "block", sha256: chunk.sha256 });
|
|
1767
1797
|
},
|
|
1768
|
-
putChunk: async (artifact, _manifest, index, bytes) =>
|
|
1769
|
-
const writeStarted = performance.now();
|
|
1770
|
-
await target(artifact).putChunk(index, bytes);
|
|
1771
|
-
cacheWriteMs += performance.now() - writeStarted;
|
|
1772
|
-
},
|
|
1798
|
+
putChunk: async (artifact, _manifest, index, bytes) => legs.time("writeMs", () => target(artifact).putChunk(index, bytes)),
|
|
1773
1799
|
sealArtifact: async (artifact) => {
|
|
1774
|
-
const
|
|
1775
|
-
const sealed = await target(artifact).seal();
|
|
1776
|
-
cacheWriteMs += performance.now() - sealStarted;
|
|
1800
|
+
const sealed = await legs.time("sealMs", () => target(artifact).seal());
|
|
1777
1801
|
return { local: sealed, bytes: sealed.bytes, contentHash: sealed.contentHash };
|
|
1778
1802
|
},
|
|
1779
1803
|
}, {
|
|
@@ -1803,12 +1827,17 @@ export class UserGroundHost {
|
|
|
1803
1827
|
reusedChunks,
|
|
1804
1828
|
// `requests` is logical object reads; the plane deltas below are
|
|
1805
1829
|
// the physical facts — actual storage GETs, sealed bytes on the
|
|
1806
|
-
// wire, refs round-trips, decrypt
|
|
1807
|
-
//
|
|
1808
|
-
//
|
|
1830
|
+
// wire, refs round-trips, decrypt CPU. The legs place local time:
|
|
1831
|
+
// plan (manifest cache + routing), fetch (storage waits, whose
|
|
1832
|
+
// decrypt share the plane delta names), hash (chunk verification),
|
|
1833
|
+
// reuse (retained-chunk re-proof), write (chunk placement), seal
|
|
1834
|
+
// (whole-file proof + durability), and idle — wall time with
|
|
1835
|
+
// nothing in flight. The staged rail decomposes its own write and
|
|
1836
|
+
// seal internals below; `stagedArtifacts` counts its placements.
|
|
1809
1837
|
requests,
|
|
1810
1838
|
...physicalPlaneDelta(planeBefore, this.direct.metrics()),
|
|
1811
|
-
|
|
1839
|
+
...legs.report(),
|
|
1840
|
+
...(stagedSet ? stagedSet.report() : {}),
|
|
1812
1841
|
...(stagedArtifacts > 0 ? { stagedArtifacts } : {}),
|
|
1813
1842
|
},
|
|
1814
1843
|
});
|
|
@@ -1826,6 +1855,7 @@ export class UserGroundHost {
|
|
|
1826
1855
|
artifacts: unique.size,
|
|
1827
1856
|
...(sole ? { kind: sole.kind } : {}),
|
|
1828
1857
|
requests,
|
|
1858
|
+
...legs.report(),
|
|
1829
1859
|
},
|
|
1830
1860
|
});
|
|
1831
1861
|
throw error;
|
|
@@ -2425,7 +2455,8 @@ export class UserGroundHost {
|
|
|
2425
2455
|
if (pages.length === 0)
|
|
2426
2456
|
return;
|
|
2427
2457
|
const started = performance.now();
|
|
2428
|
-
const
|
|
2458
|
+
const legs = new TransferLegs();
|
|
2459
|
+
const sealed = await legs.time("planMs", () => this.sealedContentHashes(resourceId, pages.map((page) => page.sha256)));
|
|
2429
2460
|
const unsealed = pages
|
|
2430
2461
|
.filter((page) => !sealed.has(page.sha256))
|
|
2431
2462
|
.map((page) => ({ contentHash: page.sha256, bytes: page.bytes.byteLength, page }));
|
|
@@ -2439,7 +2470,7 @@ export class UserGroundHost {
|
|
|
2439
2470
|
content_hash: item.contentHash,
|
|
2440
2471
|
bytes: item.bytes,
|
|
2441
2472
|
chunks: [{ sha256: item.contentHash, bytes: item.bytes }],
|
|
2442
|
-
})), Buffer.concat(segment.items.map((item) => item.page.bytes)));
|
|
2473
|
+
})), Buffer.concat(segment.items.map((item) => item.page.bytes)), legs);
|
|
2443
2474
|
uploadedBytes += wireBytes;
|
|
2444
2475
|
return stored;
|
|
2445
2476
|
},
|
|
@@ -2455,6 +2486,7 @@ export class UserGroundHost {
|
|
|
2455
2486
|
pages: pages.length,
|
|
2456
2487
|
uploadedPages: unsealed.length,
|
|
2457
2488
|
uploadedBytes,
|
|
2489
|
+
...legs.report(),
|
|
2458
2490
|
},
|
|
2459
2491
|
});
|
|
2460
2492
|
}
|