@amalgm/live 0.2.47 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/transfer/download.d.ts +17 -6
- package/dist/transfer/download.d.ts.map +1 -1
- package/dist/transfer/download.js +70 -34
- package/dist/transfer/download.js.map +1 -1
- package/dist/transfer/parallel.d.ts +4 -0
- package/dist/transfer/parallel.d.ts.map +1 -1
- package/dist/transfer/parallel.js +62 -0
- package/dist/transfer/parallel.js.map +1 -1
- package/dist-cjs/transfer/download.js +69 -32
- package/dist-cjs/transfer/download.js.map +1 -1
- package/dist-cjs/transfer/parallel.js +63 -0
- package/dist-cjs/transfer/parallel.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Download
|
|
2
|
+
* Download a set of immutable artifacts into a machine cache.
|
|
3
3
|
*
|
|
4
|
-
* The
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* The set is the primitive: one plan wave proves every manifest and asks the
|
|
5
|
+
* cache which chunks are gaps, one chunk wave fetches every gap across the
|
|
6
|
+
* whole set under a single bound, and one seal wave verifies each complete
|
|
7
|
+
* artifact. A lone artifact is a set of one — there is no second path. This
|
|
8
|
+
* shape removes per-artifact serialization: no artifact waits behind
|
|
9
|
+
* another's plan or seal, only behind the shared fetch bound. Each chunk is
|
|
10
|
+
* still its own storage request — the set changes scheduling, not the
|
|
11
|
+
* request count.
|
|
8
12
|
*
|
|
9
13
|
* Every chunk is one content-addressed storage object, so the host's ports
|
|
10
14
|
* fetch chunks independently — parallelism, transport, and decryption are the
|
|
@@ -30,6 +34,9 @@ export interface SealedArtifact<LocalArtifact> {
|
|
|
30
34
|
}
|
|
31
35
|
export interface DownloadOptions {
|
|
32
36
|
readonly concurrency?: number;
|
|
37
|
+
/** Maximum plaintext chunk bytes held by in-flight fetch effects. Raising
|
|
38
|
+
* request concurrency never silently raises this memory bound. */
|
|
39
|
+
readonly maxInFlightBytes?: number;
|
|
33
40
|
}
|
|
34
41
|
export interface DownloadReceipt<LocalArtifact> {
|
|
35
42
|
readonly artifact: TransferArtifact;
|
|
@@ -38,6 +45,10 @@ export interface DownloadReceipt<LocalArtifact> {
|
|
|
38
45
|
readonly downloadedChunks: number;
|
|
39
46
|
readonly reusedChunks: number;
|
|
40
47
|
}
|
|
41
|
-
/** Download
|
|
48
|
+
/** Download a set of immutable artifacts, fetching only machine-cache gaps.
|
|
49
|
+
* Receipts return in input order; any failure fails the whole set after
|
|
50
|
+
* letting in-flight effects settle. */
|
|
51
|
+
export declare function downloadAll<LocalArtifact>(artifacts: readonly TransferArtifact[], ports: DownloadPorts<LocalArtifact>, options?: DownloadOptions): Promise<readonly DownloadReceipt<LocalArtifact>[]>;
|
|
52
|
+
/** Download one immutable artifact — a set of one. */
|
|
42
53
|
export declare function download<LocalArtifact>(artifact: TransferArtifact, ports: DownloadPorts<LocalArtifact>, options?: DownloadOptions): Promise<DownloadReceipt<LocalArtifact>>;
|
|
43
54
|
//# sourceMappingURL=download.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAqC,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAClG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAA0B,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAO/E,MAAM,WAAW,aAAa,CAAC,aAAa;IAC1C,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,WAAW,CAAC,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,6EAA6E;IAC7E,QAAQ,CACN,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,QAAQ,CACN,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,CAAC,CAAC;IACvB,QAAQ,CACN,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,8EAA8E;IAC9E,YAAY,CACV,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,cAAc,CAAC,aAAa;IAC3C,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B;sEACkE;IAClE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,eAAe,CAAC,aAAa;IAC5C,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAQD;;uCAEuC;AACvC,wBAAsB,WAAW,CAAC,aAAa,EAC7C,SAAS,EAAE,SAAS,gBAAgB,EAAE,EACtC,KAAK,EAAE,aAAa,CAAC,aAAa,CAAC,EACnC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,SAAS,eAAe,CAAC,aAAa,CAAC,EAAE,CAAC,CA+DpD;AAED,sDAAsD;AACtD,wBAAsB,QAAQ,CAAC,aAAa,EAC1C,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,aAAa,CAAC,aAAa,CAAC,EACnC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAIzC"}
|
|
@@ -1,52 +1,88 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Download
|
|
2
|
+
* Download a set of immutable artifacts into a machine cache.
|
|
3
3
|
*
|
|
4
|
-
* The
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* The set is the primitive: one plan wave proves every manifest and asks the
|
|
5
|
+
* cache which chunks are gaps, one chunk wave fetches every gap across the
|
|
6
|
+
* whole set under a single bound, and one seal wave verifies each complete
|
|
7
|
+
* artifact. A lone artifact is a set of one — there is no second path. This
|
|
8
|
+
* shape removes per-artifact serialization: no artifact waits behind
|
|
9
|
+
* another's plan or seal, only behind the shared fetch bound. Each chunk is
|
|
10
|
+
* still its own storage request — the set changes scheduling, not the
|
|
11
|
+
* request count.
|
|
8
12
|
*
|
|
9
13
|
* Every chunk is one content-addressed storage object, so the host's ports
|
|
10
14
|
* fetch chunks independently — parallelism, transport, and decryption are the
|
|
11
15
|
* host's concern; the verification laws here are not.
|
|
12
16
|
*/
|
|
13
|
-
import { checkContentManifest } from '../contracts/content.js';
|
|
17
|
+
import { CHUNK_BYTES, checkContentManifest } from '../contracts/content.js';
|
|
14
18
|
import { assertTransferArtifact } from './artifacts.js';
|
|
15
|
-
import { DEFAULT_TRANSFER_CONCURRENCY, parallelForEach } from './parallel.js';
|
|
16
|
-
/** Download
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
19
|
+
import { DEFAULT_TRANSFER_CONCURRENCY, parallelForEach, weightedParallelForEach, } from './parallel.js';
|
|
20
|
+
/** Download a set of immutable artifacts, fetching only machine-cache gaps.
|
|
21
|
+
* Receipts return in input order; any failure fails the whole set after
|
|
22
|
+
* letting in-flight effects settle. */
|
|
23
|
+
export async function downloadAll(artifacts, ports, options = {}) {
|
|
24
|
+
const concurrency = options.concurrency ?? DEFAULT_TRANSFER_CONCURRENCY;
|
|
25
|
+
const maxInFlightBytes = options.maxInFlightBytes
|
|
26
|
+
?? DEFAULT_TRANSFER_CONCURRENCY * CHUNK_BYTES;
|
|
27
|
+
for (const artifact of artifacts)
|
|
28
|
+
assertTransferArtifact(artifact);
|
|
29
|
+
const plans = new Array(artifacts.length);
|
|
30
|
+
await parallelForEach([...artifacts.keys()], concurrency, async (slot) => {
|
|
31
|
+
const artifact = artifacts[slot];
|
|
32
|
+
if (artifact === undefined)
|
|
33
|
+
return;
|
|
34
|
+
const checked = checkContentManifest(await ports.getManifest(artifact), artifact.contentHash);
|
|
35
|
+
if (!checked.ok)
|
|
36
|
+
throw new Error(checked.error);
|
|
37
|
+
const manifest = checked.value;
|
|
38
|
+
const missing = [];
|
|
39
|
+
for (const index of manifest.chunks.keys()) {
|
|
40
|
+
if (!await ports.hasChunk(artifact, manifest, index))
|
|
41
|
+
missing.push(index);
|
|
42
|
+
}
|
|
43
|
+
plans[slot] = { artifact, manifest, missing };
|
|
44
|
+
});
|
|
45
|
+
const gaps = plans.flatMap((plan) => plan === undefined ? [] : plan.missing.map((index) => ({ plan, index })));
|
|
46
|
+
await weightedParallelForEach(gaps, concurrency, maxInFlightBytes, ({ plan, index }) => plan.manifest.chunks[index]?.bytes ?? 0, async ({ plan, index }) => {
|
|
47
|
+
const expected = plan.manifest.chunks[index];
|
|
30
48
|
if (expected === undefined)
|
|
31
49
|
throw new Error('download manifest named an unknown chunk index');
|
|
32
|
-
const bytes = await ports.getChunk(artifact, manifest, index);
|
|
50
|
+
const bytes = await ports.getChunk(plan.artifact, plan.manifest, index);
|
|
33
51
|
if (!(bytes instanceof Uint8Array)
|
|
34
52
|
|| bytes.byteLength !== expected.bytes
|
|
35
53
|
|| ports.sha256Hex(bytes) !== expected.sha256) {
|
|
36
54
|
throw new Error(`downloaded content chunk ${index} failed verification`);
|
|
37
55
|
}
|
|
38
|
-
await ports.putChunk(artifact, manifest, index, bytes);
|
|
56
|
+
await ports.putChunk(plan.artifact, plan.manifest, index, bytes);
|
|
57
|
+
});
|
|
58
|
+
const receipts = new Array(artifacts.length);
|
|
59
|
+
await parallelForEach([...plans.keys()], concurrency, async (slot) => {
|
|
60
|
+
const plan = plans[slot];
|
|
61
|
+
if (plan === undefined)
|
|
62
|
+
return;
|
|
63
|
+
const sealed = await ports.sealArtifact(plan.artifact, plan.manifest);
|
|
64
|
+
if (sealed.bytes !== plan.manifest.bytes || sealed.contentHash !== plan.manifest.contentHash) {
|
|
65
|
+
throw new Error(`downloaded artifact ${plan.artifact.contentHash} failed verification`);
|
|
66
|
+
}
|
|
67
|
+
receipts[slot] = {
|
|
68
|
+
artifact: plan.artifact,
|
|
69
|
+
manifest: plan.manifest,
|
|
70
|
+
local: sealed.local,
|
|
71
|
+
downloadedChunks: plan.missing.length,
|
|
72
|
+
reusedChunks: plan.manifest.chunks.length - plan.missing.length,
|
|
73
|
+
};
|
|
74
|
+
});
|
|
75
|
+
return receipts.map((receipt) => {
|
|
76
|
+
if (receipt === undefined)
|
|
77
|
+
throw new Error('download set lost an artifact receipt');
|
|
78
|
+
return receipt;
|
|
39
79
|
});
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
artifact
|
|
46
|
-
|
|
47
|
-
local: sealed.local,
|
|
48
|
-
downloadedChunks: missing.length,
|
|
49
|
-
reusedChunks: manifest.chunks.length - missing.length,
|
|
50
|
-
};
|
|
80
|
+
}
|
|
81
|
+
/** Download one immutable artifact — a set of one. */
|
|
82
|
+
export async function download(artifact, ports, options = {}) {
|
|
83
|
+
const [receipt] = await downloadAll([artifact], ports, options);
|
|
84
|
+
if (receipt === undefined)
|
|
85
|
+
throw new Error('download set lost an artifact receipt');
|
|
86
|
+
return receipt;
|
|
51
87
|
}
|
|
52
88
|
//# sourceMappingURL=download.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAwB,MAAM,yBAAyB,CAAC;AAElG,OAAO,EAAE,sBAAsB,EAAyB,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EACL,4BAA4B,EAC5B,eAAe,EACf,uBAAuB,GACxB,MAAM,eAAe,CAAC;AAwDvB;;uCAEuC;AACvC,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,SAAsC,EACtC,KAAmC,EACnC,UAA2B,EAAE;IAE7B,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,4BAA4B,CAAC;IACxE,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;WAC5C,4BAA4B,GAAG,WAAW,CAAC;IAChD,KAAK,MAAM,QAAQ,IAAI,SAAS;QAAE,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAEnE,MAAM,KAAK,GAAiC,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACxE,MAAM,eAAe,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QACvE,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO;QACnC,MAAM,OAAO,GAAG,oBAAoB,CAClC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,EACjC,QAAQ,CAAC,WAAW,CACrB,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5E,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAClC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,uBAAuB,CAC3B,IAAI,EACJ,WAAW,EACX,gBAAgB,EAChB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAC5D,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxE,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;eAC7B,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,KAAK;eACnC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,sBAAsB,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACnE,CAAC,CACF,CAAC;IAEF,MAAM,QAAQ,GAAmD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7F,MAAM,eAAe,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QACnE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO;QAC/B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtE,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,WAAW,KAAK,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,QAAQ,CAAC,WAAW,sBAAsB,CAAC,CAAC;QAC1F,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YACrC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;SAChE,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,IAAI,OAAO,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACpF,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AACtD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAA0B,EAC1B,KAAmC,EACnC,UAA2B,EAAE;IAE7B,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAChE,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACpF,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -4,4 +4,8 @@
|
|
|
4
4
|
export declare const DEFAULT_TRANSFER_CONCURRENCY = 8;
|
|
5
5
|
/** Run a bounded number of independent effects without changing item order. */
|
|
6
6
|
export declare function parallelForEach<T>(items: readonly T[], concurrency: number, effect: (item: T) => Promise<void>): Promise<void>;
|
|
7
|
+
/** Run independent effects under both a count bound and a total weight
|
|
8
|
+
* bound. Input order is the admission order; a failure stops new admission
|
|
9
|
+
* and waits for already-started effects to settle. */
|
|
10
|
+
export declare function weightedParallelForEach<T>(items: readonly T[], concurrency: number, maxWeight: number, weightOf: (item: T) => number, effect: (item: T) => Promise<void>): Promise<void>;
|
|
7
11
|
//# sourceMappingURL=parallel.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parallel.d.ts","sourceRoot":"","sources":["../../src/transfer/parallel.ts"],"names":[],"mappings":"AAAA;;uEAEuE;AACvE,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAE9C,+EAA+E;AAC/E,wBAAsB,eAAe,CAAC,CAAC,EACrC,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC,CAuBf"}
|
|
1
|
+
{"version":3,"file":"parallel.d.ts","sourceRoot":"","sources":["../../src/transfer/parallel.ts"],"names":[],"mappings":"AAAA;;uEAEuE;AACvE,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAE9C,+EAA+E;AAC/E,wBAAsB,eAAe,CAAC,CAAC,EACrC,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC,CAuBf;AAED;;sDAEsD;AACtD,wBAAsB,uBAAuB,CAAC,CAAC,EAC7C,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC7B,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GACjC,OAAO,CAAC,IAAI,CAAC,CAyDf"}
|
|
@@ -28,4 +28,66 @@ export async function parallelForEach(items, concurrency, effect) {
|
|
|
28
28
|
if (failure !== undefined)
|
|
29
29
|
throw failure;
|
|
30
30
|
}
|
|
31
|
+
/** Run independent effects under both a count bound and a total weight
|
|
32
|
+
* bound. Input order is the admission order; a failure stops new admission
|
|
33
|
+
* and waits for already-started effects to settle. */
|
|
34
|
+
export async function weightedParallelForEach(items, concurrency, maxWeight, weightOf, effect) {
|
|
35
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1) {
|
|
36
|
+
throw new Error('transfer concurrency must be a positive integer');
|
|
37
|
+
}
|
|
38
|
+
if (!Number.isSafeInteger(maxWeight) || maxWeight < 1) {
|
|
39
|
+
throw new Error('transfer weight bound must be a positive integer');
|
|
40
|
+
}
|
|
41
|
+
const weights = items.map((item) => {
|
|
42
|
+
const weight = weightOf(item);
|
|
43
|
+
if (!Number.isSafeInteger(weight) || weight < 1 || weight > maxWeight) {
|
|
44
|
+
throw new Error('transfer item weight exceeds its positive bound');
|
|
45
|
+
}
|
|
46
|
+
return weight;
|
|
47
|
+
});
|
|
48
|
+
await new Promise((resolve, reject) => {
|
|
49
|
+
let cursor = 0;
|
|
50
|
+
let active = 0;
|
|
51
|
+
let activeWeight = 0;
|
|
52
|
+
let failure;
|
|
53
|
+
const advance = () => {
|
|
54
|
+
if (failure !== undefined) {
|
|
55
|
+
if (active === 0)
|
|
56
|
+
reject(failure);
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
while (cursor < items.length && active < concurrency) {
|
|
60
|
+
const item = items[cursor];
|
|
61
|
+
const weight = weights[cursor];
|
|
62
|
+
if (item === undefined || weight === undefined) {
|
|
63
|
+
failure = new Error('transfer scheduler lost an item');
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
if (active > 0 && activeWeight + weight > maxWeight)
|
|
67
|
+
break;
|
|
68
|
+
cursor += 1;
|
|
69
|
+
active += 1;
|
|
70
|
+
activeWeight += weight;
|
|
71
|
+
void effect(item).then(() => {
|
|
72
|
+
active -= 1;
|
|
73
|
+
activeWeight -= weight;
|
|
74
|
+
advance();
|
|
75
|
+
}, (error) => {
|
|
76
|
+
failure ??= error;
|
|
77
|
+
active -= 1;
|
|
78
|
+
activeWeight -= weight;
|
|
79
|
+
advance();
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
if (failure !== undefined) {
|
|
83
|
+
if (active === 0)
|
|
84
|
+
reject(failure);
|
|
85
|
+
}
|
|
86
|
+
else if (cursor === items.length && active === 0) {
|
|
87
|
+
resolve();
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
advance();
|
|
91
|
+
});
|
|
92
|
+
}
|
|
31
93
|
//# sourceMappingURL=parallel.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parallel.js","sourceRoot":"","sources":["../../src/transfer/parallel.ts"],"names":[],"mappings":"AAAA;;uEAEuE;AACvE,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAE9C,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAmB,EACnB,WAAmB,EACnB,MAAkC;IAElC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAgB,CAAC;IACrB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,OAAO,KAAK,SAAS,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,KAAK,KAAK,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CACpE,CAAC;IACF,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC;AAC3C,CAAC"}
|
|
1
|
+
{"version":3,"file":"parallel.js","sourceRoot":"","sources":["../../src/transfer/parallel.ts"],"names":[],"mappings":"AAAA;;uEAEuE;AACvE,MAAM,CAAC,MAAM,4BAA4B,GAAG,CAAC,CAAC;AAE9C,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAAmB,EACnB,WAAmB,EACnB,MAAkC;IAElC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAgB,CAAC;IACrB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,OAAO,KAAK,SAAS,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,KAAK,KAAK,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CACpE,CAAC;IACF,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC;AAC3C,CAAC;AAED;;sDAEsD;AACtD,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,KAAmB,EACnB,WAAmB,EACnB,SAAiB,EACjB,QAA6B,EAC7B,MAAkC;IAElC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,OAAgB,CAAC;QACrB,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,IAAI,MAAM,KAAK,CAAC;oBAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YACD,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,GAAG,WAAW,EAAE,CAAC;gBACrD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/B,IAAI,IAAI,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBAC/C,OAAO,GAAG,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBACvD,MAAM;gBACR,CAAC;gBACD,IAAI,MAAM,GAAG,CAAC,IAAI,YAAY,GAAG,MAAM,GAAG,SAAS;oBAAE,MAAM;gBAC3D,MAAM,IAAI,CAAC,CAAC;gBACZ,MAAM,IAAI,CAAC,CAAC;gBACZ,YAAY,IAAI,MAAM,CAAC;gBACvB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CACpB,GAAG,EAAE;oBACH,MAAM,IAAI,CAAC,CAAC;oBACZ,YAAY,IAAI,MAAM,CAAC;oBACvB,OAAO,EAAE,CAAC;gBACZ,CAAC,EACD,CAAC,KAAc,EAAE,EAAE;oBACjB,OAAO,KAAK,KAAK,CAAC;oBAClB,MAAM,IAAI,CAAC,CAAC;oBACZ,YAAY,IAAI,MAAM,CAAC;oBACvB,OAAO,EAAE,CAAC;gBACZ,CAAC,CACF,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,IAAI,MAAM,KAAK,CAAC;oBAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnD,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;QACF,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,55 +1,92 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
|
-
* Download
|
|
3
|
+
* Download a set of immutable artifacts into a machine cache.
|
|
4
4
|
*
|
|
5
|
-
* The
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* The set is the primitive: one plan wave proves every manifest and asks the
|
|
6
|
+
* cache which chunks are gaps, one chunk wave fetches every gap across the
|
|
7
|
+
* whole set under a single bound, and one seal wave verifies each complete
|
|
8
|
+
* artifact. A lone artifact is a set of one — there is no second path. This
|
|
9
|
+
* shape removes per-artifact serialization: no artifact waits behind
|
|
10
|
+
* another's plan or seal, only behind the shared fetch bound. Each chunk is
|
|
11
|
+
* still its own storage request — the set changes scheduling, not the
|
|
12
|
+
* request count.
|
|
9
13
|
*
|
|
10
14
|
* Every chunk is one content-addressed storage object, so the host's ports
|
|
11
15
|
* fetch chunks independently — parallelism, transport, and decryption are the
|
|
12
16
|
* host's concern; the verification laws here are not.
|
|
13
17
|
*/
|
|
14
18
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
19
|
+
exports.downloadAll = downloadAll;
|
|
15
20
|
exports.download = download;
|
|
16
21
|
const content_js_1 = require("../contracts/content.js");
|
|
17
22
|
const artifacts_js_1 = require("./artifacts.js");
|
|
18
23
|
const parallel_js_1 = require("./parallel.js");
|
|
19
|
-
/** Download
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
24
|
+
/** Download a set of immutable artifacts, fetching only machine-cache gaps.
|
|
25
|
+
* Receipts return in input order; any failure fails the whole set after
|
|
26
|
+
* letting in-flight effects settle. */
|
|
27
|
+
async function downloadAll(artifacts, ports, options = {}) {
|
|
28
|
+
const concurrency = options.concurrency ?? parallel_js_1.DEFAULT_TRANSFER_CONCURRENCY;
|
|
29
|
+
const maxInFlightBytes = options.maxInFlightBytes
|
|
30
|
+
?? parallel_js_1.DEFAULT_TRANSFER_CONCURRENCY * content_js_1.CHUNK_BYTES;
|
|
31
|
+
for (const artifact of artifacts)
|
|
32
|
+
(0, artifacts_js_1.assertTransferArtifact)(artifact);
|
|
33
|
+
const plans = new Array(artifacts.length);
|
|
34
|
+
await (0, parallel_js_1.parallelForEach)([...artifacts.keys()], concurrency, async (slot) => {
|
|
35
|
+
const artifact = artifacts[slot];
|
|
36
|
+
if (artifact === undefined)
|
|
37
|
+
return;
|
|
38
|
+
const checked = (0, content_js_1.checkContentManifest)(await ports.getManifest(artifact), artifact.contentHash);
|
|
39
|
+
if (!checked.ok)
|
|
40
|
+
throw new Error(checked.error);
|
|
41
|
+
const manifest = checked.value;
|
|
42
|
+
const missing = [];
|
|
43
|
+
for (const index of manifest.chunks.keys()) {
|
|
44
|
+
if (!await ports.hasChunk(artifact, manifest, index))
|
|
45
|
+
missing.push(index);
|
|
46
|
+
}
|
|
47
|
+
plans[slot] = { artifact, manifest, missing };
|
|
48
|
+
});
|
|
49
|
+
const gaps = plans.flatMap((plan) => plan === undefined ? [] : plan.missing.map((index) => ({ plan, index })));
|
|
50
|
+
await (0, parallel_js_1.weightedParallelForEach)(gaps, concurrency, maxInFlightBytes, ({ plan, index }) => plan.manifest.chunks[index]?.bytes ?? 0, async ({ plan, index }) => {
|
|
51
|
+
const expected = plan.manifest.chunks[index];
|
|
33
52
|
if (expected === undefined)
|
|
34
53
|
throw new Error('download manifest named an unknown chunk index');
|
|
35
|
-
const bytes = await ports.getChunk(artifact, manifest, index);
|
|
54
|
+
const bytes = await ports.getChunk(plan.artifact, plan.manifest, index);
|
|
36
55
|
if (!(bytes instanceof Uint8Array)
|
|
37
56
|
|| bytes.byteLength !== expected.bytes
|
|
38
57
|
|| ports.sha256Hex(bytes) !== expected.sha256) {
|
|
39
58
|
throw new Error(`downloaded content chunk ${index} failed verification`);
|
|
40
59
|
}
|
|
41
|
-
await ports.putChunk(artifact, manifest, index, bytes);
|
|
60
|
+
await ports.putChunk(plan.artifact, plan.manifest, index, bytes);
|
|
61
|
+
});
|
|
62
|
+
const receipts = new Array(artifacts.length);
|
|
63
|
+
await (0, parallel_js_1.parallelForEach)([...plans.keys()], concurrency, async (slot) => {
|
|
64
|
+
const plan = plans[slot];
|
|
65
|
+
if (plan === undefined)
|
|
66
|
+
return;
|
|
67
|
+
const sealed = await ports.sealArtifact(plan.artifact, plan.manifest);
|
|
68
|
+
if (sealed.bytes !== plan.manifest.bytes || sealed.contentHash !== plan.manifest.contentHash) {
|
|
69
|
+
throw new Error(`downloaded artifact ${plan.artifact.contentHash} failed verification`);
|
|
70
|
+
}
|
|
71
|
+
receipts[slot] = {
|
|
72
|
+
artifact: plan.artifact,
|
|
73
|
+
manifest: plan.manifest,
|
|
74
|
+
local: sealed.local,
|
|
75
|
+
downloadedChunks: plan.missing.length,
|
|
76
|
+
reusedChunks: plan.manifest.chunks.length - plan.missing.length,
|
|
77
|
+
};
|
|
78
|
+
});
|
|
79
|
+
return receipts.map((receipt) => {
|
|
80
|
+
if (receipt === undefined)
|
|
81
|
+
throw new Error('download set lost an artifact receipt');
|
|
82
|
+
return receipt;
|
|
42
83
|
});
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
artifact
|
|
49
|
-
|
|
50
|
-
local: sealed.local,
|
|
51
|
-
downloadedChunks: missing.length,
|
|
52
|
-
reusedChunks: manifest.chunks.length - missing.length,
|
|
53
|
-
};
|
|
84
|
+
}
|
|
85
|
+
/** Download one immutable artifact — a set of one. */
|
|
86
|
+
async function download(artifact, ports, options = {}) {
|
|
87
|
+
const [receipt] = await downloadAll([artifact], ports, options);
|
|
88
|
+
if (receipt === undefined)
|
|
89
|
+
throw new Error('download set lost an artifact receipt');
|
|
90
|
+
return receipt;
|
|
54
91
|
}
|
|
55
92
|
//# sourceMappingURL=download.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":";AAAA
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;GAeG;;AAoEH,kCAmEC;AAGD,4BAQC;AAhJD,wDAAkG;AAElG,iDAA+E;AAC/E,+CAIuB;AAwDvB;;uCAEuC;AAChC,KAAK,UAAU,WAAW,CAC/B,SAAsC,EACtC,KAAmC,EACnC,UAA2B,EAAE;IAE7B,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,0CAA4B,CAAC;IACxE,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB;WAC5C,0CAA4B,GAAG,wBAAW,CAAC;IAChD,KAAK,MAAM,QAAQ,IAAI,SAAS;QAAE,IAAA,qCAAsB,EAAC,QAAQ,CAAC,CAAC;IAEnE,MAAM,KAAK,GAAiC,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACxE,MAAM,IAAA,6BAAe,EAAC,CAAC,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QACvE,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO;QACnC,MAAM,OAAO,GAAG,IAAA,iCAAoB,EAClC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,EACjC,QAAQ,CAAC,WAAW,CACrB,CAAC;QACF,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;QAC/B,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3C,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5E,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAClC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IAC5E,MAAM,IAAA,qCAAuB,EAC3B,IAAI,EACJ,WAAW,EACX,gBAAgB,EAChB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,EAC5D,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACxE,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;eAC7B,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,KAAK;eACnC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,sBAAsB,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IACnE,CAAC,CACF,CAAC;IAEF,MAAM,QAAQ,GAAmD,IAAI,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAC7F,MAAM,IAAA,6BAAe,EAAC,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QACnE,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO;QAC/B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtE,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,WAAW,KAAK,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,QAAQ,CAAC,WAAW,sBAAsB,CAAC,CAAC;QAC1F,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,GAAG;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YACrC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM;SAChE,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC9B,IAAI,OAAO,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QACpF,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,sDAAsD;AAC/C,KAAK,UAAU,QAAQ,CAC5B,QAA0B,EAC1B,KAAmC,EACnC,UAA2B,EAAE;IAE7B,MAAM,CAAC,OAAO,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IAChE,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IACpF,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.DEFAULT_TRANSFER_CONCURRENCY = void 0;
|
|
4
4
|
exports.parallelForEach = parallelForEach;
|
|
5
|
+
exports.weightedParallelForEach = weightedParallelForEach;
|
|
5
6
|
/** Eight full-size frames keep one ordinary high-latency connection busy
|
|
6
7
|
* without making the SDK's default memory bound unreasonably large. Hosts
|
|
7
8
|
* with a stricter budget can always pass a smaller explicit window. */
|
|
@@ -32,4 +33,66 @@ async function parallelForEach(items, concurrency, effect) {
|
|
|
32
33
|
if (failure !== undefined)
|
|
33
34
|
throw failure;
|
|
34
35
|
}
|
|
36
|
+
/** Run independent effects under both a count bound and a total weight
|
|
37
|
+
* bound. Input order is the admission order; a failure stops new admission
|
|
38
|
+
* and waits for already-started effects to settle. */
|
|
39
|
+
async function weightedParallelForEach(items, concurrency, maxWeight, weightOf, effect) {
|
|
40
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1) {
|
|
41
|
+
throw new Error('transfer concurrency must be a positive integer');
|
|
42
|
+
}
|
|
43
|
+
if (!Number.isSafeInteger(maxWeight) || maxWeight < 1) {
|
|
44
|
+
throw new Error('transfer weight bound must be a positive integer');
|
|
45
|
+
}
|
|
46
|
+
const weights = items.map((item) => {
|
|
47
|
+
const weight = weightOf(item);
|
|
48
|
+
if (!Number.isSafeInteger(weight) || weight < 1 || weight > maxWeight) {
|
|
49
|
+
throw new Error('transfer item weight exceeds its positive bound');
|
|
50
|
+
}
|
|
51
|
+
return weight;
|
|
52
|
+
});
|
|
53
|
+
await new Promise((resolve, reject) => {
|
|
54
|
+
let cursor = 0;
|
|
55
|
+
let active = 0;
|
|
56
|
+
let activeWeight = 0;
|
|
57
|
+
let failure;
|
|
58
|
+
const advance = () => {
|
|
59
|
+
if (failure !== undefined) {
|
|
60
|
+
if (active === 0)
|
|
61
|
+
reject(failure);
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
while (cursor < items.length && active < concurrency) {
|
|
65
|
+
const item = items[cursor];
|
|
66
|
+
const weight = weights[cursor];
|
|
67
|
+
if (item === undefined || weight === undefined) {
|
|
68
|
+
failure = new Error('transfer scheduler lost an item');
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
if (active > 0 && activeWeight + weight > maxWeight)
|
|
72
|
+
break;
|
|
73
|
+
cursor += 1;
|
|
74
|
+
active += 1;
|
|
75
|
+
activeWeight += weight;
|
|
76
|
+
void effect(item).then(() => {
|
|
77
|
+
active -= 1;
|
|
78
|
+
activeWeight -= weight;
|
|
79
|
+
advance();
|
|
80
|
+
}, (error) => {
|
|
81
|
+
failure ??= error;
|
|
82
|
+
active -= 1;
|
|
83
|
+
activeWeight -= weight;
|
|
84
|
+
advance();
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
if (failure !== undefined) {
|
|
88
|
+
if (active === 0)
|
|
89
|
+
reject(failure);
|
|
90
|
+
}
|
|
91
|
+
else if (cursor === items.length && active === 0) {
|
|
92
|
+
resolve();
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
advance();
|
|
96
|
+
});
|
|
97
|
+
}
|
|
35
98
|
//# sourceMappingURL=parallel.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parallel.js","sourceRoot":"","sources":["../../src/transfer/parallel.ts"],"names":[],"mappings":";;;AAMA,0CA2BC;
|
|
1
|
+
{"version":3,"file":"parallel.js","sourceRoot":"","sources":["../../src/transfer/parallel.ts"],"names":[],"mappings":";;;AAMA,0CA2BC;AAKD,0DA+DC;AArGD;;uEAEuE;AAC1D,QAAA,4BAA4B,GAAG,CAAC,CAAC;AAE9C,+EAA+E;AACxE,KAAK,UAAU,eAAe,CACnC,KAAmB,EACnB,WAAmB,EACnB,MAAkC;IAElC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,OAAgB,CAAC;IACrB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,OAAO,OAAO,KAAK,SAAS,IAAI,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACtD,MAAM,KAAK,GAAG,MAAM,CAAC;YACrB,MAAM,IAAI,CAAC,CAAC;YACZ,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,IAAI,KAAK,SAAS;gBAAE,SAAS;YACjC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,KAAK,KAAK,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CACpE,CAAC;IACF,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,OAAO,CAAC;AAC3C,CAAC;AAED;;sDAEsD;AAC/C,KAAK,UAAU,uBAAuB,CAC3C,KAAmB,EACnB,WAAmB,EACnB,SAAiB,EACjB,QAA6B,EAC7B,MAAkC;IAElC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,SAAS,EAAE,CAAC;YACtE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;IACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,OAAgB,CAAC;QACrB,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,IAAI,MAAM,KAAK,CAAC;oBAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YACD,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,GAAG,WAAW,EAAE,CAAC;gBACrD,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/B,IAAI,IAAI,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBAC/C,OAAO,GAAG,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBACvD,MAAM;gBACR,CAAC;gBACD,IAAI,MAAM,GAAG,CAAC,IAAI,YAAY,GAAG,MAAM,GAAG,SAAS;oBAAE,MAAM;gBAC3D,MAAM,IAAI,CAAC,CAAC;gBACZ,MAAM,IAAI,CAAC,CAAC;gBACZ,YAAY,IAAI,MAAM,CAAC;gBACvB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CACpB,GAAG,EAAE;oBACH,MAAM,IAAI,CAAC,CAAC;oBACZ,YAAY,IAAI,MAAM,CAAC;oBACvB,OAAO,EAAE,CAAC;gBACZ,CAAC,EACD,CAAC,KAAc,EAAE,EAAE;oBACjB,OAAO,KAAK,KAAK,CAAC;oBAClB,MAAM,IAAI,CAAC,CAAC;oBACZ,YAAY,IAAI,MAAM,CAAC;oBACvB,OAAO,EAAE,CAAC;gBACZ,CAAC,CACF,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,IAAI,MAAM,KAAK,CAAC;oBAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;iBAAM,IAAI,MAAM,KAAK,KAAK,CAAC,MAAM,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnD,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC,CAAC;QACF,OAAO,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@amalgm/live",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.49",
|
|
4
4
|
"description": "Portable realtime SDK for Amalgm — wire contracts, mutation discipline, and delivery state machines behind host-injected adapters.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist-cjs/index.js",
|