@amalgm/live 0.2.16 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PURPOSE.md +8 -0
- package/dist/contracts/content.d.ts +9 -4
- package/dist/contracts/content.d.ts.map +1 -1
- package/dist/contracts/content.js +10 -5
- package/dist/contracts/content.js.map +1 -1
- package/dist/transfer/batches.d.ts +19 -0
- package/dist/transfer/batches.d.ts.map +1 -0
- package/dist/transfer/batches.js +40 -0
- package/dist/transfer/batches.js.map +1 -0
- package/dist/transfer/download.d.ts +5 -0
- package/dist/transfer/download.d.ts.map +1 -1
- package/dist/transfer/download.js +49 -12
- package/dist/transfer/download.js.map +1 -1
- package/dist/transfer/upload.d.ts +3 -0
- package/dist/transfer/upload.d.ts.map +1 -1
- package/dist/transfer/upload.js +35 -10
- package/dist/transfer/upload.js.map +1 -1
- package/package.json +1 -1
package/PURPOSE.md
CHANGED
|
@@ -150,6 +150,14 @@ its cutover.
|
|
|
150
150
|
writes received bytes once before their registered adapter consumes them.
|
|
151
151
|
Reopening a complete artifact to recreate or repeat an earlier integrity
|
|
152
152
|
proof is redundant work, not additional correctness.
|
|
153
|
+
31. **A content block is not a round trip.** Text and binary retain independent
|
|
154
|
+
content-addressed block identities, while cold upload and download move
|
|
155
|
+
adjacent missing blocks in bounded binary batches. Batching may reduce
|
|
156
|
+
transport work but can never weaken per-block verification or commit order.
|
|
157
|
+
32. **A content layout owns a cloud namespace.** A writer change that derives
|
|
158
|
+
different block identities for the same complete bytes advances the
|
|
159
|
+
content contract and storage namespace together. Readers remain additive;
|
|
160
|
+
an immutable manifest is never reinterpreted or overwritten in place.
|
|
153
161
|
|
|
154
162
|
## Five surfaces, one behavior
|
|
155
163
|
|
|
@@ -10,7 +10,12 @@
|
|
|
10
10
|
* host effects; this file owns the shape and its laws.
|
|
11
11
|
*/
|
|
12
12
|
import { type Checked } from './validate.js';
|
|
13
|
-
|
|
13
|
+
/** Version 3 changes only the canonical writer layout: ordinary files begin
|
|
14
|
+
* at Dropbox's stable 4 MiB block size. Version 2 remains readable because
|
|
15
|
+
* immutable cloud heads created by an older writer must still materialize. */
|
|
16
|
+
export declare const LEGACY_CONTENT_CONTRACT = "amalgm-entity-content@2";
|
|
17
|
+
export declare const CONTENT_CONTRACT = "amalgm-entity-content@3";
|
|
18
|
+
export type ContentContract = typeof CONTENT_CONTRACT | typeof LEGACY_CONTENT_CONTRACT;
|
|
14
19
|
export declare const MIN_CONTENT_BLOCK_BYTES: number;
|
|
15
20
|
export declare const MAX_CONTENT_BLOCK_BYTES: number;
|
|
16
21
|
/** Maximum accepted block size. Kept as the public manifest-bound name. */
|
|
@@ -21,7 +26,7 @@ export interface ContentChunk {
|
|
|
21
26
|
readonly bytes: number;
|
|
22
27
|
}
|
|
23
28
|
export interface ContentManifest {
|
|
24
|
-
readonly contract:
|
|
29
|
+
readonly contract: ContentContract;
|
|
25
30
|
readonly contentHash: string;
|
|
26
31
|
readonly bytes: number;
|
|
27
32
|
readonly chunks: readonly ContentChunk[];
|
|
@@ -39,8 +44,8 @@ export declare function checkContentManifest(input: unknown, expectedHash?: stri
|
|
|
39
44
|
* hand-built manifest cannot drift from the derivation rule.
|
|
40
45
|
*/
|
|
41
46
|
export declare function manifestFromChunks(contentHash: string, chunks: readonly ContentChunk[]): Checked<ContentManifest>;
|
|
42
|
-
/** Syncthing's adaptive
|
|
43
|
-
*
|
|
47
|
+
/** Dropbox-sized cold blocks, widened by Syncthing's adaptive rule only when
|
|
48
|
+
* needed to keep a manifest below 2,000 blocks, capped at 16 MiB. */
|
|
44
49
|
export declare function contentBlockSize(totalBytes: number): number;
|
|
45
50
|
/** Split a byte count into adaptive full blocks, then the remainder. */
|
|
46
51
|
export declare function chunkSizes(totalBytes: number): number[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../src/contracts/content.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,OAAO,EAAwB,MAAM,eAAe,CAAC;AAEnE,eAAO,MAAM,gBAAgB,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"content.d.ts","sourceRoot":"","sources":["../../src/contracts/content.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,KAAK,OAAO,EAAwB,MAAM,eAAe,CAAC;AAEnE;;8EAE8E;AAC9E,eAAO,MAAM,uBAAuB,4BAA4B,CAAC;AACjE,eAAO,MAAM,gBAAgB,4BAA4B,CAAC;AAC1D,MAAM,MAAM,eAAe,GAAG,OAAO,gBAAgB,GAAG,OAAO,uBAAuB,CAAC;AAEvF,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AACvD,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD,2EAA2E;AAC3E,eAAO,MAAM,WAAW,QAA0B,CAAC;AACnD,eAAO,MAAM,kBAAkB,OAAQ,CAAC;AAIxC,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC;CAC1C;AAED;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,EAAE,YAAY,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAoCjH;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,OAAO,CAAC,eAAe,CAAC,CAK1B;AAED;qEACqE;AACrE,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAO3D;AAED,wEAAwE;AACxE,wBAAgB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAQvD"}
|
|
@@ -10,8 +10,12 @@
|
|
|
10
10
|
* host effects; this file owns the shape and its laws.
|
|
11
11
|
*/
|
|
12
12
|
import { pass, fail, isRecord } from './validate.js';
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
/** Version 3 changes only the canonical writer layout: ordinary files begin
|
|
14
|
+
* at Dropbox's stable 4 MiB block size. Version 2 remains readable because
|
|
15
|
+
* immutable cloud heads created by an older writer must still materialize. */
|
|
16
|
+
export const LEGACY_CONTENT_CONTRACT = 'amalgm-entity-content@2';
|
|
17
|
+
export const CONTENT_CONTRACT = 'amalgm-entity-content@3';
|
|
18
|
+
export const MIN_CONTENT_BLOCK_BYTES = 4 * 1024 * 1024;
|
|
15
19
|
export const MAX_CONTENT_BLOCK_BYTES = 16 * 1024 * 1024;
|
|
16
20
|
/** Maximum accepted block size. Kept as the public manifest-bound name. */
|
|
17
21
|
export const CHUNK_BYTES = MAX_CONTENT_BLOCK_BYTES;
|
|
@@ -25,6 +29,7 @@ const HASH = /^[0-9a-f]{64}$/;
|
|
|
25
29
|
*/
|
|
26
30
|
export function checkContentManifest(input, expectedHash = null) {
|
|
27
31
|
if (!isRecord(input) ||
|
|
32
|
+
(input.contract !== CONTENT_CONTRACT && input.contract !== LEGACY_CONTENT_CONTRACT) ||
|
|
28
33
|
typeof input.contentHash !== 'string' ||
|
|
29
34
|
!HASH.test(input.contentHash) ||
|
|
30
35
|
(expectedHash !== null && input.contentHash !== expectedHash) ||
|
|
@@ -48,7 +53,7 @@ export function checkContentManifest(input, expectedHash = null) {
|
|
|
48
53
|
return fail('empty entity content must have no chunks');
|
|
49
54
|
}
|
|
50
55
|
return pass({
|
|
51
|
-
contract:
|
|
56
|
+
contract: input.contract,
|
|
52
57
|
contentHash: input.contentHash,
|
|
53
58
|
bytes,
|
|
54
59
|
chunks: chunks.map((chunk) => ({ sha256: chunk.sha256, bytes: chunk.bytes })),
|
|
@@ -62,8 +67,8 @@ export function checkContentManifest(input, expectedHash = null) {
|
|
|
62
67
|
export function manifestFromChunks(contentHash, chunks) {
|
|
63
68
|
return checkContentManifest({ contract: CONTENT_CONTRACT, contentHash, bytes: chunks.reduce((t, c) => t + c.bytes, 0), chunks }, contentHash);
|
|
64
69
|
}
|
|
65
|
-
/** Syncthing's adaptive
|
|
66
|
-
*
|
|
70
|
+
/** Dropbox-sized cold blocks, widened by Syncthing's adaptive rule only when
|
|
71
|
+
* needed to keep a manifest below 2,000 blocks, capped at 16 MiB. */
|
|
67
72
|
export function contentBlockSize(totalBytes) {
|
|
68
73
|
if (!Number.isSafeInteger(totalBytes) || totalBytes < 0)
|
|
69
74
|
return MIN_CONTENT_BLOCK_BYTES;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"content.js","sourceRoot":"","sources":["../../src/contracts/content.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAgB,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEnE,MAAM,CAAC,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"content.js","sourceRoot":"","sources":["../../src/contracts/content.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAgB,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEnE;;8EAE8E;AAC9E,MAAM,CAAC,MAAM,uBAAuB,GAAG,yBAAyB,CAAC;AACjE,MAAM,CAAC,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;AAG1D,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACxD,2EAA2E;AAC3E,MAAM,CAAC,MAAM,WAAW,GAAG,uBAAuB,CAAC;AACnD,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAExC,MAAM,IAAI,GAAG,gBAAgB,CAAC;AAc9B;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAc,EAAE,eAA8B,IAAI;IACrF,IACE,CAAC,QAAQ,CAAC,KAAK,CAAC;QAChB,CAAC,KAAK,CAAC,QAAQ,KAAK,gBAAgB,IAAI,KAAK,CAAC,QAAQ,KAAK,uBAAuB,CAAC;QACnF,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QACrC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC;QAC7B,CAAC,YAAY,KAAK,IAAI,IAAI,KAAK,CAAC,WAAW,KAAK,YAAY,CAAC;QAC7D,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC;QACjC,KAAK,CAAC,KAAgB,GAAG,CAAC;QAC3B,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;QAC5B,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,kBAAkB;QACxC,KAAK,CAAC,MAAM,CAAC,IAAI,CACf,CAAC,KAAK,EAAE,EAAE,CACR,CAAC,QAAQ,CAAC,KAAK,CAAC;YAChB,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC;YACjC,KAAK,CAAC,KAAgB,GAAG,CAAC;YAC1B,KAAK,CAAC,KAAgB,GAAG,WAAW,CACxC,EACD,CAAC;QACD,OAAO,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACpD,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAe,CAAC;IACpC,MAAM,MAAM,GAAG,KAAK,CAAC,MAA0D,CAAC;IAChF,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,EAAE,CAAC;QACtE,OAAO,IAAI,CAAC,+CAA+C,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,KAAK,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,IAAI,CAAC,0CAA0C,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,IAAI,CAAC;QACV,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;KAC9E,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,WAAmB,EACnB,MAA+B;IAE/B,OAAO,oBAAoB,CACzB,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EACnG,WAAW,CACZ,CAAC;AACJ,CAAC;AAED;qEACqE;AACrE,MAAM,UAAU,gBAAgB,CAAC,UAAkB;IACjD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;QAAE,OAAO,uBAAuB,CAAC;IACxF,IAAI,KAAK,GAAG,uBAAuB,CAAC;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC,GAAG,kBAAkB,IAAI,KAAK,GAAG,uBAAuB,EAAE,CAAC;QAC7F,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,UAAU,CAAC,UAAkB;IAC3C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IACnE,MAAM,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,SAAS,GAAG,UAAU,EAAE,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,UAAU,EAAE,CAAC;QACxE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cold file transport batches.
|
|
3
|
+
*
|
|
4
|
+
* Content blocks are durable identities. They are not network requests.
|
|
5
|
+
* Adjacent missing blocks may cross the wire together while remaining
|
|
6
|
+
* independently addressed and verified at both ends.
|
|
7
|
+
*/
|
|
8
|
+
import type { ContentManifest } from '../contracts/content.js';
|
|
9
|
+
/** One batch may fill the existing binary frame bound. Hosts lower transfer
|
|
10
|
+
* concurrency when their memory budget cannot admit eight full batches. */
|
|
11
|
+
export declare const FILE_TRANSFER_BATCH_BYTES: number;
|
|
12
|
+
export interface ChunkBatch {
|
|
13
|
+
readonly indexes: readonly number[];
|
|
14
|
+
readonly offset: number;
|
|
15
|
+
readonly bytes: number;
|
|
16
|
+
}
|
|
17
|
+
/** Group ascending, adjacent chunk indexes without crossing the byte bound. */
|
|
18
|
+
export declare function planChunkBatches(indexes: readonly number[], manifest: ContentManifest): readonly ChunkBatch[];
|
|
19
|
+
//# sourceMappingURL=batches.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batches.d.ts","sourceRoot":"","sources":["../../src/transfer/batches.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D;2EAC2E;AAC3E,eAAO,MAAM,yBAAyB,QAAmB,CAAC;AAE1D,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,+EAA+E;AAC/E,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,QAAQ,EAAE,eAAe,GACxB,SAAS,UAAU,EAAE,CA0BvB"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cold file transport batches.
|
|
3
|
+
*
|
|
4
|
+
* Content blocks are durable identities. They are not network requests.
|
|
5
|
+
* Adjacent missing blocks may cross the wire together while remaining
|
|
6
|
+
* independently addressed and verified at both ends.
|
|
7
|
+
*/
|
|
8
|
+
/** One batch may fill the existing binary frame bound. Hosts lower transfer
|
|
9
|
+
* concurrency when their memory budget cannot admit eight full batches. */
|
|
10
|
+
export const FILE_TRANSFER_BATCH_BYTES = 16 * 1024 * 1024;
|
|
11
|
+
/** Group ascending, adjacent chunk indexes without crossing the byte bound. */
|
|
12
|
+
export function planChunkBatches(indexes, manifest) {
|
|
13
|
+
const offsets = [];
|
|
14
|
+
let offset = 0;
|
|
15
|
+
for (const chunk of manifest.chunks) {
|
|
16
|
+
offsets.push(offset);
|
|
17
|
+
offset += chunk.bytes;
|
|
18
|
+
}
|
|
19
|
+
const batches = [];
|
|
20
|
+
for (const index of [...indexes].sort((left, right) => left - right)) {
|
|
21
|
+
const chunk = manifest.chunks[index];
|
|
22
|
+
const chunkOffset = offsets[index];
|
|
23
|
+
if (!chunk || chunkOffset === undefined)
|
|
24
|
+
throw new Error(`content manifest has no chunk ${index}`);
|
|
25
|
+
const prior = batches.at(-1);
|
|
26
|
+
const adjacent = prior?.indexes.at(-1) === index - 1;
|
|
27
|
+
if (prior && adjacent && prior.bytes + chunk.bytes <= FILE_TRANSFER_BATCH_BYTES) {
|
|
28
|
+
batches[batches.length - 1] = {
|
|
29
|
+
indexes: [...prior.indexes, index],
|
|
30
|
+
offset: prior.offset,
|
|
31
|
+
bytes: prior.bytes + chunk.bytes,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
else {
|
|
35
|
+
batches.push({ indexes: [index], offset: chunkOffset, bytes: chunk.bytes });
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return batches;
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=batches.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batches.js","sourceRoot":"","sources":["../../src/transfer/batches.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH;2EAC2E;AAC3E,MAAM,CAAC,MAAM,yBAAyB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAQ1D,+EAA+E;AAC/E,MAAM,UAAU,gBAAgB,CAC9B,OAA0B,EAC1B,QAAyB;IAEzB,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,MAAM,IAAI,KAAK,CAAC,KAAK,CAAC;IACxB,CAAC;IAED,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;QACrE,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrC,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,KAAK,IAAI,WAAW,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,EAAE,CAAC,CAAC;QACnG,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,QAAQ,GAAG,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC;QACrD,IAAI,KAAK,IAAI,QAAQ,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,yBAAyB,EAAE,CAAC;YAChF,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG;gBAC5B,OAAO,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC;gBAClC,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,KAAK,EAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK;aACjC,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -15,7 +15,12 @@ export interface DownloadPorts<LocalArtifact> {
|
|
|
15
15
|
/** True only when this exact content-addressed chunk is locally verified. */
|
|
16
16
|
hasChunk(artifact: TransferArtifact, manifest: ContentManifest, index: number): Promise<boolean>;
|
|
17
17
|
getChunk(artifact: TransferArtifact, manifest: ContentManifest, index: number): Promise<Uint8Array>;
|
|
18
|
+
/** Fetch adjacent logical file blocks in one binary response, concatenated
|
|
19
|
+
* in `indexes` order. */
|
|
20
|
+
getChunkBatch?(artifact: TransferArtifact, manifest: ContentManifest, indexes: readonly number[]): Promise<Uint8Array>;
|
|
18
21
|
putChunk(artifact: TransferArtifact, manifest: ContentManifest, index: number, bytes: Uint8Array): Promise<void>;
|
|
22
|
+
/** Persist one already-verified contiguous file batch. */
|
|
23
|
+
putChunkBatch?(artifact: TransferArtifact, manifest: ContentManifest, indexes: readonly number[], bytes: Uint8Array): Promise<void>;
|
|
19
24
|
/** Verify and expose the complete immutable artifact assembled from cache. */
|
|
20
25
|
sealArtifact(artifact: TransferArtifact, manifest: ContentManifest): Promise<SealedArtifact<LocalArtifact>>;
|
|
21
26
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAwB,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAA0B,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAwB,KAAK,eAAe,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAA0B,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAI/E,MAAM,WAAW,aAAa,CAAC,aAAa;IAC1C,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,WAAW,CAAC,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1D,6EAA6E;IAC7E,QAAQ,CACN,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,QAAQ,CACN,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,UAAU,CAAC,CAAC;IACvB;6BACyB;IACzB,aAAa,CAAC,CACZ,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,OAAO,CAAC,UAAU,CAAC,CAAC;IACvB,QAAQ,CACN,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,0DAA0D;IAC1D,aAAa,CAAC,CACZ,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,8EAA8E;IAC9E,YAAY,CACV,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC,CAAC;CAC3C;AAED,MAAM,WAAW,cAAc,CAAC,aAAa;IAC3C,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,eAAe,CAAC,aAAa;IAC5C,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;IAC9B,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAwBD,yEAAyE;AACzE,wBAAsB,QAAQ,CAAC,aAAa,EAC1C,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,aAAa,CAAC,aAAa,CAAC,EACnC,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAwDzC"}
|
|
@@ -8,7 +8,26 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { checkContentManifest } from '../contracts/content.js';
|
|
10
10
|
import { assertTransferArtifact } from './artifacts.js';
|
|
11
|
+
import { planChunkBatches } from './batches.js';
|
|
11
12
|
import { DEFAULT_TRANSFER_CONCURRENCY, parallelForEach } from './parallel.js';
|
|
13
|
+
const checkedBatchChunks = (bytes, indexes, manifest, sha256Hex) => {
|
|
14
|
+
const chunks = [];
|
|
15
|
+
let offset = 0;
|
|
16
|
+
for (const index of indexes) {
|
|
17
|
+
const expected = manifest.chunks[index];
|
|
18
|
+
if (!expected)
|
|
19
|
+
throw new Error(`download manifest named an unknown chunk index`);
|
|
20
|
+
const chunk = bytes.subarray(offset, offset + expected.bytes);
|
|
21
|
+
if (chunk.byteLength !== expected.bytes || sha256Hex(chunk) !== expected.sha256) {
|
|
22
|
+
throw new Error(`downloaded content chunk ${index} failed verification`);
|
|
23
|
+
}
|
|
24
|
+
chunks.push(chunk);
|
|
25
|
+
offset += expected.bytes;
|
|
26
|
+
}
|
|
27
|
+
if (offset !== bytes.byteLength)
|
|
28
|
+
throw new Error('downloaded file batch has the wrong byte count');
|
|
29
|
+
return chunks;
|
|
30
|
+
};
|
|
12
31
|
/** Download one immutable artifact, fetching only machine-cache gaps. */
|
|
13
32
|
export async function download(artifact, ports, options = {}) {
|
|
14
33
|
assertTransferArtifact(artifact);
|
|
@@ -21,18 +40,36 @@ export async function download(artifact, ports, options = {}) {
|
|
|
21
40
|
if (!await ports.hasChunk(artifact, manifest, index))
|
|
22
41
|
missing.push(index);
|
|
23
42
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
43
|
+
if (artifact.kind === 'file' && ports.getChunkBatch) {
|
|
44
|
+
await parallelForEach(planChunkBatches(missing, manifest), options.concurrency ?? DEFAULT_TRANSFER_CONCURRENCY, async (batch) => {
|
|
45
|
+
const bytes = await ports.getChunkBatch(artifact, manifest, batch.indexes);
|
|
46
|
+
if (!(bytes instanceof Uint8Array))
|
|
47
|
+
throw new Error('downloaded file batch is not binary bytes');
|
|
48
|
+
const chunks = checkedBatchChunks(bytes, batch.indexes, manifest, ports.sha256Hex);
|
|
49
|
+
if (ports.putChunkBatch) {
|
|
50
|
+
await ports.putChunkBatch(artifact, manifest, batch.indexes, bytes);
|
|
51
|
+
}
|
|
52
|
+
else {
|
|
53
|
+
for (let offset = 0; offset < batch.indexes.length; offset += 1) {
|
|
54
|
+
await ports.putChunk(artifact, manifest, batch.indexes[offset], chunks[offset]);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
else {
|
|
60
|
+
await parallelForEach(missing, options.concurrency ?? DEFAULT_TRANSFER_CONCURRENCY, async (index) => {
|
|
61
|
+
const expected = manifest.chunks[index];
|
|
62
|
+
if (expected === undefined)
|
|
63
|
+
throw new Error('download manifest named an unknown chunk index');
|
|
64
|
+
const bytes = await ports.getChunk(artifact, manifest, index);
|
|
65
|
+
if (!(bytes instanceof Uint8Array)
|
|
66
|
+
|| bytes.byteLength !== expected.bytes
|
|
67
|
+
|| ports.sha256Hex(bytes) !== expected.sha256) {
|
|
68
|
+
throw new Error(`downloaded content chunk ${index} failed verification`);
|
|
69
|
+
}
|
|
70
|
+
await ports.putChunk(artifact, manifest, index, bytes);
|
|
71
|
+
});
|
|
72
|
+
}
|
|
36
73
|
const sealed = await ports.sealArtifact(artifact, manifest);
|
|
37
74
|
if (sealed.bytes !== manifest.bytes || sealed.contentHash !== manifest.contentHash) {
|
|
38
75
|
throw new Error(`downloaded artifact ${artifact.contentHash} failed verification`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,oBAAoB,EAAwB,MAAM,yBAAyB,CAAC;AAErF,OAAO,EAAE,sBAAsB,EAAyB,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,4BAA4B,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"download.js","sourceRoot":"","sources":["../../src/transfer/download.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,oBAAoB,EAAwB,MAAM,yBAAyB,CAAC;AAErF,OAAO,EAAE,sBAAsB,EAAyB,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AA6D9E,MAAM,kBAAkB,GAAG,CACzB,KAAiB,EACjB,OAA0B,EAC1B,QAAyB,EACzB,SAAoB,EACG,EAAE;IACzB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACjF,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAChF,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,sBAAsB,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC;IAC3B,CAAC;IACD,IAAI,MAAM,KAAK,KAAK,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACnG,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,QAA0B,EAC1B,KAAmC,EACnC,UAA2B,EAAE;IAE7B,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,oBAAoB,CAClC,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,EACjC,QAAQ,CAAC,WAAW,CACrB,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAE/B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACpD,MAAM,eAAe,CACnB,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,EACnC,OAAO,CAAC,WAAW,IAAI,4BAA4B,EACnD,KAAK,EAAE,KAAK,EAAE,EAAE;YACd,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,aAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5E,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;gBAAE,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YACjG,MAAM,MAAM,GAAG,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YACnF,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;gBACxB,MAAM,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtE,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;oBAChE,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAE,EAAE,MAAM,CAAC,MAAM,CAAE,CAAC,CAAC;gBACpF,CAAC;YACH,CAAC;QACH,CAAC,CACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,IAAI,4BAA4B,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAClG,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxC,IAAI,QAAQ,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YAC9F,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YAC9D,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC;mBAC7B,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,KAAK;mBACnC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,KAAK,sBAAsB,CAAC,CAAC;YAC3E,CAAC;YACD,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5D,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,WAAW,EAAE,CAAC;QACnF,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,WAAW,sBAAsB,CAAC,CAAC;IACrF,CAAC;IACD,OAAO;QACL,QAAQ;QACR,QAAQ;QACR,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,gBAAgB,EAAE,OAAO,CAAC,MAAM;QAChC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;KACtD,CAAC;AACJ,CAAC"}
|
|
@@ -26,6 +26,9 @@ export interface UploadPorts {
|
|
|
26
26
|
/** Return the manifest indexes not yet durable at the authority. */
|
|
27
27
|
missingChunks(artifact: TransferArtifact, manifest: ContentManifest): Promise<readonly number[]>;
|
|
28
28
|
putChunk(artifact: TransferArtifact, manifest: ContentManifest, index: number, bytes: Uint8Array): Promise<void>;
|
|
29
|
+
/** Move adjacent logical file blocks in one binary request. The blocks keep
|
|
30
|
+
* their individual manifest identities and are verified before this call. */
|
|
31
|
+
putChunkBatch?(artifact: TransferArtifact, manifest: ContentManifest, indexes: readonly number[], bytes: Uint8Array): Promise<void>;
|
|
29
32
|
/** Make the artifact visible. Called only after every missing chunk lands. */
|
|
30
33
|
putManifest(artifact: TransferArtifact, manifest: ContentManifest): Promise<void>;
|
|
31
34
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/transfer/upload.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAA0B,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"upload.d.ts","sourceRoot":"","sources":["../../src/transfer/upload.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,yBAAyB,CAAC;AACjC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAA0B,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAI/E,MAAM,WAAW,YAAY;IAC3B,wEAAwE;IACxE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sEAAsE;IACtE,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAC5B;+EAC2E;IAC3E,QAAQ,CAAC,gBAAgB,CAAC,EAAE,OAAO,CAAC;IACpC,qEAAqE;IACrE,MAAM,CAAC,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/C,0DAA0D;IAC1D,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,oEAAoE;IACpE,aAAa,CACX,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,GACxB,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC;IAC9B,QAAQ,CACN,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB;iFAC6E;IAC7E,aAAa,CAAC,CACZ,QAAQ,EAAE,gBAAgB,EAC1B,QAAQ,EAAE,eAAe,EACzB,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,KAAK,EAAE,UAAU,GAChB,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,8EAA8E;IAC9E,WAAW,CAAC,QAAQ,EAAE,gBAAgB,EAAE,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACnF;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,QAAQ,EAAE,gBAAgB,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAyCD;;;0CAG0C;AAC1C,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,gBAAgB,EAC1B,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,SAAS,GACnB,eAAe,CAejB;AA6BD,2EAA2E;AAC3E,wBAAsB,MAAM,CAC1B,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAwExB"}
|
package/dist/transfer/upload.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import { checkContentManifest, MAX_CONTENT_BLOCK_BYTES, chunkSizes, manifestFromChunks, } from '../contracts/content.js';
|
|
9
9
|
import { assertTransferArtifact } from './artifacts.js';
|
|
10
|
+
import { planChunkBatches } from './batches.js';
|
|
10
11
|
import { DEFAULT_TRANSFER_CONCURRENCY, parallelForEach } from './parallel.js';
|
|
11
12
|
const planChunks = (sizes) => {
|
|
12
13
|
let offset = 0;
|
|
@@ -67,6 +68,21 @@ const readExact = async (source, chunk) => {
|
|
|
67
68
|
}
|
|
68
69
|
return bytes;
|
|
69
70
|
};
|
|
71
|
+
const assertBatchChunks = (bytes, indexes, manifest, sha256Hex) => {
|
|
72
|
+
let offset = 0;
|
|
73
|
+
for (const index of indexes) {
|
|
74
|
+
const expected = manifest.chunks[index];
|
|
75
|
+
if (!expected)
|
|
76
|
+
throw new Error(`content manifest has no chunk ${index}`);
|
|
77
|
+
const chunk = bytes.subarray(offset, offset + expected.bytes);
|
|
78
|
+
if (chunk.byteLength !== expected.bytes || sha256Hex(chunk) !== expected.sha256) {
|
|
79
|
+
throw new Error(`upload source changed while reading chunk ${index}`);
|
|
80
|
+
}
|
|
81
|
+
offset += expected.bytes;
|
|
82
|
+
}
|
|
83
|
+
if (offset !== bytes.byteLength)
|
|
84
|
+
throw new Error('upload source returned the wrong byte count for a file batch');
|
|
85
|
+
};
|
|
70
86
|
/** Upload one immutable artifact, sending only authority-declared gaps. */
|
|
71
87
|
export async function upload(artifact, source, ports, options = {}) {
|
|
72
88
|
assertTransferArtifact(artifact);
|
|
@@ -111,16 +127,25 @@ export async function upload(artifact, source, ports, options = {}) {
|
|
|
111
127
|
throw new Error(`upload source does not match ${artifact.contentHash}`);
|
|
112
128
|
}
|
|
113
129
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
await
|
|
123
|
-
|
|
130
|
+
if (artifact.kind === 'file' && ports.putChunkBatch) {
|
|
131
|
+
await parallelForEach(planChunkBatches(missing, manifest), options.concurrency ?? DEFAULT_TRANSFER_CONCURRENCY, async (batch) => {
|
|
132
|
+
const bytes = await readExact(source, { index: batch.indexes[0], offset: batch.offset, bytes: batch.bytes });
|
|
133
|
+
assertBatchChunks(bytes, batch.indexes, manifest, ports.sha256Hex);
|
|
134
|
+
await ports.putChunkBatch(artifact, manifest, batch.indexes, bytes);
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
await parallelForEach(missing, options.concurrency ?? DEFAULT_TRANSFER_CONCURRENCY, async (index) => {
|
|
139
|
+
const chunk = plan[index];
|
|
140
|
+
if (chunk === undefined)
|
|
141
|
+
throw new Error('upload authority returned an unknown chunk index');
|
|
142
|
+
const bytes = await readExact(source, chunk);
|
|
143
|
+
if (ports.sha256Hex(bytes) !== manifest.chunks[index]?.sha256) {
|
|
144
|
+
throw new Error(`upload source changed while reading chunk ${index}`);
|
|
145
|
+
}
|
|
146
|
+
await ports.putChunk(artifact, manifest, index, bytes);
|
|
147
|
+
});
|
|
148
|
+
}
|
|
124
149
|
await ports.putManifest(artifact, manifest);
|
|
125
150
|
return {
|
|
126
151
|
artifact,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"upload.js","sourceRoot":"","sources":["../../src/transfer/upload.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,UAAU,EACV,kBAAkB,GAEnB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,sBAAsB,EAAyB,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,4BAA4B,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"upload.js","sourceRoot":"","sources":["../../src/transfer/upload.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EACL,oBAAoB,EACpB,uBAAuB,EACvB,UAAU,EACV,kBAAkB,GAEnB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,sBAAsB,EAAyB,MAAM,gBAAgB,CAAC;AAC/E,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AA0D9E,MAAM,UAAU,GAAG,CAAC,KAAwB,EAA2B,EAAE;IACvE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,MAAM,KAAK,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAC7C,MAAM,IAAI,IAAI,CAAC;QACf,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,UAAkB,EAAE,SAAiB,EAAY,EAAE;IACrE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,SAAS,GAAG,UAAU,EAAE,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,SAAS,EAAE,CAAC;QACvE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF;;;gFAGgF;AAChF,MAAM,iBAAiB,GAAG,CAAC,QAA0B,EAAE,UAAkB,EAAqB,EAAE;IAC9F,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC;QACtB,KAAK,MAAM;YACT,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;QAChC,KAAK,MAAM;YACT,IAAI,UAAU,GAAG,uBAAuB;gBAAE,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC1G,OAAO,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QAC9C,KAAK,KAAK;YACR,OAAO,UAAU,CAAC,UAAU,EAAE,uBAAuB,CAAC,CAAC;IAC3D,CAAC;AACH,CAAC,CAAC;AAEF;;;0CAG0C;AAC1C,MAAM,UAAU,oBAAoB,CAClC,QAA0B,EAC1B,KAAiB,EACjB,SAAoB;IAEpB,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACjC,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;QACjE,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,IAAI,IAAI,CAAC;IACjB,CAAC;IACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACjE,IAAI,CAAC,OAAO,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAChD,OAAO,OAAO,CAAC,KAAK,CAAC;AACvB,CAAC;AAED,MAAM,SAAS,GAAG,KAAK,EAAE,MAAoB,EAAE,KAAmB,EAAuB,EAAE;IACzF,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAC3D,IAAI,CAAC,CAAC,KAAK,YAAY,UAAU,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,yDAAyD,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CACxB,KAAiB,EACjB,OAA0B,EAC1B,QAAyB,EACzB,SAAoB,EACd,EAAE;IACR,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,KAAK,EAAE,CAAC,CAAC;QACzE,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC9D,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAChF,MAAM,IAAI,KAAK,CAAC,6CAA6C,KAAK,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC;IAC3B,CAAC;IACD,IAAI,MAAM,KAAK,KAAK,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;AACnH,CAAC,CAAC;AAEF,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,QAA0B,EAC1B,MAAoB,EACpB,KAAkB,EAClB,UAAyB,EAAE;IAE3B,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACjC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QAC5D,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,MAAM,CAAC,gBAAgB,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,aAAa,GAAG,iBAAiB,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAChE,IAAI,IAA6B,CAAC;IAClC,IAAI,QAAyB,CAAC;IAC9B,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,KAAK,MAAM,CAAC,KAAK;eAChD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,aAAa,CAAC,KAAK,CAAC,CAAC;eAChF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,aAAa,CAAC,MAAM,EAAE,CAAC;YACzD,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;QACxB,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,UAAU,CAAC,aAAa,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,OAAO,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAChD,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC;IAC3B,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,KAAK,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;IACnE,IAAI,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM;WACvC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;IAC7E,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;QAC3D,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC;QACpD,MAAM,eAAe,CACnB,gBAAgB,CAAC,OAAO,EAAE,QAAQ,CAAC,EACnC,OAAO,CAAC,WAAW,IAAI,4BAA4B,EACnD,KAAK,EAAE,KAAK,EAAE,EAAE;YACd,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAE,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;YAC9G,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YACnE,MAAM,KAAK,CAAC,aAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACvE,CAAC,CACF,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,MAAM,eAAe,CAAC,OAAO,EAAE,OAAO,CAAC,WAAW,IAAI,4BAA4B,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAClG,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;YAC7F,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YAC7C,IAAI,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;gBAC9D,MAAM,IAAI,KAAK,CAAC,6CAA6C,KAAK,EAAE,CAAC,CAAC;YACxE,CAAC;YACD,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,CAAC,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC5C,OAAO;QACL,QAAQ;QACR,QAAQ;QACR,cAAc,EAAE,OAAO,CAAC,MAAM;QAC9B,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM;KACtD,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@amalgm/live",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.18",
|
|
4
4
|
"description": "Portable realtime SDK for Amalgm — wire contracts, mutation discipline, and delivery state machines behind host-injected adapters.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "UNLICENSED",
|