@statewalker/webrun-content-transfer 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -28
- package/package.json +9 -5
package/README.md
CHANGED
|
@@ -1,20 +1,48 @@
|
|
|
1
1
|
# @statewalker/webrun-content-transfer
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
## What it is
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
Moves objects between two content stores (`ContentStore` from `@statewalker/webrun-content-store`)
|
|
6
|
+
by sending only the chunks the destination does not have. Either store can be remote: one side
|
|
7
|
+
serves its store as a `Duplex` from `@statewalker/webrun-streams`, the other wraps a `Duplex` as a
|
|
8
|
+
store proxy.
|
|
6
9
|
|
|
7
|
-
|
|
10
|
+
## Why it exists
|
|
8
11
|
|
|
9
|
-
|
|
12
|
+
Chunked storage only saves bandwidth if the transfer asks first which chunks are missing. This
|
|
13
|
+
package does that negotiation in batches, sends the missing chunks through a bounded worker pool,
|
|
14
|
+
re-hashes each one on arrival, and registers the object on the destination. Because it talks to
|
|
15
|
+
stores through the `ContentStore` interface and to peers through a plain `Duplex`, it works the
|
|
16
|
+
same in one process, over WebSocket, WebRTC or HTTP streams. `chunkTransfer` plugs it into
|
|
17
|
+
`@statewalker/webrun-files-sync` so file copies move only changed chunks.
|
|
10
18
|
|
|
11
|
-
##
|
|
19
|
+
## How to use
|
|
12
20
|
|
|
13
21
|
```bash
|
|
14
|
-
pnpm add @statewalker/webrun-content-transfer
|
|
22
|
+
pnpm add @statewalker/webrun-content-transfer @statewalker/webrun-content-store
|
|
15
23
|
```
|
|
16
24
|
|
|
17
|
-
|
|
25
|
+
| Import | Contents |
|
|
26
|
+
| --- | --- |
|
|
27
|
+
| `@statewalker/webrun-content-transfer` | `transfer`, `remoteStore`, `serveStore`, `chunkTransfer`, and the types `TransferOptions`, `TransferLimits`, `TransferCheckpoint`, `TransferEvent`, `HashContent`, `AssemblingStore`, `Capabilities`, `FileTransfer`, `FileSyncAction`. ESM, runs in browsers, Node and workers. |
|
|
28
|
+
|
|
29
|
+
- `transfer(objectIds, from, to, opts?)`: async generator of events: `resumed`, `negotiated`
|
|
30
|
+
(`objects`, `chunks`, `missing`), `chunk-sent` (`chunkId`, `size`), `object-done` (`objectId`).
|
|
31
|
+
Drain it to run the transfer.
|
|
32
|
+
- `serveStore(store)`: returns a `Duplex` handler that serves a `ContentStore`.
|
|
33
|
+
- `remoteStore(call)`: wraps a `Duplex` as an `AssemblingStore` (a `ContentStore` with
|
|
34
|
+
`putManifest` and `capabilities`).
|
|
35
|
+
- `chunkTransfer(local, remote)`: a `Transfer` for `@statewalker/webrun-files-sync`. For each
|
|
36
|
+
copied file it stores the source in `local`, transfers missing chunks to `remote`, and writes the
|
|
37
|
+
file from `remote` to the destination.
|
|
38
|
+
|
|
39
|
+
`TransferOptions`: `hashContent?` (enables per-chunk verification), `limits?`
|
|
40
|
+
(`concurrency`, default 4; `maxBufferedBytes`, default 8 MiB; `batchSize`, default 256), and
|
|
41
|
+
`checkpoint?`.
|
|
42
|
+
|
|
43
|
+
## Examples
|
|
44
|
+
|
|
45
|
+
Between two local stores:
|
|
18
46
|
|
|
19
47
|
```typescript
|
|
20
48
|
import { createHash } from "node:crypto";
|
|
@@ -27,43 +55,109 @@ async function sha256(bytes: AsyncIterable<Uint8Array>): Promise<string> {
|
|
|
27
55
|
for await (const chunk of bytes) h.update(chunk);
|
|
28
56
|
return `sha256:${h.digest("hex")}`;
|
|
29
57
|
}
|
|
58
|
+
async function* streamOf(b: Uint8Array) {
|
|
59
|
+
yield b;
|
|
60
|
+
}
|
|
30
61
|
const mkStore = () =>
|
|
31
62
|
createContentStore({ chunks: memBlobStore(), manifests: memBlobStore(), hashContent: sha256 });
|
|
32
63
|
|
|
33
64
|
const from = mkStore();
|
|
34
65
|
const to = mkStore();
|
|
35
|
-
|
|
36
|
-
const { id } = await from.put(streamOf(payload));
|
|
66
|
+
const { id } = await from.put(streamOf(new Uint8Array(100_000).map((_, i) => (i * 7919) % 251)));
|
|
37
67
|
|
|
38
|
-
// Move only the chunks `to` is missing; re-hash-verify each; emit progress.
|
|
39
68
|
for await (const event of transfer([id], from, to, { hashContent: sha256 })) {
|
|
40
|
-
if (event.type === "
|
|
69
|
+
if (event.type === "negotiated") console.log(`${event.missing} of ${event.chunks} chunks to send`);
|
|
41
70
|
}
|
|
42
|
-
// to.read(id) now
|
|
71
|
+
// to.read(id) now yields the same bytes.
|
|
43
72
|
```
|
|
44
73
|
|
|
45
|
-
|
|
74
|
+
To a store behind a duplex (here served in the same process):
|
|
46
75
|
|
|
47
76
|
```typescript
|
|
48
|
-
import { remoteStore, serveStore } from "@statewalker/webrun-content-transfer";
|
|
77
|
+
import { remoteStore, serveStore, transfer } from "@statewalker/webrun-content-transfer";
|
|
78
|
+
|
|
79
|
+
const remote = remoteStore(serveStore(mkStore()));
|
|
80
|
+
for await (const _ of transfer([id], from, remote, { hashContent: sha256 })) {
|
|
81
|
+
}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
In a real deployment, the server passes `serveStore(store)` to its `webrun-streams` transport and
|
|
85
|
+
the client passes the transport's `Duplex` to `remoteStore`.
|
|
86
|
+
|
|
87
|
+
Inside a file sync:
|
|
88
|
+
|
|
89
|
+
```typescript
|
|
90
|
+
import { execute, plan } from "@statewalker/webrun-files-sync";
|
|
91
|
+
import { chunkTransfer } from "@statewalker/webrun-content-transfer";
|
|
92
|
+
|
|
93
|
+
const opts = { hashContent: sha256, transfer: chunkTransfer(mkStore(), mkStore()) };
|
|
94
|
+
for await (const _ of execute(await plan(a, b, "copy", opts), a, b, opts)) {
|
|
95
|
+
}
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## Internals
|
|
99
|
+
|
|
100
|
+
### One transfer, four steps
|
|
49
101
|
|
|
50
|
-
const handler = serveStore(serverStore); // server: Duplex handler over a ContentStore
|
|
51
|
-
const remote = remoteStore(clientDuplex); // client: an AssemblingStore proxy
|
|
52
|
-
await drain(transfer([id], from, remote, { hashContent: sha256 })); // upload to the remote
|
|
53
102
|
```
|
|
103
|
+
from.getManifest(id) for each object ──> union of chunk ids
|
|
104
|
+
│
|
|
105
|
+
▼
|
|
106
|
+
to.hasChunks(batch of 256) ──> missing chunk list (negotiated)
|
|
107
|
+
│
|
|
108
|
+
▼
|
|
109
|
+
N workers: from.getChunk ─> re-hash ─> to.putChunk (chunk-sent)
|
|
110
|
+
│
|
|
111
|
+
▼
|
|
112
|
+
register each manifest on the destination (object-done)
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Negotiation is batched, so N chunk ids cost `ceil(N / batchSize)` requests instead of N. The worker
|
|
116
|
+
pool is capped by `concurrency` and by `maxBufferedBytes` of chunk data in flight.
|
|
117
|
+
|
|
118
|
+
### Assembling on the remote side costs only the manifest
|
|
119
|
+
|
|
120
|
+
When the destination is a `remoteStore`, the last step sends the manifest with `putManifest`; the
|
|
121
|
+
server assembles the object from chunks it already holds. Against a local store, the object is
|
|
122
|
+
registered from its chunks directly.
|
|
123
|
+
|
|
124
|
+
### The wire format
|
|
125
|
+
|
|
126
|
+
Each call is one request and one response. A message is a 4-byte big-endian header length, a JSON
|
|
127
|
+
header, then the body bytes. Operations: `capabilities`, `getManifest`, `hasChunks`, `getChunk`,
|
|
128
|
+
`putChunk`, `putManifest`. The server reports protocol `content-transfer/1`.
|
|
129
|
+
|
|
130
|
+
### Resuming is re-running
|
|
131
|
+
|
|
132
|
+
The destination's own chunk set is the resume point: calling `transfer` again re-negotiates and
|
|
133
|
+
sends only what is still missing. A `checkpoint` is optional; when given, the generator first
|
|
134
|
+
emits `{ type: "resumed", done }` with the number of chunks in `checkpoint.done`. The source is
|
|
135
|
+
always re-checked for every requested object.
|
|
136
|
+
|
|
137
|
+
### What fails, and how it looks
|
|
54
138
|
|
|
55
|
-
|
|
139
|
+
- Without `hashContent` in the options, chunks are **not** verified. With it, a corrupted chunk
|
|
140
|
+
stops the transfer with `content-transfer: integrity check failed for chunk <id> (re-hash <hash>)`
|
|
141
|
+
and the object is not registered. Both stores must use the same hash.
|
|
142
|
+
- A requested object the source does not have:
|
|
143
|
+
`content-transfer: source is missing object <id>`.
|
|
144
|
+
- A `remoteStore` proxy supports only the chunk-level and manifest operations. Calling `put`,
|
|
145
|
+
`has`, `read`, `remove` or `gc` on it throws
|
|
146
|
+
`content-transfer: remote proxy does not support "<name>"`.
|
|
147
|
+
- A peer that closes the stream mid-message: `content-transfer protocol: unexpected end of stream`.
|
|
148
|
+
- `chunkTransfer` handles only `copy` and `update` actions; anything else throws
|
|
149
|
+
`content-transfer: chunkTransfer only handles copy/update, got <kind>`.
|
|
150
|
+
- `chunkTransfer` writes the destination file by reading the object back from `remote`, so
|
|
151
|
+
`remote` must be a full `ContentStore`. A `remoteStore` proxy as `remote` fails with
|
|
152
|
+
`content-transfer: remote proxy does not support "read"`. `chunkTransfer` also calls `transfer`
|
|
153
|
+
without `hashContent`, so its chunks are not re-hashed.
|
|
56
154
|
|
|
57
|
-
|
|
58
|
-
- **`remoteStore(call: Duplex): AssemblingStore`** — client-side proxy `ContentStore` over a duplex (adds `putManifest` / `capabilities`).
|
|
59
|
-
- **`serveStore(store): Duplex`** — server-side handler exposing a `ContentStore` over a duplex.
|
|
60
|
-
- **`chunkTransfer(local, remote): FileTransfer`** — the `files-sync`-shaped `Transfer` adapter.
|
|
155
|
+
### Dependencies
|
|
61
156
|
|
|
62
|
-
`
|
|
157
|
+
`@statewalker/webrun-content-store` (store types), `@statewalker/webrun-streams` (the `Duplex`
|
|
158
|
+
type) and `@statewalker/webrun-files` (the `FilesApi` type used by `chunkTransfer`). All imports
|
|
159
|
+
are type-only; the package has no runtime dependencies.
|
|
63
160
|
|
|
64
|
-
##
|
|
161
|
+
## License
|
|
65
162
|
|
|
66
|
-
|
|
67
|
-
- **Resumable + bounded.** A serializable `TransferCheckpoint` (done / pending chunk sets) resumes an interrupted transfer after re-verifying the source still holds every object; the pipeline is bounded by `TransferLimits` + `maxBufferedBytes` backpressure.
|
|
68
|
-
- **Symmetric wire.** `hasChunks` negotiation is batched; the `remoteStore` proxy can `putManifest` so assemble costs only the manifest, not the object's bytes.
|
|
69
|
-
- Built red/green TDD.
|
|
163
|
+
MIT
|
package/package.json
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@statewalker/webrun-content-transfer",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"description": "Resumable, chunk-aware content transfer over webrun-streams",
|
|
5
|
+
"bugs": {
|
|
6
|
+
"url": "https://github.com/statewalker/webrun-sync/issues"
|
|
7
|
+
},
|
|
8
|
+
"homepage": "https://github.com/statewalker/webrun-sync/tree/main/packages/webrun-content-transfer#readme",
|
|
5
9
|
"license": "MIT",
|
|
6
10
|
"repository": {
|
|
7
11
|
"type": "git",
|
|
8
|
-
"url": "https://github.com/statewalker/webrun-
|
|
12
|
+
"url": "git+https://github.com/statewalker/webrun-sync.git",
|
|
9
13
|
"directory": "packages/webrun-content-transfer"
|
|
10
14
|
},
|
|
11
15
|
"private": false,
|
|
@@ -29,7 +33,7 @@
|
|
|
29
33
|
"dependencies": {
|
|
30
34
|
"@statewalker/webrun-files": "^0.10.0",
|
|
31
35
|
"@statewalker/webrun-streams": "^0.2.0",
|
|
32
|
-
"@statewalker/webrun-content-store": "^0.2.
|
|
36
|
+
"@statewalker/webrun-content-store": "^0.2.3"
|
|
33
37
|
},
|
|
34
38
|
"devDependencies": {
|
|
35
39
|
"@statewalker/webrun-files-mem": "^0.10.0",
|
|
@@ -37,8 +41,8 @@
|
|
|
37
41
|
"vitest": "^5.0.3",
|
|
38
42
|
"typescript": "^7.0.2",
|
|
39
43
|
"@types/node": "^26.6.4",
|
|
40
|
-
"@statewalker/webrun-storage": "^0.2.
|
|
41
|
-
"@statewalker/webrun-files-sync": "^0.2.
|
|
44
|
+
"@statewalker/webrun-storage": "^0.2.2",
|
|
45
|
+
"@statewalker/webrun-files-sync": "^0.2.3"
|
|
42
46
|
},
|
|
43
47
|
"scripts": {
|
|
44
48
|
"build": "rm -rf dist && rolldown -c && tsc --emitDeclarationOnly --declaration",
|