@freqhole/reliquary 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/automerge/acl-filtering.d.ts +87 -0
- package/dist/automerge/acl-filtering.d.ts.map +1 -0
- package/dist/automerge/acl-filtering.js +138 -0
- package/dist/automerge/acl-filtering.test.d.ts +2 -0
- package/dist/automerge/acl-filtering.test.d.ts.map +1 -0
- package/dist/automerge/acl-filtering.test.js +302 -0
- package/dist/automerge/index.d.ts +5 -0
- package/dist/automerge/index.d.ts.map +1 -0
- package/dist/automerge/index.js +7 -0
- package/dist/automerge/iroh-network-adapter.d.ts +178 -0
- package/dist/automerge/iroh-network-adapter.d.ts.map +1 -0
- package/dist/automerge/iroh-network-adapter.js +691 -0
- package/dist/automerge/iroh-network-adapter.test.d.ts +2 -0
- package/dist/automerge/iroh-network-adapter.test.d.ts.map +1 -0
- package/dist/automerge/iroh-network-adapter.test.js +818 -0
- package/dist/automerge/types.d.ts +66 -0
- package/dist/automerge/types.d.ts.map +1 -0
- package/dist/automerge/types.js +10 -0
- package/dist/blobs/bytes-backend.d.ts +41 -0
- package/dist/blobs/bytes-backend.d.ts.map +1 -0
- package/dist/blobs/bytes-backend.js +252 -0
- package/dist/blobs/bytes-backend.test.d.ts +2 -0
- package/dist/blobs/bytes-backend.test.d.ts.map +1 -0
- package/dist/blobs/bytes-backend.test.js +241 -0
- package/dist/blobs/db.d.ts +8 -0
- package/dist/blobs/db.d.ts.map +1 -0
- package/dist/blobs/db.js +141 -0
- package/dist/blobs/db.test.d.ts +2 -0
- package/dist/blobs/db.test.d.ts.map +1 -0
- package/dist/blobs/db.test.js +120 -0
- package/dist/blobs/index.d.ts +5 -0
- package/dist/blobs/index.d.ts.map +1 -0
- package/dist/blobs/index.js +2 -0
- package/dist/blobs/store.d.ts +47 -0
- package/dist/blobs/store.d.ts.map +1 -0
- package/dist/blobs/store.js +263 -0
- package/dist/blobs/store.test.d.ts +2 -0
- package/dist/blobs/store.test.d.ts.map +1 -0
- package/dist/blobs/store.test.js +377 -0
- package/dist/blobs/types.d.ts +55 -0
- package/dist/blobs/types.d.ts.map +1 -0
- package/dist/blobs/types.js +8 -0
- package/dist/ensure/client.d.ts +19 -0
- package/dist/ensure/client.d.ts.map +1 -0
- package/dist/ensure/client.js +74 -0
- package/dist/ensure/client.test.d.ts +2 -0
- package/dist/ensure/client.test.d.ts.map +1 -0
- package/dist/ensure/client.test.js +133 -0
- package/dist/ensure/index.d.ts +4 -0
- package/dist/ensure/index.d.ts.map +1 -0
- package/dist/ensure/index.js +4 -0
- package/dist/ensure/responder.d.ts +20 -0
- package/dist/ensure/responder.d.ts.map +1 -0
- package/dist/ensure/responder.js +100 -0
- package/dist/ensure/responder.test.d.ts +2 -0
- package/dist/ensure/responder.test.d.ts.map +1 -0
- package/dist/ensure/responder.test.js +191 -0
- package/dist/ensure/types.d.ts +19 -0
- package/dist/ensure/types.d.ts.map +1 -0
- package/dist/ensure/types.js +4 -0
- package/dist/ensure/types.test.d.ts +2 -0
- package/dist/ensure/types.test.d.ts.map +1 -0
- package/dist/ensure/types.test.js +81 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.test.d.ts +2 -0
- package/dist/index.test.d.ts.map +1 -0
- package/dist/index.test.js +7 -0
- package/dist/solid/blob-url.d.ts +10 -0
- package/dist/solid/blob-url.d.ts.map +1 -0
- package/dist/solid/blob-url.js +34 -0
- package/dist/solid/blob-url.test.d.ts +2 -0
- package/dist/solid/blob-url.test.d.ts.map +1 -0
- package/dist/solid/blob-url.test.js +62 -0
- package/dist/solid/connection-summary.d.ts +25 -0
- package/dist/solid/connection-summary.d.ts.map +1 -0
- package/dist/solid/connection-summary.js +29 -0
- package/dist/solid/connection-summary.test.d.ts +2 -0
- package/dist/solid/connection-summary.test.d.ts.map +1 -0
- package/dist/solid/connection-summary.test.js +57 -0
- package/dist/solid/doc-store.d.ts +29 -0
- package/dist/solid/doc-store.d.ts.map +1 -0
- package/dist/solid/doc-store.js +79 -0
- package/dist/solid/doc-store.test.d.ts +2 -0
- package/dist/solid/doc-store.test.d.ts.map +1 -0
- package/dist/solid/doc-store.test.js +234 -0
- package/dist/solid/index.d.ts +4 -0
- package/dist/solid/index.d.ts.map +1 -0
- package/dist/solid/index.js +9 -0
- package/dist/solid/transfer-progress.d.ts +21 -0
- package/dist/solid/transfer-progress.d.ts.map +1 -0
- package/dist/solid/transfer-progress.js +32 -0
- package/dist/solid/transfer-progress.test.d.ts +2 -0
- package/dist/solid/transfer-progress.test.d.ts.map +1 -0
- package/dist/solid/transfer-progress.test.js +76 -0
- package/dist/testing/blob-fixtures.d.ts +19 -0
- package/dist/testing/blob-fixtures.d.ts.map +1 -0
- package/dist/testing/blob-fixtures.js +46 -0
- package/dist/testing/blob-fixtures.test.d.ts +2 -0
- package/dist/testing/blob-fixtures.test.d.ts.map +1 -0
- package/dist/testing/blob-fixtures.test.js +46 -0
- package/dist/testing/idb-harness.d.ts +10 -0
- package/dist/testing/idb-harness.d.ts.map +1 -0
- package/dist/testing/idb-harness.js +17 -0
- package/dist/testing/idb-harness.test.d.ts +2 -0
- package/dist/testing/idb-harness.test.d.ts.map +1 -0
- package/dist/testing/idb-harness.test.js +48 -0
- package/dist/testing/index.d.ts +7 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +11 -0
- package/dist/testing/media.d.ts +15 -0
- package/dist/testing/media.d.ts.map +1 -0
- package/dist/testing/media.js +37 -0
- package/dist/testing/media.test.d.ts +2 -0
- package/dist/testing/media.test.d.ts.map +1 -0
- package/dist/testing/media.test.js +47 -0
- package/dist/testing/mock-blob-fetch.d.ts +33 -0
- package/dist/testing/mock-blob-fetch.d.ts.map +1 -0
- package/dist/testing/mock-blob-fetch.js +39 -0
- package/dist/testing/mock-blob-fetch.test.d.ts +2 -0
- package/dist/testing/mock-blob-fetch.test.d.ts.map +1 -0
- package/dist/testing/mock-blob-fetch.test.js +65 -0
- package/dist/testing/mock-node.d.ts +36 -0
- package/dist/testing/mock-node.d.ts.map +1 -0
- package/dist/testing/mock-node.js +97 -0
- package/dist/testing/mock-node.test.d.ts +2 -0
- package/dist/testing/mock-node.test.d.ts.map +1 -0
- package/dist/testing/mock-node.test.js +66 -0
- package/dist/testing/streams.d.ts +68 -0
- package/dist/testing/streams.d.ts.map +1 -0
- package/dist/testing/streams.js +99 -0
- package/dist/testing/streams.test.d.ts +2 -0
- package/dist/testing/streams.test.d.ts.map +1 -0
- package/dist/testing/streams.test.js +82 -0
- package/dist/transfer/cancellation.d.ts +8 -0
- package/dist/transfer/cancellation.d.ts.map +1 -0
- package/dist/transfer/cancellation.js +19 -0
- package/dist/transfer/index.d.ts +8 -0
- package/dist/transfer/index.d.ts.map +1 -0
- package/dist/transfer/index.js +4 -0
- package/dist/transfer/prefetch.d.ts +38 -0
- package/dist/transfer/prefetch.d.ts.map +1 -0
- package/dist/transfer/prefetch.js +61 -0
- package/dist/transfer/prefetch.test.d.ts +2 -0
- package/dist/transfer/prefetch.test.d.ts.map +1 -0
- package/dist/transfer/prefetch.test.js +136 -0
- package/dist/transfer/serve.d.ts +66 -0
- package/dist/transfer/serve.d.ts.map +1 -0
- package/dist/transfer/serve.js +126 -0
- package/dist/transfer/serve.test.d.ts +2 -0
- package/dist/transfer/serve.test.d.ts.map +1 -0
- package/dist/transfer/serve.test.js +101 -0
- package/dist/transfer/snatch.d.ts +39 -0
- package/dist/transfer/snatch.d.ts.map +1 -0
- package/dist/transfer/snatch.js +289 -0
- package/dist/transfer/snatch.test.d.ts +2 -0
- package/dist/transfer/snatch.test.d.ts.map +1 -0
- package/dist/transfer/snatch.test.js +337 -0
- package/dist/transfer/types.d.ts +85 -0
- package/dist/transfer/types.d.ts.map +1 -0
- package/dist/transfer/types.js +15 -0
- package/dist/utils/hash.d.ts +13 -0
- package/dist/utils/hash.d.ts.map +1 -0
- package/dist/utils/hash.js +31 -0
- package/dist/utils/hash.test.d.ts +2 -0
- package/dist/utils/hash.test.d.ts.map +1 -0
- package/dist/utils/hash.test.js +56 -0
- package/dist/utils/image-utils.d.ts +28 -0
- package/dist/utils/image-utils.d.ts.map +1 -0
- package/dist/utils/image-utils.js +113 -0
- package/dist/utils/image-utils.test.d.ts +2 -0
- package/dist/utils/image-utils.test.d.ts.map +1 -0
- package/dist/utils/image-utils.test.js +181 -0
- package/dist/utils/index.d.ts +4 -0
- package/dist/utils/index.d.ts.map +1 -0
- package/dist/utils/index.js +3 -0
- package/dist/utils/log.d.ts +22 -0
- package/dist/utils/log.d.ts.map +1 -0
- package/dist/utils/log.js +100 -0
- package/dist/utils/log.test.d.ts +2 -0
- package/dist/utils/log.test.d.ts.map +1 -0
- package/dist/utils/log.test.js +113 -0
- package/dist/worker/blob-worker-client.d.ts +118 -0
- package/dist/worker/blob-worker-client.d.ts.map +1 -0
- package/dist/worker/blob-worker-client.js +361 -0
- package/dist/worker/blob-worker-client.test.d.ts +2 -0
- package/dist/worker/blob-worker-client.test.d.ts.map +1 -0
- package/dist/worker/blob-worker-client.test.js +137 -0
- package/dist/worker/blob-worker-logic.d.ts +120 -0
- package/dist/worker/blob-worker-logic.d.ts.map +1 -0
- package/dist/worker/blob-worker-logic.js +368 -0
- package/dist/worker/blob-worker-logic.test.d.ts +2 -0
- package/dist/worker/blob-worker-logic.test.d.ts.map +1 -0
- package/dist/worker/blob-worker-logic.test.js +191 -0
- package/dist/worker/blob-worker.d.ts +21 -0
- package/dist/worker/blob-worker.d.ts.map +1 -0
- package/dist/worker/blob-worker.js +44 -0
- package/dist/worker/index.d.ts +12 -0
- package/dist/worker/index.d.ts.map +1 -0
- package/dist/worker/index.js +6 -0
- package/dist/worker/midden-blake3.d.ts +23 -0
- package/dist/worker/midden-blake3.d.ts.map +1 -0
- package/dist/worker/midden-blake3.js +47 -0
- package/dist/worker/midden-blake3.test.d.ts +2 -0
- package/dist/worker/midden-blake3.test.d.ts.map +1 -0
- package/dist/worker/midden-blake3.test.js +25 -0
- package/dist/worker/midden-not-bundled.stub.d.ts +1 -0
- package/dist/worker/midden-not-bundled.stub.d.ts.map +1 -0
- package/dist/worker/midden-not-bundled.stub.js +10 -0
- package/dist/worker/midden-stub.d.ts +10 -0
- package/dist/worker/midden-stub.d.ts.map +1 -0
- package/dist/worker/midden-stub.js +31 -0
- package/dist/worker/midden-stub.test.d.ts +2 -0
- package/dist/worker/midden-stub.test.d.ts.map +1 -0
- package/dist/worker/midden-stub.test.js +25 -0
- package/dist/worker/midden-worker-client.d.ts +103 -0
- package/dist/worker/midden-worker-client.d.ts.map +1 -0
- package/dist/worker/midden-worker-client.js +246 -0
- package/dist/worker/midden-worker-client.test.d.ts +2 -0
- package/dist/worker/midden-worker-client.test.d.ts.map +1 -0
- package/dist/worker/midden-worker-client.test.js +151 -0
- package/dist/worker/midden-worker-contract.d.ts +87 -0
- package/dist/worker/midden-worker-contract.d.ts.map +1 -0
- package/dist/worker/midden-worker-contract.js +30 -0
- package/package.json +90 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { createPrefetcher } from "./prefetch.js";
|
|
3
|
+
function item(id, cost) {
|
|
4
|
+
return { id, cost };
|
|
5
|
+
}
|
|
6
|
+
describe("Prefetcher", () => {
|
|
7
|
+
it("selects a budget-limited prefix and fetches every selected item", async () => {
|
|
8
|
+
const prefetcher = createPrefetcher();
|
|
9
|
+
const fetched = [];
|
|
10
|
+
const settled = [];
|
|
11
|
+
let resolveAll;
|
|
12
|
+
const done = new Promise((resolve) => {
|
|
13
|
+
resolveAll = resolve;
|
|
14
|
+
});
|
|
15
|
+
const items = [item("a", 10), item("b", 10), item("c", 10), item("d", 10)];
|
|
16
|
+
prefetcher.run(items, {
|
|
17
|
+
budget: 25,
|
|
18
|
+
costOf: (it) => it.cost,
|
|
19
|
+
fetchItem: async (it) => {
|
|
20
|
+
fetched.push(it.id);
|
|
21
|
+
},
|
|
22
|
+
onSettled: (it) => {
|
|
23
|
+
settled.push(it.id);
|
|
24
|
+
if (settled.length === 3)
|
|
25
|
+
resolveAll();
|
|
26
|
+
},
|
|
27
|
+
});
|
|
28
|
+
await done;
|
|
29
|
+
// budget 25: a (15 left), b (5 left), c (-5, still selected since
|
|
30
|
+
// budget was > 0 before subtracting) - the budget check happens in
|
|
31
|
+
// the loop condition, the subtraction after, so the item that
|
|
32
|
+
// crosses zero is still included.
|
|
33
|
+
expect(fetched.sort()).toEqual(["a", "b", "c"]);
|
|
34
|
+
expect(settled.sort()).toEqual(["a", "b", "c"]);
|
|
35
|
+
});
|
|
36
|
+
it("fires onPending for every selected item before fetching starts", async () => {
|
|
37
|
+
const prefetcher = createPrefetcher();
|
|
38
|
+
const pending = [];
|
|
39
|
+
let resolveAll;
|
|
40
|
+
const done = new Promise((resolve) => {
|
|
41
|
+
resolveAll = resolve;
|
|
42
|
+
});
|
|
43
|
+
prefetcher.run([item("a", 5), item("b", 5)], {
|
|
44
|
+
budget: 100,
|
|
45
|
+
costOf: (it) => it.cost,
|
|
46
|
+
fetchItem: async () => { },
|
|
47
|
+
onPending: (it) => pending.push(it.id),
|
|
48
|
+
onSettled: () => {
|
|
49
|
+
if (pending.length === 2)
|
|
50
|
+
resolveAll();
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
await done;
|
|
54
|
+
expect(pending).toEqual(["a", "b"]);
|
|
55
|
+
});
|
|
56
|
+
it("stops selecting once the budget runs out", async () => {
|
|
57
|
+
const prefetcher = createPrefetcher();
|
|
58
|
+
const pending = [];
|
|
59
|
+
let resolveAll;
|
|
60
|
+
const done = new Promise((resolve) => {
|
|
61
|
+
resolveAll = resolve;
|
|
62
|
+
});
|
|
63
|
+
prefetcher.run([item("a", 100), item("b", 5), item("c", 5)], {
|
|
64
|
+
budget: 50,
|
|
65
|
+
costOf: (it) => it.cost,
|
|
66
|
+
fetchItem: async () => { },
|
|
67
|
+
onPending: (it) => pending.push(it.id),
|
|
68
|
+
onSettled: () => resolveAll(),
|
|
69
|
+
});
|
|
70
|
+
await done;
|
|
71
|
+
// "a" alone exhausts the budget (100 > 50 remaining after selection),
|
|
72
|
+
// so "b"/"c" are never even considered.
|
|
73
|
+
expect(pending).toEqual(["a"]);
|
|
74
|
+
});
|
|
75
|
+
it("supersedes an in-flight run when run() is called again", async () => {
|
|
76
|
+
const prefetcher = createPrefetcher();
|
|
77
|
+
const fetchOrder = [];
|
|
78
|
+
let releaseFirst;
|
|
79
|
+
const firstGate = new Promise((resolve) => {
|
|
80
|
+
releaseFirst = resolve;
|
|
81
|
+
});
|
|
82
|
+
prefetcher.run([item("a", 1), item("b", 1)], {
|
|
83
|
+
budget: 10,
|
|
84
|
+
costOf: (it) => it.cost,
|
|
85
|
+
concurrency: 1,
|
|
86
|
+
fetchItem: async (it) => {
|
|
87
|
+
fetchOrder.push(it.id);
|
|
88
|
+
if (it.id === "a")
|
|
89
|
+
await firstGate;
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
// give the first run a tick to select+start fetching "a", then
|
|
93
|
+
// supersede it before it reaches "b".
|
|
94
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
95
|
+
let secondDone;
|
|
96
|
+
const secondFinished = new Promise((resolve) => {
|
|
97
|
+
secondDone = resolve;
|
|
98
|
+
});
|
|
99
|
+
prefetcher.run([item("c", 1)], {
|
|
100
|
+
budget: 10,
|
|
101
|
+
costOf: (it) => it.cost,
|
|
102
|
+
fetchItem: async (it) => {
|
|
103
|
+
fetchOrder.push(it.id);
|
|
104
|
+
},
|
|
105
|
+
onSettled: () => secondDone(),
|
|
106
|
+
});
|
|
107
|
+
await secondFinished;
|
|
108
|
+
releaseFirst();
|
|
109
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
110
|
+
expect(fetchOrder).toContain("a");
|
|
111
|
+
expect(fetchOrder).toContain("c");
|
|
112
|
+
expect(fetchOrder).not.toContain("b");
|
|
113
|
+
});
|
|
114
|
+
it("fires onSettled for selected-but-not-yet-fetched items when superseded", async () => {
|
|
115
|
+
const prefetcher = createPrefetcher();
|
|
116
|
+
const settled = [];
|
|
117
|
+
prefetcher.run([item("a", 1), item("b", 1), item("c", 1)], {
|
|
118
|
+
budget: 10,
|
|
119
|
+
costOf: (it) => it.cost,
|
|
120
|
+
concurrency: 1,
|
|
121
|
+
fetchItem: async () => {
|
|
122
|
+
await new Promise((r) => setTimeout(r, 20));
|
|
123
|
+
},
|
|
124
|
+
onSettled: (it) => settled.push(it.id),
|
|
125
|
+
});
|
|
126
|
+
await new Promise((r) => setTimeout(r, 0));
|
|
127
|
+
prefetcher.run([], { budget: 0, costOf: () => 0, fetchItem: async () => { } });
|
|
128
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
129
|
+
// "a" was mid-flight when superseded (its own batch already started,
|
|
130
|
+
// so it still completes); "b" and "c" were selected but never reached
|
|
131
|
+
// a batch - both get a settled callback anyway so ui pending state
|
|
132
|
+
// never leaks.
|
|
133
|
+
expect(settled).toContain("b");
|
|
134
|
+
expect(settled).toContain("c");
|
|
135
|
+
});
|
|
136
|
+
});
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import type { BlobCapableNode } from "./types.js";
|
|
2
|
+
/** how long an imported blob stays available after its last request,
|
|
3
|
+
* absent an explicit override. */
|
|
4
|
+
export declare const DEFAULT_RELEASE_AFTER_MS: number;
|
|
5
|
+
export interface BlobServerOptions {
|
|
6
|
+
/** ms an imported blob stays available after its last request. */
|
|
7
|
+
releaseAfterMs?: number;
|
|
8
|
+
}
|
|
9
|
+
export interface ServedBlobInfo {
|
|
10
|
+
blake3: string;
|
|
11
|
+
size: number;
|
|
12
|
+
mime?: string;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* server-side import cache for peer-served blobs. imports a locally-known
|
|
16
|
+
* blob into the transport node's store on first request, keyed by
|
|
17
|
+
* whatever id the app addresses that blob by (a sha256, a rest-style blob
|
|
18
|
+
* id, ...) - not necessarily the blake3 hash the peer ends up downloading
|
|
19
|
+
* by. every repeat request for the same id resets the release timer
|
|
20
|
+
* rather than re-importing.
|
|
21
|
+
*/
|
|
22
|
+
export declare class BlobServer {
|
|
23
|
+
private readonly node;
|
|
24
|
+
private readonly options;
|
|
25
|
+
private readonly served;
|
|
26
|
+
constructor(node: BlobCapableNode, options?: BlobServerOptions);
|
|
27
|
+
/** true when `id` is currently tracked (imported and not yet released). */
|
|
28
|
+
has(id: string): boolean;
|
|
29
|
+
/** the tracked info for `id`, without resetting its release timer, or
|
|
30
|
+
* `null` when not currently tracked. */
|
|
31
|
+
peek(id: string): ServedBlobInfo | null;
|
|
32
|
+
/** import (on first request) and (always) reset the release timer for
|
|
33
|
+
* `id`. `resolve` is only called on a cache miss - a repeat request
|
|
34
|
+
* never re-reads or re-imports the source bytes. */
|
|
35
|
+
serve(id: string, resolve: () => Promise<{
|
|
36
|
+
bytes: Uint8Array;
|
|
37
|
+
size: number;
|
|
38
|
+
mime?: string;
|
|
39
|
+
}>): Promise<ServedBlobInfo>;
|
|
40
|
+
/** reset the release timer for `id` without re-importing. a no-op when
|
|
41
|
+
* `id` isn't currently tracked. */
|
|
42
|
+
touch(id: string): void;
|
|
43
|
+
/** stop tracking `id` and release it from the node's store immediately,
|
|
44
|
+
* instead of waiting out the release timer. */
|
|
45
|
+
release(id: string): void;
|
|
46
|
+
/** number of ids currently tracked (import-cache size). */
|
|
47
|
+
size(): number;
|
|
48
|
+
/** clear every release timer and release every tracked blob from the
|
|
49
|
+
* node's store - call on shutdown so nothing stays pinned behind it. */
|
|
50
|
+
dispose(): void;
|
|
51
|
+
private scheduleRelease;
|
|
52
|
+
private releaseFromNode;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* answer a peer's blob request: resolve the blob's bytes locally (via
|
|
56
|
+
* `resolve`, only called on a cache miss) and stage it into the
|
|
57
|
+
* transport node's store through `server`, returning the blake3 hash +
|
|
58
|
+
* size the peer downloads by. returns `null` when `resolve` reports no
|
|
59
|
+
* such blob.
|
|
60
|
+
*/
|
|
61
|
+
export declare function serveBlobRequest(server: BlobServer, id: string, resolve: (id: string) => Promise<{
|
|
62
|
+
bytes: Uint8Array;
|
|
63
|
+
size: number;
|
|
64
|
+
mime?: string;
|
|
65
|
+
} | null>): Promise<ServedBlobInfo | null>;
|
|
66
|
+
//# sourceMappingURL=serve.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serve.d.ts","sourceRoot":"","sources":["../../src/transfer/serve.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAIlD;mCACmC;AACnC,eAAO,MAAM,wBAAwB,QAAiB,CAAC;AAEvD,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAMD;;;;;;;GAOG;AACH,qBAAa,UAAU;IAInB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;gBAGtC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,iBAAsB;IAGlD,2EAA2E;IAC3E,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIxB;6CACyC;IACzC,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,cAAc,GAAG,IAAI;IAMvC;;yDAEqD;IAC/C,KAAK,CACT,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,OAAO,CAAC;QAAE,KAAK,EAAE,UAAU,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,GACzE,OAAO,CAAC,cAAc,CAAC;IAkB1B;wCACoC;IACpC,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAKvB;oDACgD;IAChD,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAQzB,2DAA2D;IAC3D,IAAI,IAAI,MAAM;IAId;6EACyE;IACzE,OAAO,IAAI,IAAI;IAQf,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,eAAe;CAOxB;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CACpC,MAAM,EAAE,UAAU,EAClB,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,KAAK,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAAC,GAC1F,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,CAWhC"}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// serve side of p2p blob transfer: import cache + release timer.
|
|
2
|
+
//
|
|
3
|
+
// answering a peer's blob request means staging the blob's bytes into the
|
|
4
|
+
// transport node's own store (so verified-transfer downloads can read it),
|
|
5
|
+
// then releasing that stage once nobody has asked for it in a while so
|
|
6
|
+
// served memory doesn't grow unboundedly. `BlobServer` is that cache;
|
|
7
|
+
// `serveBlobRequest` is a thin resolver-based wrapper over it.
|
|
8
|
+
import { log } from "../utils/log.js";
|
|
9
|
+
const TAG = "transfer.serve";
|
|
10
|
+
/** how long an imported blob stays available after its last request,
|
|
11
|
+
* absent an explicit override. */
|
|
12
|
+
export const DEFAULT_RELEASE_AFTER_MS = 10 * 60 * 1000;
|
|
13
|
+
/**
|
|
14
|
+
* server-side import cache for peer-served blobs. imports a locally-known
|
|
15
|
+
* blob into the transport node's store on first request, keyed by
|
|
16
|
+
* whatever id the app addresses that blob by (a sha256, a rest-style blob
|
|
17
|
+
* id, ...) - not necessarily the blake3 hash the peer ends up downloading
|
|
18
|
+
* by. every repeat request for the same id resets the release timer
|
|
19
|
+
* rather than re-importing.
|
|
20
|
+
*/
|
|
21
|
+
export class BlobServer {
|
|
22
|
+
node;
|
|
23
|
+
options;
|
|
24
|
+
served = new Map();
|
|
25
|
+
constructor(node, options = {}) {
|
|
26
|
+
this.node = node;
|
|
27
|
+
this.options = options;
|
|
28
|
+
}
|
|
29
|
+
/** true when `id` is currently tracked (imported and not yet released). */
|
|
30
|
+
has(id) {
|
|
31
|
+
return this.served.has(id);
|
|
32
|
+
}
|
|
33
|
+
/** the tracked info for `id`, without resetting its release timer, or
|
|
34
|
+
* `null` when not currently tracked. */
|
|
35
|
+
peek(id) {
|
|
36
|
+
const entry = this.served.get(id);
|
|
37
|
+
if (!entry)
|
|
38
|
+
return null;
|
|
39
|
+
return { blake3: entry.blake3, size: entry.size, mime: entry.mime };
|
|
40
|
+
}
|
|
41
|
+
/** import (on first request) and (always) reset the release timer for
|
|
42
|
+
* `id`. `resolve` is only called on a cache miss - a repeat request
|
|
43
|
+
* never re-reads or re-imports the source bytes. */
|
|
44
|
+
async serve(id, resolve) {
|
|
45
|
+
const cached = this.served.get(id);
|
|
46
|
+
if (cached) {
|
|
47
|
+
this.scheduleRelease(id, cached);
|
|
48
|
+
return { blake3: cached.blake3, size: cached.size, mime: cached.mime };
|
|
49
|
+
}
|
|
50
|
+
if (!this.node.import_blob) {
|
|
51
|
+
throw new Error("node does not support import_blob");
|
|
52
|
+
}
|
|
53
|
+
const { bytes, size, mime } = await resolve();
|
|
54
|
+
const blake3 = await this.node.import_blob(bytes);
|
|
55
|
+
const info = { blake3, size, mime };
|
|
56
|
+
this.scheduleRelease(id, info);
|
|
57
|
+
return info;
|
|
58
|
+
}
|
|
59
|
+
/** reset the release timer for `id` without re-importing. a no-op when
|
|
60
|
+
* `id` isn't currently tracked. */
|
|
61
|
+
touch(id) {
|
|
62
|
+
const cached = this.served.get(id);
|
|
63
|
+
if (cached)
|
|
64
|
+
this.scheduleRelease(id, cached);
|
|
65
|
+
}
|
|
66
|
+
/** stop tracking `id` and release it from the node's store immediately,
|
|
67
|
+
* instead of waiting out the release timer. */
|
|
68
|
+
release(id) {
|
|
69
|
+
const entry = this.served.get(id);
|
|
70
|
+
if (!entry)
|
|
71
|
+
return;
|
|
72
|
+
clearTimeout(entry.releaseTimer);
|
|
73
|
+
this.served.delete(id);
|
|
74
|
+
this.releaseFromNode(entry.blake3);
|
|
75
|
+
}
|
|
76
|
+
/** number of ids currently tracked (import-cache size). */
|
|
77
|
+
size() {
|
|
78
|
+
return this.served.size;
|
|
79
|
+
}
|
|
80
|
+
/** clear every release timer and release every tracked blob from the
|
|
81
|
+
* node's store - call on shutdown so nothing stays pinned behind it. */
|
|
82
|
+
dispose() {
|
|
83
|
+
for (const [id, entry] of this.served) {
|
|
84
|
+
clearTimeout(entry.releaseTimer);
|
|
85
|
+
this.releaseFromNode(entry.blake3);
|
|
86
|
+
this.served.delete(id);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
scheduleRelease(id, info) {
|
|
90
|
+
const existing = this.served.get(id);
|
|
91
|
+
if (existing)
|
|
92
|
+
clearTimeout(existing.releaseTimer);
|
|
93
|
+
const releaseAfterMs = this.options.releaseAfterMs ?? DEFAULT_RELEASE_AFTER_MS;
|
|
94
|
+
const releaseTimer = setTimeout(() => {
|
|
95
|
+
this.served.delete(id);
|
|
96
|
+
this.releaseFromNode(info.blake3);
|
|
97
|
+
}, releaseAfterMs);
|
|
98
|
+
this.served.set(id, { ...info, releaseTimer });
|
|
99
|
+
}
|
|
100
|
+
releaseFromNode(blake3) {
|
|
101
|
+
try {
|
|
102
|
+
void this.node.release_blob?.(blake3);
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
log.debug(TAG, "release_blob failed (non-fatal):", err);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* answer a peer's blob request: resolve the blob's bytes locally (via
|
|
111
|
+
* `resolve`, only called on a cache miss) and stage it into the
|
|
112
|
+
* transport node's store through `server`, returning the blake3 hash +
|
|
113
|
+
* size the peer downloads by. returns `null` when `resolve` reports no
|
|
114
|
+
* such blob.
|
|
115
|
+
*/
|
|
116
|
+
export async function serveBlobRequest(server, id, resolve) {
|
|
117
|
+
const cached = server.peek(id);
|
|
118
|
+
if (cached) {
|
|
119
|
+
server.touch(id);
|
|
120
|
+
return cached;
|
|
121
|
+
}
|
|
122
|
+
const blob = await resolve(id);
|
|
123
|
+
if (!blob)
|
|
124
|
+
return null;
|
|
125
|
+
return server.serve(id, async () => blob);
|
|
126
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serve.test.d.ts","sourceRoot":"","sources":["../../src/transfer/serve.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { BlobServer, serveBlobRequest } from "./serve.js";
|
|
3
|
+
beforeEach(() => {
|
|
4
|
+
vi.useFakeTimers();
|
|
5
|
+
});
|
|
6
|
+
afterEach(() => {
|
|
7
|
+
vi.useRealTimers();
|
|
8
|
+
});
|
|
9
|
+
function makeNode() {
|
|
10
|
+
let counter = 0;
|
|
11
|
+
return {
|
|
12
|
+
node_id: () => "me",
|
|
13
|
+
import_blob: vi.fn(async () => `blake3-${++counter}`),
|
|
14
|
+
release_blob: vi.fn(),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
describe("BlobServer", () => {
|
|
18
|
+
it("imports on first serve and reuses the cached blake3 on a repeat request", async () => {
|
|
19
|
+
const node = makeNode();
|
|
20
|
+
const server = new BlobServer(node);
|
|
21
|
+
const getBytes = vi.fn(async () => new Uint8Array([1, 2, 3]));
|
|
22
|
+
const first = await server.serve("sha-a", async () => ({ bytes: await getBytes(), size: 3 }));
|
|
23
|
+
const second = await server.serve("sha-a", async () => ({ bytes: await getBytes(), size: 3 }));
|
|
24
|
+
expect(first.blake3).toBe(second.blake3);
|
|
25
|
+
expect(node.import_blob).toHaveBeenCalledTimes(1);
|
|
26
|
+
expect(getBytes).toHaveBeenCalledTimes(1);
|
|
27
|
+
});
|
|
28
|
+
it("releases from the node after the release timer elapses", async () => {
|
|
29
|
+
const node = makeNode();
|
|
30
|
+
const server = new BlobServer(node, { releaseAfterMs: 1000 });
|
|
31
|
+
await server.serve("sha-a", async () => ({ bytes: new Uint8Array([1]), size: 1 }));
|
|
32
|
+
expect(server.has("sha-a")).toBe(true);
|
|
33
|
+
vi.advanceTimersByTime(999);
|
|
34
|
+
expect(server.has("sha-a")).toBe(true);
|
|
35
|
+
expect(node.release_blob).not.toHaveBeenCalled();
|
|
36
|
+
vi.advanceTimersByTime(1);
|
|
37
|
+
expect(server.has("sha-a")).toBe(false);
|
|
38
|
+
expect(node.release_blob).toHaveBeenCalledWith("blake3-1");
|
|
39
|
+
});
|
|
40
|
+
it("resets the release timer on every repeat request", async () => {
|
|
41
|
+
const node = makeNode();
|
|
42
|
+
const server = new BlobServer(node, { releaseAfterMs: 1000 });
|
|
43
|
+
await server.serve("sha-a", async () => ({ bytes: new Uint8Array([1]), size: 1 }));
|
|
44
|
+
vi.advanceTimersByTime(700);
|
|
45
|
+
await server.serve("sha-a", async () => ({ bytes: new Uint8Array([1]), size: 1 }));
|
|
46
|
+
vi.advanceTimersByTime(700);
|
|
47
|
+
// 1400ms elapsed total, but the second serve reset the clock at 700ms,
|
|
48
|
+
// so only 700ms have passed since the last touch - still tracked.
|
|
49
|
+
expect(server.has("sha-a")).toBe(true);
|
|
50
|
+
vi.advanceTimersByTime(300);
|
|
51
|
+
expect(server.has("sha-a")).toBe(false);
|
|
52
|
+
});
|
|
53
|
+
it("releases immediately on request, without waiting for the timer", async () => {
|
|
54
|
+
const node = makeNode();
|
|
55
|
+
const server = new BlobServer(node);
|
|
56
|
+
await server.serve("sha-a", async () => ({ bytes: new Uint8Array([1]), size: 1 }));
|
|
57
|
+
server.release("sha-a");
|
|
58
|
+
expect(server.has("sha-a")).toBe(false);
|
|
59
|
+
expect(node.release_blob).toHaveBeenCalledWith("blake3-1");
|
|
60
|
+
});
|
|
61
|
+
it("dispose releases every tracked blob", async () => {
|
|
62
|
+
const node = makeNode();
|
|
63
|
+
const server = new BlobServer(node);
|
|
64
|
+
await server.serve("a", async () => ({ bytes: new Uint8Array([1]), size: 1 }));
|
|
65
|
+
await server.serve("b", async () => ({ bytes: new Uint8Array([2]), size: 1 }));
|
|
66
|
+
server.dispose();
|
|
67
|
+
expect(server.size()).toBe(0);
|
|
68
|
+
expect(node.release_blob).toHaveBeenCalledTimes(2);
|
|
69
|
+
});
|
|
70
|
+
it("rejects when the node has no import_blob", async () => {
|
|
71
|
+
const server = new BlobServer({ node_id: () => "me" });
|
|
72
|
+
await expect(server.serve("a", async () => ({ bytes: new Uint8Array([1]), size: 1 }))).rejects.toThrow(/import_blob/);
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
describe("serveBlobRequest", () => {
|
|
76
|
+
it("resolves bytes on a cache miss and stages them via the server", async () => {
|
|
77
|
+
const node = makeNode();
|
|
78
|
+
const server = new BlobServer(node);
|
|
79
|
+
const resolve = vi.fn(async () => ({ bytes: new Uint8Array([1, 2]), size: 2, mime: "audio/mpeg" }));
|
|
80
|
+
const result = await serveBlobRequest(server, "sha-a", resolve);
|
|
81
|
+
expect(result).toEqual({ blake3: "blake3-1", size: 2, mime: "audio/mpeg" });
|
|
82
|
+
expect(resolve).toHaveBeenCalledWith("sha-a");
|
|
83
|
+
});
|
|
84
|
+
it("returns null when resolve reports no such blob, without touching the server", async () => {
|
|
85
|
+
const node = makeNode();
|
|
86
|
+
const server = new BlobServer(node);
|
|
87
|
+
const resolve = vi.fn(async () => null);
|
|
88
|
+
const result = await serveBlobRequest(server, "missing", resolve);
|
|
89
|
+
expect(result).toBeNull();
|
|
90
|
+
expect(node.import_blob).not.toHaveBeenCalled();
|
|
91
|
+
});
|
|
92
|
+
it("skips resolve entirely on a cache hit", async () => {
|
|
93
|
+
const node = makeNode();
|
|
94
|
+
const server = new BlobServer(node);
|
|
95
|
+
await server.serve("sha-a", async () => ({ bytes: new Uint8Array([1]), size: 1 }));
|
|
96
|
+
const resolve = vi.fn(async () => ({ bytes: new Uint8Array([9]), size: 9 }));
|
|
97
|
+
const result = await serveBlobRequest(server, "sha-a", resolve);
|
|
98
|
+
expect(resolve).not.toHaveBeenCalled();
|
|
99
|
+
expect(result?.size).toBe(1);
|
|
100
|
+
});
|
|
101
|
+
});
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { BlobCapableNode, DiskSnatchResult, SnatchInfo, SnatchOptions, SnatchResult } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* download a blob's bytes from the first peer (in `peerNodeIds` order)
|
|
4
|
+
* that can serve it. a peer failure (offline, transport error, hash
|
|
5
|
+
* mismatch) tries the next peer; a cancelled error (see `cancellation.ts`)
|
|
6
|
+
* rethrows immediately with no next-peer retry.
|
|
7
|
+
*/
|
|
8
|
+
export declare function snatchBlob(node: BlobCapableNode, peerNodeIds: string[], info: SnatchInfo, options?: SnatchOptions): Promise<SnatchResult>;
|
|
9
|
+
/**
|
|
10
|
+
* download a blob straight to a user-chosen disk location, writing each
|
|
11
|
+
* verified chunk to `writable` at its explicit offset as it streams in
|
|
12
|
+
* (no full payload ever held in memory). falls back to a fully-buffered
|
|
13
|
+
* `snatchBlob` + single write when the node has no streaming strategy.
|
|
14
|
+
*
|
|
15
|
+
* on a mid-stream peer failure, `writable` is truncated back to zero
|
|
16
|
+
* before the next peer is tried, so a retry never appends onto partial
|
|
17
|
+
* data. on a cancelled error, `writable` is left exactly as it is (no
|
|
18
|
+
* truncate, no close) so a resumed snatch can rewrite the same offsets;
|
|
19
|
+
* the caller owns pausing/closing/discarding it in that case.
|
|
20
|
+
*/
|
|
21
|
+
export declare function snatchBlobToDisk(node: BlobCapableNode, peerNodeIds: string[], info: SnatchInfo, writable: FileSystemWritableFileStream, options?: SnatchOptions): Promise<DiskSnatchResult>;
|
|
22
|
+
/**
|
|
23
|
+
* pause an in-flight snatch registered under `downloadId`. the transfer
|
|
24
|
+
* stops at the next chunk boundary and its promise rejects with a
|
|
25
|
+
* cancelled error (see `isCancelledError`). the partial stays pinned in
|
|
26
|
+
* the transport's store - resume by calling `snatchBlob`/
|
|
27
|
+
* `snatchBlobToDisk` again with the same `downloadId`/blake3 (only
|
|
28
|
+
* missing ranges transfer). returns false when `node` has no
|
|
29
|
+
* `download_cancel` or the download already settled.
|
|
30
|
+
*/
|
|
31
|
+
export declare function pauseSnatchDownload(node: BlobCapableNode, downloadId: string): Promise<boolean>;
|
|
32
|
+
/**
|
|
33
|
+
* discard a paused partial: releases the gc pin a paused download left
|
|
34
|
+
* behind so the transport can reclaim it. call when the user cancels for
|
|
35
|
+
* good rather than pausing to resume later. best-effort - failures are
|
|
36
|
+
* logged, not thrown.
|
|
37
|
+
*/
|
|
38
|
+
export declare function discardPausedDownload(node: BlobCapableNode, blake3Hash: string): Promise<void>;
|
|
39
|
+
//# sourceMappingURL=snatch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"snatch.d.ts","sourceRoot":"","sources":["../../src/transfer/snatch.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EACV,eAAe,EACf,gBAAgB,EAChB,UAAU,EACV,aAAa,EACb,YAAY,EACb,MAAM,YAAY,CAAC;AA0KpB;;;;;GAKG;AACH,wBAAsB,UAAU,CAC9B,IAAI,EAAE,eAAe,EACrB,WAAW,EAAE,MAAM,EAAE,EACrB,IAAI,EAAE,UAAU,EAChB,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,YAAY,CAAC,CAwBvB;AAiDD;;;;;;;;;;;GAWG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,eAAe,EACrB,WAAW,EAAE,MAAM,EAAE,EACrB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,4BAA4B,EACtC,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAAC,gBAAgB,CAAC,CAsD3B;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACvC,IAAI,EAAE,eAAe,EACrB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,OAAO,CAAC,CAGlB;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,eAAe,EACrB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC,CAOf"}
|