cursedbelt-core 2.13.3 → 2.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,71 @@
1
+ /**
2
+ * The wire contract of an upload PRE-CHECK — "which of these do you already have?", asked before
3
+ * a single byte of the files is sent (forge task 3579).
4
+ *
5
+ * The owner, 2026-10-22: *"They should find and skip dups efficiently without wasting bandwidth or
6
+ * processing power … They need to know dups based on the hashes rather than a long check such as
7
+ * copying the file first … I will sometimes reimport from huge directories just in case they have a
8
+ * new file that was missed on previous uploads and I want it to be fast."*
9
+ *
10
+ * ── Two questions, one route ───────────────────────────────────────────────────────────────
11
+ * 1. **`sizes`** — which of these byte sizes does the library hold at all? A file whose size
12
+ * matches nothing stored CANNOT be a byte-identical copy, so it is uploaded without being hashed
13
+ * first (binary-server hashes what it receives, as it always has). On a folder of mostly new
14
+ * videos that is nearly every file, and nearly all of the hashing.
15
+ * 2. **`sha256`** — which of these whole-file SHA-256s does the library hold? Asked only for the
16
+ * files whose size DID match, so the proof (the hash) is paid exactly where it can decide
17
+ * something. A hit is never sent.
18
+ *
19
+ * A route answers both in one body, so a client may ask either or both. Every app's library is
20
+ * scoped by the app's own rules (an account, an owner) — the route answers for the CALLER only,
21
+ * which is also what keeps the answer from saying anything about anyone else's files.
22
+ *
23
+ * ── SQL: one bound parameter, any length ───────────────────────────────────────────────────
24
+ * D1 refuses a statement with more than 100 bound parameters, so `IN (?, ?, …)` over a batch of
25
+ * 1,000 hashes cannot be written. Bind the list ONCE as JSON instead —
26
+ * `WHERE checksum IN (SELECT value FROM json_each(?))` with {@link jsonList} — which D1 and
27
+ * `bun:sqlite` both run against the column's index.
28
+ */
29
+ /** The most sizes one pre-check may ask about. A drop of 5,000 files asks once. */
30
+ export declare const PRECHECK_MAX_SIZES = 5000;
31
+ /** The most hashes one pre-check may ask about — a client sends them in batches this size or less. */
32
+ export declare const PRECHECK_MAX_HASHES = 1000;
33
+ /** What the client sends. Either list may be absent or empty. */
34
+ export interface PrecheckRequest {
35
+ sizes?: number[];
36
+ sha256?: string[];
37
+ }
38
+ /** Why a hash is refused: the library holds it, or once held it and its owner deleted it. */
39
+ export type PrecheckReason = "exists" | "deleted";
40
+ /** One hash the library already answers for. */
41
+ export interface PrecheckHit {
42
+ reason: PrecheckReason;
43
+ /** What the library calls it — shown to the person beside the skipped file. */
44
+ name?: string;
45
+ /** The app's own id for the item, when it has one worth linking to. */
46
+ id?: string;
47
+ }
48
+ /** What the route answers. */
49
+ export interface PrecheckResponse {
50
+ /** The asked sizes that the library holds at least one file of. */
51
+ sizes: number[];
52
+ /** The asked hashes the library already answers for; any hash absent here is new. */
53
+ have: Record<string, PrecheckHit>;
54
+ }
55
+ /** Is this a lowercase hex SHA-256 — the only spelling every store in the fleet records? */
56
+ export declare function isSha256Hex(value: unknown): value is string;
57
+ export type ParsedPrecheck = {
58
+ ok: true;
59
+ sizes: number[];
60
+ sha256: string[];
61
+ } | {
62
+ ok: false;
63
+ error: string;
64
+ };
65
+ /**
66
+ * Validate a pre-check body — the ONE parser every app's route calls, so the limits and the
67
+ * spelling are the same everywhere. Duplicates collapse; order is not kept.
68
+ */
69
+ export declare function parsePrecheckRequest(body: unknown): ParsedPrecheck;
70
+ /** A list bound as ONE SQL parameter, for `IN (SELECT value FROM json_each(?))`. */
71
+ export declare function jsonList(values: readonly (string | number)[]): string;
@@ -0,0 +1,76 @@
1
+ /**
2
+ * The wire contract of an upload PRE-CHECK — "which of these do you already have?", asked before
3
+ * a single byte of the files is sent (forge task 3579).
4
+ *
5
+ * The owner, 2026-10-22: *"They should find and skip dups efficiently without wasting bandwidth or
6
+ * processing power … They need to know dups based on the hashes rather than a long check such as
7
+ * copying the file first … I will sometimes reimport from huge directories just in case they have a
8
+ * new file that was missed on previous uploads and I want it to be fast."*
9
+ *
10
+ * ── Two questions, one route ───────────────────────────────────────────────────────────────
11
+ * 1. **`sizes`** — which of these byte sizes does the library hold at all? A file whose size
12
+ * matches nothing stored CANNOT be a byte-identical copy, so it is uploaded without being hashed
13
+ * first (binary-server hashes what it receives, as it always has). On a folder of mostly new
14
+ * videos that is nearly every file, and nearly all of the hashing.
15
+ * 2. **`sha256`** — which of these whole-file SHA-256s does the library hold? Asked only for the
16
+ * files whose size DID match, so the proof (the hash) is paid exactly where it can decide
17
+ * something. A hit is never sent.
18
+ *
19
+ * A route answers both in one body, so a client may ask either or both. Every app's library is
20
+ * scoped by the app's own rules (an account, an owner) — the route answers for the CALLER only,
21
+ * which is also what keeps the answer from saying anything about anyone else's files.
22
+ *
23
+ * ── SQL: one bound parameter, any length ───────────────────────────────────────────────────
24
+ * D1 refuses a statement with more than 100 bound parameters, so `IN (?, ?, …)` over a batch of
25
+ * 1,000 hashes cannot be written. Bind the list ONCE as JSON instead —
26
+ * `WHERE checksum IN (SELECT value FROM json_each(?))` with {@link jsonList} — which D1 and
27
+ * `bun:sqlite` both run against the column's index.
28
+ */
29
+ /** The most sizes one pre-check may ask about. A drop of 5,000 files asks once. */
30
+ export const PRECHECK_MAX_SIZES = 5000;
31
+ /** The most hashes one pre-check may ask about — a client sends them in batches this size or less. */
32
+ export const PRECHECK_MAX_HASHES = 1000;
33
+ const SHA256_HEX = /^[0-9a-f]{64}$/;
34
+ /** Is this a lowercase hex SHA-256 — the only spelling every store in the fleet records? */
35
+ export function isSha256Hex(value) {
36
+ return typeof value === "string" && SHA256_HEX.test(value);
37
+ }
38
+ /**
39
+ * Validate a pre-check body — the ONE parser every app's route calls, so the limits and the
40
+ * spelling are the same everywhere. Duplicates collapse; order is not kept.
41
+ */
42
+ export function parsePrecheckRequest(body) {
43
+ if (typeof body !== "object" || body === null || Array.isArray(body)) {
44
+ return { ok: false, error: "expected a JSON object with `sizes` and/or `sha256`" };
45
+ }
46
+ const { sizes: rawSizes, sha256: rawHashes } = body;
47
+ if (rawSizes !== undefined && !Array.isArray(rawSizes))
48
+ return { ok: false, error: "`sizes` must be an array" };
49
+ if (rawHashes !== undefined && !Array.isArray(rawHashes))
50
+ return { ok: false, error: "`sha256` must be an array" };
51
+ const sizes = new Set();
52
+ for (const size of rawSizes ?? []) {
53
+ if (!Number.isSafeInteger(size) || size < 0) {
54
+ return { ok: false, error: "every size must be a whole number of bytes" };
55
+ }
56
+ sizes.add(size);
57
+ }
58
+ const hashes = new Set();
59
+ for (const hash of rawHashes ?? []) {
60
+ const spelled = typeof hash === "string" ? hash.toLowerCase() : hash;
61
+ if (!isSha256Hex(spelled))
62
+ return { ok: false, error: "every sha256 must be 64 hex characters" };
63
+ hashes.add(spelled);
64
+ }
65
+ if (sizes.size > PRECHECK_MAX_SIZES) {
66
+ return { ok: false, error: `at most ${PRECHECK_MAX_SIZES} sizes per pre-check` };
67
+ }
68
+ if (hashes.size > PRECHECK_MAX_HASHES) {
69
+ return { ok: false, error: `at most ${PRECHECK_MAX_HASHES} hashes per pre-check` };
70
+ }
71
+ return { ok: true, sizes: [...sizes], sha256: [...hashes] };
72
+ }
73
+ /** A list bound as ONE SQL parameter, for `IN (SELECT value FROM json_each(?))`. */
74
+ export function jsonList(values) {
75
+ return JSON.stringify(values);
76
+ }
@@ -0,0 +1,26 @@
1
+ /**
2
+ * The largest file hashed in ONE native `crypto.subtle.digest` call — read whole into memory.
3
+ *
4
+ * Native hashing ran 1,960 MiB/s on this Mac against 184 MiB/s for the incremental one, so a
5
+ * file is read whole whenever that is safe. 256 MiB is far under what a phone tab survives, and
6
+ * covers every photo, every song and most clips; past it the incremental hasher streams in
7
+ * {@link STREAM_SLICE_BYTES} slices at roughly the speed an external drive reads anyway.
8
+ */
9
+ export declare const NATIVE_HASH_MAX_BYTES: number;
10
+ /**
11
+ * One read while streaming. Small enough that the incremental hasher returns to the event loop
12
+ * every ~10 ms (a 2 MiB slice at ~190 MiB/s), so a tab hashing an 8 GB disc stays responsive.
13
+ */
14
+ export declare const STREAM_SLICE_BYTES: number;
15
+ export interface HashOptions {
16
+ /** 0..1 as the bytes are read. */
17
+ onProgress?: (fraction: number) => void;
18
+ signal?: AbortSignal;
19
+ /** Override {@link NATIVE_HASH_MAX_BYTES} — the specs force the streaming road with it. */
20
+ nativeMaxBytes?: number;
21
+ }
22
+ /**
23
+ * The whole-file SHA-256 of a Blob (a browser `File`, a `Bun.file`) of any size, as lowercase hex
24
+ * — the same digest binary-server records in `file_index.checksum` for the same bytes.
25
+ */
26
+ export declare function sha256OfBlob(blob: Blob, options?: HashOptions): Promise<string>;
@@ -0,0 +1,46 @@
1
+ import { Sha256 } from "./sha256.js";
2
+ /**
3
+ * The largest file hashed in ONE native `crypto.subtle.digest` call — read whole into memory.
4
+ *
5
+ * Native hashing ran 1,960 MiB/s on this Mac against 184 MiB/s for the incremental one, so a
6
+ * file is read whole whenever that is safe. 256 MiB is far under what a phone tab survives, and
7
+ * covers every photo, every song and most clips; past it the incremental hasher streams in
8
+ * {@link STREAM_SLICE_BYTES} slices at roughly the speed an external drive reads anyway.
9
+ */
10
+ export const NATIVE_HASH_MAX_BYTES = 256 * 1024 * 1024;
11
+ /**
12
+ * One read while streaming. Small enough that the incremental hasher returns to the event loop
13
+ * every ~10 ms (a 2 MiB slice at ~190 MiB/s), so a tab hashing an 8 GB disc stays responsive.
14
+ */
15
+ export const STREAM_SLICE_BYTES = 2 * 1024 * 1024;
16
+ function hex(digest) {
17
+ const view = new Uint8Array(digest);
18
+ let out = "";
19
+ for (let i = 0; i < view.length; i++)
20
+ out += view[i].toString(16).padStart(2, "0");
21
+ return out;
22
+ }
23
+ function throwIfAborted(signal) {
24
+ if (signal?.aborted)
25
+ throw signal.reason ?? new DOMException("The upload was canceled.", "AbortError");
26
+ }
27
+ /**
28
+ * The whole-file SHA-256 of a Blob (a browser `File`, a `Bun.file`) of any size, as lowercase hex
29
+ * — the same digest binary-server records in `file_index.checksum` for the same bytes.
30
+ */
31
+ export async function sha256OfBlob(blob, options = {}) {
32
+ const { onProgress, signal, nativeMaxBytes = NATIVE_HASH_MAX_BYTES } = options;
33
+ throwIfAborted(signal);
34
+ if (blob.size <= nativeMaxBytes) {
35
+ const digest = await crypto.subtle.digest("SHA-256", await blob.arrayBuffer());
36
+ onProgress?.(1);
37
+ return hex(digest);
38
+ }
39
+ const hasher = new Sha256();
40
+ for (let offset = 0; offset < blob.size; offset += STREAM_SLICE_BYTES) {
41
+ throwIfAborted(signal);
42
+ hasher.update(new Uint8Array(await blob.slice(offset, offset + STREAM_SLICE_BYTES).arrayBuffer()));
43
+ onProgress?.(Math.min(1, (offset + STREAM_SLICE_BYTES) / blob.size));
44
+ }
45
+ return hasher.hex();
46
+ }
@@ -0,0 +1,21 @@
1
+ /**
2
+ * `cursedbelt-core/upload-precheck` — skip a duplicate upload by its hash BEFORE sending it
3
+ * (forge task 3579). Every app's dropzone, uploader and Mac ingest uses this one module, so the
4
+ * order of the work (sizes → memo → hash → ask) and the route's contract are the same everywhere.
5
+ *
6
+ * - `planUploads` — the client side: decide which files to send.
7
+ * - `parsePrecheckRequest`, `jsonList` — the server side: each app's route parses with the first
8
+ * and binds its lists with the second.
9
+ * - `sha256OfBlob` — the whole-file SHA-256 binary-server records, of a Blob of any size.
10
+ * - `indexedDbHashMemo`, `memoryHashMemo` — the memo that makes a re-import of the same folder
11
+ * read nothing.
12
+ *
13
+ * Pure of any framework; a browser bundle and a Bun script may both import it. Its sibling
14
+ * `cursedbelt-core/duplicates` answers the other question — two files that are LIKELY the same
15
+ * (a re-encode, a copy under another name) — which is never enough to skip an upload on.
16
+ */
17
+ export { isSha256Hex, jsonList, PRECHECK_MAX_HASHES, PRECHECK_MAX_SIZES, type ParsedPrecheck, type PrecheckHit, type PrecheckReason, type PrecheckRequest, type PrecheckResponse, parsePrecheckRequest, } from "./contract.js";
18
+ export { type HashOptions, NATIVE_HASH_MAX_BYTES, STREAM_SLICE_BYTES, sha256OfBlob } from "./hash.js";
19
+ export { BROWSER_MEMO_MAX_ENTRIES, type HashMemo, indexedDbHashMemo, memoKey, memoryHashMemo, statIdentity, } from "./memo.js";
20
+ export { type PlanOptions, type PlanResult, type PlanStats, type PrecheckAsk, planUploads, type Verdict, } from "./plan.js";
21
+ export { Sha256 } from "./sha256.js";
@@ -0,0 +1,21 @@
1
+ /**
2
+ * `cursedbelt-core/upload-precheck` — skip a duplicate upload by its hash BEFORE sending it
3
+ * (forge task 3579). Every app's dropzone, uploader and Mac ingest uses this one module, so the
4
+ * order of the work (sizes → memo → hash → ask) and the route's contract are the same everywhere.
5
+ *
6
+ * - `planUploads` — the client side: decide which files to send.
7
+ * - `parsePrecheckRequest`, `jsonList` — the server side: each app's route parses with the first
8
+ * and binds its lists with the second.
9
+ * - `sha256OfBlob` — the whole-file SHA-256 binary-server records, of a Blob of any size.
10
+ * - `indexedDbHashMemo`, `memoryHashMemo` — the memo that makes a re-import of the same folder
11
+ * read nothing.
12
+ *
13
+ * Pure of any framework; a browser bundle and a Bun script may both import it. Its sibling
14
+ * `cursedbelt-core/duplicates` answers the other question — two files that are LIKELY the same
15
+ * (a re-encode, a copy under another name) — which is never enough to skip an upload on.
16
+ */
17
+ export { isSha256Hex, jsonList, PRECHECK_MAX_HASHES, PRECHECK_MAX_SIZES, parsePrecheckRequest, } from "./contract.js";
18
+ export { NATIVE_HASH_MAX_BYTES, STREAM_SLICE_BYTES, sha256OfBlob } from "./hash.js";
19
+ export { BROWSER_MEMO_MAX_ENTRIES, indexedDbHashMemo, memoKey, memoryHashMemo, statIdentity, } from "./memo.js";
20
+ export { planUploads, } from "./plan.js";
21
+ export { Sha256 } from "./sha256.js";
@@ -0,0 +1,52 @@
1
+ /**
2
+ * The memory that makes a RE-IMPORT cheap: a file's hash, remembered by where it was, how big and
3
+ * how old — so a second drop of the same huge folder hashes nothing it already hashed.
4
+ *
5
+ * Collections' Mac sweeper keeps the same key for its linked folders (`apps/collections/scripts/ingestLinked.ts`:
6
+ * path, size, mtime) and has re-walked `tod` every sweep on it since forge task 3508. A file whose
7
+ * path, size AND modification time all match a remembered one is the same file unless something
8
+ * rewrote it and set its clock back; anything else misses, and is hashed again.
9
+ *
10
+ * A memo only ever saves work: a miss, a broken store or a private window means "hash it", never a
11
+ * wrong answer, because the hash it returns is still checked against the library.
12
+ */
13
+ /** The batch interface — one transaction per batch, not one per file. */
14
+ export interface HashMemo {
15
+ get(keys: readonly string[]): Promise<Map<string, string>>;
16
+ set(entries: readonly (readonly [key: string, sha256: string])[]): Promise<void>;
17
+ }
18
+ /**
19
+ * The key a file is remembered under. `path` is its path inside the drop when known, else its name.
20
+ *
21
+ * `identity` is what a Mac script can add for free from the `stat` it already took —
22
+ * {@link statIdentity}. It closes the one gap path+size+date leave: a file DELETED and replaced by a
23
+ * different one with the same name, the same size and the same modification time to the
24
+ * millisecond (`cp -p`, or a tool that copies dates). The replacement is a new inode, and any write
25
+ * moves the change time, which no tool can set back. A browser `File` exposes neither, so the
26
+ * browser's key is path+size+date alone — two different files agreeing on all three is the case
27
+ * left, and it takes a deliberately copied date.
28
+ */
29
+ export declare function memoKey(path: string, size: number, lastModified: number | undefined, identity?: string): string;
30
+ /**
31
+ * A file's identity beyond its path, size and date: its inode and its change time (`ctime`), both
32
+ * in the `stat` a script already has. Measured on the linked `tod` folder (APFS, 2026-10-22): files
33
+ * ingested that day still carried change times from months before — reading, hashing and filing a
34
+ * file never moves it — so this costs a re-hash only when something really touched the file.
35
+ */
36
+ export declare function statIdentity(stat: {
37
+ ino: number | bigint;
38
+ ctimeMs: number;
39
+ }): string;
40
+ /** A memo held in memory — the Mac scripts' and the specs'. */
41
+ export declare function memoryHashMemo(): HashMemo & {
42
+ readonly size: number;
43
+ };
44
+ /** How many files a browser memo remembers before it forgets the oldest quarter. ~150 bytes each. */
45
+ export declare const BROWSER_MEMO_MAX_ENTRIES = 200000;
46
+ /**
47
+ * A memo kept in this origin's IndexedDB, so it survives a reload and a re-import next week. Each
48
+ * app is its own origin, so each keeps its own. Where IndexedDB is missing or refuses (a private
49
+ * window, a test DOM), every call quietly answers "not remembered" — the upload still works, it
50
+ * just hashes.
51
+ */
52
+ export declare function indexedDbHashMemo(name?: string, maxEntries?: number): HashMemo;
@@ -0,0 +1,160 @@
1
+ /**
2
+ * The memory that makes a RE-IMPORT cheap: a file's hash, remembered by where it was, how big and
3
+ * how old — so a second drop of the same huge folder hashes nothing it already hashed.
4
+ *
5
+ * Collections' Mac sweeper keeps the same key for its linked folders (`apps/collections/scripts/ingestLinked.ts`:
6
+ * path, size, mtime) and has re-walked `tod` every sweep on it since forge task 3508. A file whose
7
+ * path, size AND modification time all match a remembered one is the same file unless something
8
+ * rewrote it and set its clock back; anything else misses, and is hashed again.
9
+ *
10
+ * A memo only ever saves work: a miss, a broken store or a private window means "hash it", never a
11
+ * wrong answer, because the hash it returns is still checked against the library.
12
+ */
13
+ /**
14
+ * The key a file is remembered under. `path` is its path inside the drop when known, else its name.
15
+ *
16
+ * `identity` is what a Mac script can add for free from the `stat` it already took —
17
+ * {@link statIdentity}. It closes the one gap path+size+date leave: a file DELETED and replaced by a
18
+ * different one with the same name, the same size and the same modification time to the
19
+ * millisecond (`cp -p`, or a tool that copies dates). The replacement is a new inode, and any write
20
+ * moves the change time, which no tool can set back. A browser `File` exposes neither, so the
21
+ * browser's key is path+size+date alone — two different files agreeing on all three is the case
22
+ * left, and it takes a deliberately copied date.
23
+ */
24
+ export function memoKey(path, size, lastModified, identity) {
25
+ const key = `${path}\u0000${size}\u0000${lastModified ?? ""}`;
26
+ return identity ? `${key}\u0000${identity}` : key;
27
+ }
28
+ /**
29
+ * A file's identity beyond its path, size and date: its inode and its change time (`ctime`), both
30
+ * in the `stat` a script already has. Measured on the linked `tod` folder (APFS, 2026-10-22): files
31
+ * ingested that day still carried change times from months before — reading, hashing and filing a
32
+ * file never moves it — so this costs a re-hash only when something really touched the file.
33
+ */
34
+ export function statIdentity(stat) {
35
+ return `${stat.ino}:${Math.trunc(stat.ctimeMs)}`;
36
+ }
37
+ /** A memo held in memory — the Mac scripts' and the specs'. */
38
+ export function memoryHashMemo() {
39
+ const map = new Map();
40
+ return {
41
+ get size() {
42
+ return map.size;
43
+ },
44
+ async get(keys) {
45
+ const out = new Map();
46
+ for (const key of keys) {
47
+ const hit = map.get(key);
48
+ if (hit)
49
+ out.set(key, hit);
50
+ }
51
+ return out;
52
+ },
53
+ async set(entries) {
54
+ for (const [key, sha] of entries)
55
+ map.set(key, sha);
56
+ },
57
+ };
58
+ }
59
+ /** How many files a browser memo remembers before it forgets the oldest quarter. ~150 bytes each. */
60
+ export const BROWSER_MEMO_MAX_ENTRIES = 200_000;
61
+ const STORE = "hashes";
62
+ function request(req) {
63
+ return new Promise((resolve, reject) => {
64
+ req.onsuccess = () => resolve(req.result);
65
+ req.onerror = () => reject(req.error);
66
+ });
67
+ }
68
+ function done(tx) {
69
+ return new Promise((resolve, reject) => {
70
+ tx.oncomplete = () => resolve();
71
+ tx.onerror = () => reject(tx.error);
72
+ tx.onabort = () => reject(tx.error);
73
+ });
74
+ }
75
+ /**
76
+ * A memo kept in this origin's IndexedDB, so it survives a reload and a re-import next week. Each
77
+ * app is its own origin, so each keeps its own. Where IndexedDB is missing or refuses (a private
78
+ * window, a test DOM), every call quietly answers "not remembered" — the upload still works, it
79
+ * just hashes.
80
+ */
81
+ export function indexedDbHashMemo(name = "upload-precheck", maxEntries = BROWSER_MEMO_MAX_ENTRIES) {
82
+ let opened;
83
+ const open = () => {
84
+ opened ??= new Promise((resolve) => {
85
+ try {
86
+ if (typeof indexedDB === "undefined")
87
+ return resolve(null);
88
+ const req = indexedDB.open(name, 1);
89
+ req.onupgradeneeded = () => {
90
+ const store = req.result.createObjectStore(STORE, { keyPath: "k" });
91
+ store.createIndex("at", "at");
92
+ };
93
+ req.onsuccess = () => resolve(req.result);
94
+ req.onerror = () => resolve(null);
95
+ req.onblocked = () => resolve(null);
96
+ }
97
+ catch {
98
+ resolve(null);
99
+ }
100
+ });
101
+ return opened;
102
+ };
103
+ const prune = async (db) => {
104
+ const tx = db.transaction(STORE, "readwrite");
105
+ const store = tx.objectStore(STORE);
106
+ const count = await request(store.count());
107
+ if (count > maxEntries) {
108
+ let drop = Math.ceil(count / 4);
109
+ const cursor = store.index("at").openCursor();
110
+ await new Promise((resolve, reject) => {
111
+ cursor.onerror = () => reject(cursor.error);
112
+ cursor.onsuccess = () => {
113
+ const at = cursor.result;
114
+ if (!at || drop-- <= 0)
115
+ return resolve();
116
+ at.delete();
117
+ at.continue();
118
+ };
119
+ });
120
+ }
121
+ await done(tx);
122
+ };
123
+ return {
124
+ async get(keys) {
125
+ const out = new Map();
126
+ try {
127
+ const db = await open();
128
+ if (!db || keys.length === 0)
129
+ return out;
130
+ const tx = db.transaction(STORE, "readonly");
131
+ const store = tx.objectStore(STORE);
132
+ const rows = await Promise.all(keys.map((key) => request(store.get(key))));
133
+ for (const row of rows)
134
+ if (row)
135
+ out.set(row.k, row.sha);
136
+ }
137
+ catch {
138
+ // a refused read is a miss
139
+ }
140
+ return out;
141
+ },
142
+ async set(entries) {
143
+ try {
144
+ const db = await open();
145
+ if (!db || entries.length === 0)
146
+ return;
147
+ const tx = db.transaction(STORE, "readwrite");
148
+ const store = tx.objectStore(STORE);
149
+ const at = Date.now();
150
+ for (const [k, sha] of entries)
151
+ store.put({ k, sha, at });
152
+ await done(tx);
153
+ await prune(db);
154
+ }
155
+ catch {
156
+ // a refused write only costs the next re-import a hash
157
+ }
158
+ },
159
+ };
160
+ }
@@ -0,0 +1,112 @@
1
+ /**
2
+ * Decide, for a drop of files, which ones to SEND — before sending any (forge task 3579).
3
+ *
4
+ * The order of the work is the whole design, cheapest question first:
5
+ *
6
+ * 1. **Sizes, one request.** A file whose byte size the library holds nothing of — and that no
7
+ * other file in this drop shares — cannot be a copy of anything. It is uploaded at once, never
8
+ * hashed here (binary-server hashes what it receives anyway). A folder of new videos is decided
9
+ * almost entirely by this step.
10
+ * 2. **The memo.** A file this browser (or script) hashed before, at the same path with the same
11
+ * size and date, takes its remembered hash — a second import of the same huge folder reads none
12
+ * of its bytes.
13
+ * 3. **Hash the rest**, a few at a time and within a memory budget, and ask the library about the
14
+ * hashes AS they finish: one request in flight, each carrying every hash completed since the
15
+ * last. Uploads of the first new files begin while later ones are still being hashed.
16
+ * 4. **A repeat inside the drop** (the same bytes under two names) is caught by its hash without
17
+ * asking anybody, and only the first is sent.
18
+ *
19
+ * A pre-check that FAILS never blocks an upload: the sizes question failing sends everything as
20
+ * it would have before the pre-check existed (each app's own after-the-fact check still stands),
21
+ * and is reported as `degraded`. It costs bandwidth, never a file.
22
+ */
23
+ import { type PrecheckHit, type PrecheckRequest, type PrecheckResponse } from "./contract.js";
24
+ import { type HashOptions } from "./hash.js";
25
+ import { type HashMemo } from "./memo.js";
26
+ /** Asks the app's pre-check route. Each app supplies its own fetch. */
27
+ export type PrecheckAsk = (request: PrecheckRequest) => Promise<PrecheckResponse>;
28
+ export type Verdict<T> =
29
+ /** Send it. `sha256` is set when it was hashed on the way (an app may pass it on). */
30
+ {
31
+ kind: "upload";
32
+ item: T;
33
+ sha256?: string;
34
+ }
35
+ /** The library already answers for these bytes. Do not send. */
36
+ | {
37
+ kind: "duplicate";
38
+ item: T;
39
+ sha256: string;
40
+ hit: PrecheckHit;
41
+ }
42
+ /** Another file in this same drop has these bytes; only that one is sent. */
43
+ | {
44
+ kind: "repeat";
45
+ item: T;
46
+ sha256: string;
47
+ of: T;
48
+ };
49
+ export interface PlanStats {
50
+ files: number;
51
+ bytes: number;
52
+ /** Files sent without being hashed, because their size matched nothing. */
53
+ sizeOnly: number;
54
+ /** Files whose hash came from the memo — no bytes read. */
55
+ memoHits: number;
56
+ /** Files hashed here, and how many bytes that read. */
57
+ hashedFiles: number;
58
+ hashedBytes: number;
59
+ /** Requests made to the pre-check route. */
60
+ asks: number;
61
+ /** Bytes the plan kept from being sent. */
62
+ savedBytes: number;
63
+ /** Files whose hash failed (an unreadable file) — sent unhashed rather than ending the plan. */
64
+ hashErrors: number;
65
+ /** Set when the pre-check could not be asked; the plan then sent what it could not rule out. */
66
+ degraded?: string;
67
+ }
68
+ export interface PlanResult<T> {
69
+ upload: Extract<Verdict<T>, {
70
+ kind: "upload";
71
+ }>[];
72
+ duplicates: Extract<Verdict<T>, {
73
+ kind: "duplicate";
74
+ }>[];
75
+ repeats: Extract<Verdict<T>, {
76
+ kind: "repeat";
77
+ }>[];
78
+ stats: PlanStats;
79
+ }
80
+ export interface PlanOptions<T> {
81
+ items: readonly T[];
82
+ /** The bytes of an item. Its `size` is the size asked about. */
83
+ blobOf: (item: T) => Blob;
84
+ /** The item's path inside the drop (or its name) — the memo's key. Without it there is no memo. */
85
+ pathOf?: (item: T) => string;
86
+ /** Modification time in ms (a `File`'s `lastModified`). Part of the memo's key. */
87
+ lastModifiedOf?: (item: T) => number | undefined;
88
+ /** A script's {@link statIdentity} of the item — part of the memo's key where it is known. */
89
+ identityOf?: (item: T) => string | undefined;
90
+ ask: PrecheckAsk;
91
+ memo?: HashMemo;
92
+ /** Defaults to {@link sha256OfBlob}. The specs count calls through it. */
93
+ hash?: (blob: Blob, options: HashOptions) => Promise<string>;
94
+ /** Files hashed at once. Default 3 — a disk is the bottleneck past that. */
95
+ concurrency?: number;
96
+ /** Bytes held in memory by hashes in flight. Default 320 MiB; one file is always allowed. */
97
+ maxInFlightBytes?: number;
98
+ /**
99
+ * How long a finished hash waits for company before it is asked about (default 250 ms), unless
100
+ * {@link batchSize} are already waiting or the hashing is over. Bounds a drop of 10,000 small
101
+ * files to about one request per batch rather than one per file, at a quarter-second's delay.
102
+ */
103
+ lingerMs?: number;
104
+ /** Hashes that are asked about at once without waiting out the linger. Default 200. */
105
+ batchSize?: number;
106
+ signal?: AbortSignal;
107
+ /** Each verdict as it is reached — start uploading on the first `upload`, not at the end. */
108
+ onVerdict?: (verdict: Verdict<T>) => void;
109
+ /** Hashing progress of one item, 0..1. */
110
+ onHashProgress?: (item: T, fraction: number) => void;
111
+ }
112
+ export declare function planUploads<T>(options: PlanOptions<T>): Promise<PlanResult<T>>;