cursedbelt-core 2.13.3 → 2.14.1

package/src/core/upload-precheck/index.ts

@@ -0,0 +1,46 @@
+/**
+ * `cursedbelt-core/upload-precheck` — skip a duplicate upload by its hash BEFORE sending it
+ * (forge task 3579). Every app's dropzone, uploader and Mac ingest uses this one module, so the
+ * order of the work (sizes → memo → hash → ask) and the route's contract are the same everywhere.
+ *
+ * - `planUploads` — the client side: decide which files to send.
+ * - `parsePrecheckRequest`, `jsonList` — the server side: each app's route parses with the first
+ *   and binds its lists with the second.
+ * - `sha256OfBlob` — the whole-file SHA-256 binary-server records, of a Blob of any size.
+ * - `indexedDbHashMemo`, `memoryHashMemo` — the memo that makes a re-import of the same folder
+ *   read nothing.
+ *
+ * Pure of any framework; a browser bundle and a Bun script may both import it. Its sibling
+ * `cursedbelt-core/duplicates` answers the other question — two files that are LIKELY the same
+ * (a re-encode, a copy under another name) — which is never enough to skip an upload on.
+ */
+export {
+	isSha256Hex,
+	jsonList,
+	PRECHECK_MAX_HASHES,
+	PRECHECK_MAX_SIZES,
+	type ParsedPrecheck,
+	type PrecheckHit,
+	type PrecheckReason,
+	type PrecheckRequest,
+	type PrecheckResponse,
+	parsePrecheckRequest,
+} from "./contract.js";
+export { type HashOptions, NATIVE_HASH_MAX_BYTES, STREAM_SLICE_BYTES, sha256OfBlob } from "./hash.js";
+export {
+	BROWSER_MEMO_MAX_ENTRIES,
+	type HashMemo,
+	indexedDbHashMemo,
+	memoKey,
+	memoryHashMemo,
+	statIdentity,
+} from "./memo.js";
+export {
+	type PlanOptions,
+	type PlanResult,
+	type PlanStats,
+	type PrecheckAsk,
+	planUploads,
+	type Verdict,
+} from "./plan.js";
+export { Sha256 } from "./sha256.js";

package/src/core/upload-precheck/memo.ts

@@ -0,0 +1,170 @@
+/**
+ * The memory that makes a RE-IMPORT cheap: a file's hash, remembered by where it was, how big and
+ * how old — so a second drop of the same huge folder hashes nothing it already hashed.
+ *
+ * Collections' Mac sweeper keeps the same key for its linked folders (`apps/collections/scripts/ingestLinked.ts`:
+ * path, size, mtime) and has re-walked `tod` every sweep on it since forge task 3508. A file whose
+ * path, size AND modification time all match a remembered one is the same file unless something
+ * rewrote it and set its clock back; anything else misses, and is hashed again.
+ *
+ * A memo only ever saves work: a miss, a broken store or a private window means "hash it", never a
+ * wrong answer, because the hash it returns is still checked against the library.
+ */
+
+/** The batch interface — one transaction per batch, not one per file. */
+export interface HashMemo {
+	get(keys: readonly string[]): Promise<Map<string, string>>;
+	set(entries: readonly (readonly [key: string, sha256: string])[]): Promise<void>;
+}
+
+/**
+ * The key a file is remembered under. `path` is its path inside the drop when known, else its name.
+ *
+ * `identity` is what a Mac script can add for free from the `stat` it already took —
+ * {@link statIdentity}. It closes the one gap path+size+date leave: a file DELETED and replaced by a
+ * different one with the same name, the same size and the same modification time to the
+ * millisecond (`cp -p`, or a tool that copies dates). The replacement is a new inode, and any write
+ * moves the change time, which no tool can set back. A browser `File` exposes neither, so the
+ * browser's key is path+size+date alone — two different files agreeing on all three is the case
+ * left, and it takes a deliberately copied date.
+ */
+export function memoKey(path: string, size: number, lastModified: number | undefined, identity?: string): string {
+	const key = `${path}\u0000${size}\u0000${lastModified ?? ""}`;
+	return identity ? `${key}\u0000${identity}` : key;
+}
+
+/**
+ * A file's identity beyond its path, size and date: its inode and its change time (`ctime`), both
+ * in the `stat` a script already has. Measured on the linked `tod` folder (APFS, 2026-10-22): files
+ * ingested that day still carried change times from months before — reading, hashing and filing a
+ * file never moves it — so this costs a re-hash only when something really touched the file.
+ */
+export function statIdentity(stat: { ino: number | bigint; ctimeMs: number }): string {
+	return `${stat.ino}:${Math.trunc(stat.ctimeMs)}`;
+}
+
+/** A memo held in memory — the Mac scripts' and the specs'. */
+export function memoryHashMemo(): HashMemo & { readonly size: number } {
+	const map = new Map<string, string>();
+	return {
+		get size() {
+			return map.size;
+		},
+		async get(keys) {
+			const out = new Map<string, string>();
+			for (const key of keys) {
+				const hit = map.get(key);
+				if (hit) out.set(key, hit);
+			}
+			return out;
+		},
+		async set(entries) {
+			for (const [key, sha] of entries) map.set(key, sha);
+		},
+	};
+}
+
+/** How many files a browser memo remembers before it forgets the oldest quarter. ~150 bytes each. */
+export const BROWSER_MEMO_MAX_ENTRIES = 200_000;
+
+const STORE = "hashes";
+
+interface Row {
+	k: string;
+	sha: string;
+	at: number;
+}
+
+function request<T>(req: IDBRequest<T>): Promise<T> {
+	return new Promise((resolve, reject) => {
+		req.onsuccess = () => resolve(req.result);
+		req.onerror = () => reject(req.error);
+	});
+}
+
+function done(tx: IDBTransaction): Promise<void> {
+	return new Promise((resolve, reject) => {
+		tx.oncomplete = () => resolve();
+		tx.onerror = () => reject(tx.error);
+		tx.onabort = () => reject(tx.error);
+	});
+}
+
+/**
+ * A memo kept in this origin's IndexedDB, so it survives a reload and a re-import next week. Each
+ * app is its own origin, so each keeps its own. Where IndexedDB is missing or refuses (a private
+ * window, a test DOM), every call quietly answers "not remembered" — the upload still works, it
+ * just hashes.
+ */
+export function indexedDbHashMemo(name = "upload-precheck", maxEntries = BROWSER_MEMO_MAX_ENTRIES): HashMemo {
+	let opened: Promise<IDBDatabase | null> | undefined;
+	const open = (): Promise<IDBDatabase | null> => {
+		opened ??= new Promise<IDBDatabase | null>((resolve) => {
+			try {
+				if (typeof indexedDB === "undefined") return resolve(null);
+				const req = indexedDB.open(name, 1);
+				req.onupgradeneeded = () => {
+					const store = req.result.createObjectStore(STORE, { keyPath: "k" });
+					store.createIndex("at", "at");
+				};
+				req.onsuccess = () => resolve(req.result);
+				req.onerror = () => resolve(null);
+				req.onblocked = () => resolve(null);
+			} catch {
+				resolve(null);
+			}
+		});
+		return opened;
+	};
+
+	const prune = async (db: IDBDatabase): Promise<void> => {
+		const tx = db.transaction(STORE, "readwrite");
+		const store = tx.objectStore(STORE);
+		const count = await request(store.count());
+		if (count > maxEntries) {
+			let drop = Math.ceil(count / 4);
+			const cursor = store.index("at").openCursor();
+			await new Promise<void>((resolve, reject) => {
+				cursor.onerror = () => reject(cursor.error);
+				cursor.onsuccess = () => {
+					const at = cursor.result;
+					if (!at || drop-- <= 0) return resolve();
+					at.delete();
+					at.continue();
+				};
+			});
+		}
+		await done(tx);
+	};
+
+	return {
+		async get(keys) {
+			const out = new Map<string, string>();
+			try {
+				const db = await open();
+				if (!db || keys.length === 0) return out;
+				const tx = db.transaction(STORE, "readonly");
+				const store = tx.objectStore(STORE);
+				const rows = await Promise.all(keys.map((key) => request(store.get(key) as IDBRequest<Row | undefined>)));
+				for (const row of rows) if (row) out.set(row.k, row.sha);
+			} catch {
+				// a refused read is a miss
+			}
+			return out;
+		},
+		async set(entries) {
+			try {
+				const db = await open();
+				if (!db || entries.length === 0) return;
+				const tx = db.transaction(STORE, "readwrite");
+				const store = tx.objectStore(STORE);
+				const at = Date.now();
+				for (const [k, sha] of entries) store.put({ k, sha, at } satisfies Row);
+				await done(tx);
+				await prune(db);
+			} catch {
+				// a refused write only costs the next re-import a hash
+			}
+		},
+	};
+}

package/src/core/upload-precheck/plan.spec.ts

@@ -0,0 +1,335 @@
+/**
+ * The owner's case, proved: a SECOND copy of a folder is decided without sending a byte, and —
+ * with the memo — without reading one (forge task 3579).
+ */
+import { describe, expect, test } from "bun:test";
+import type { PrecheckHit, PrecheckRequest, PrecheckResponse } from "./contract.js";
+import { sha256OfBlob } from "./hash.js";
+import { memoryHashMemo, statIdentity } from "./memo.js";
+import { planUploads, type Verdict } from "./plan.js";
+
+interface Dropped {
+	path: string;
+	blob: Blob;
+	lastModified: number;
+}
+
+const file = (path: string, text: string, lastModified = 1): Dropped => ({
+	path,
+	blob: new Blob([text]),
+	lastModified,
+});
+
+/** A library that holds what it is given, and counts what it is asked. */
+function library() {
+	const bySha = new Map<string, PrecheckHit>();
+	const sizes = new Set<number>();
+	const asked: PrecheckRequest[] = [];
+	return {
+		asked,
+		async store(dropped: Dropped) {
+			bySha.set(await sha256OfBlob(dropped.blob), { reason: "exists", name: dropped.path });
+			sizes.add(dropped.blob.size);
+		},
+		async ask(request: PrecheckRequest): Promise<PrecheckResponse> {
+			asked.push(request);
+			const have: Record<string, PrecheckHit> = {};
+			for (const sha of request.sha256 ?? []) {
+				const hit = bySha.get(sha);
+				if (hit) have[sha] = hit;
+			}
+			return { sizes: (request.sizes ?? []).filter((s) => sizes.has(s)), have };
+		},
+	};
+}
+
+function counter() {
+	const hashed: string[] = [];
+	return {
+		hashed,
+		hash: async (blob: Blob) => {
+			hashed.push(await blob.text());
+			return sha256OfBlob(blob);
+		},
+	};
+}
+
+const folder = (): Dropped[] => [
+	file("trip/a.jpg", "alpha alpha"),
+	file("trip/b.jpg", "bravo bravo bravo"),
+	file("trip/c.mov", "charlie charlie charlie charlie"),
+	file("trip/d.mov", "delta"),
+];
+
+async function importInto(lib: ReturnType<typeof library>, drop: Dropped[], extra: Partial<Parameters<typeof planUploads<Dropped>>[0]> = {}) {
+	const plan = await planUploads({
+		items: drop,
+		blobOf: (d) => d.blob,
+		pathOf: (d) => d.path,
+		lastModifiedOf: (d) => d.lastModified,
+		ask: (r) => lib.ask(r),
+		...extra,
+	});
+	for (const { item } of plan.upload) await lib.store(item);
+	return plan;
+}
+
+describe("planUploads", () => {
+	test("a first import of new files hashes NOTHING — no size matches, so nothing could be a copy", async () => {
+		const lib = library();
+		const { hashed, hash } = counter();
+		const plan = await importInto(lib, folder(), { hash });
+		expect(plan.upload).toHaveLength(4);
+		expect(plan.stats.sizeOnly).toBe(4);
+		expect(hashed).toEqual([]);
+		expect(lib.asked).toHaveLength(1); // the one sizes question
+	});
+
+	test("🔴 the second copy of the folder sends nothing, and asks twice in all — sizes once, hashes once", async () => {
+		const lib = library();
+		await importInto(lib, folder());
+		lib.asked.length = 0;
+		const { hashed, hash } = counter();
+		const again = await importInto(lib, folder(), { hash });
+		expect(again.upload).toEqual([]);
+		expect(again.duplicates.map((d) => d.item.path).sort()).toEqual(["trip/a.jpg", "trip/b.jpg", "trip/c.mov", "trip/d.mov"]);
+		expect(again.stats.savedBytes).toBe(again.stats.bytes);
+		expect(hashed).toHaveLength(4);
+		// one sizes ask, then every hash in ONE request — they finished inside the linger, so they
+		// waited for each other rather than going one request per file
+		expect(lib.asked).toHaveLength(2);
+		expect(lib.asked[0]).toEqual({ sizes: expect.any(Array) });
+	});
+
+	test("🔴 with the memo, a re-import of the same folder reads NO bytes at all", async () => {
+		const lib = library();
+		const memo = memoryHashMemo();
+		await importInto(lib, folder(), { memo });
+		// the first import hashed nothing (all new sizes), so prime the memo the way a second
+		// browser session would have: one import that had to hash
+		await importInto(lib, folder(), { memo });
+		expect(memo.size).toBe(4);
+		const { hashed, hash } = counter();
+		const third = await importInto(lib, folder(), { memo, hash });
+		expect(third.upload).toEqual([]);
+		expect(third.stats.memoHits).toBe(4);
+		expect(hashed).toEqual([]);
+	});
+
+	test("a missed file in a huge re-import is the only one sent, and the only one not hashed", async () => {
+		const lib = library();
+		const big = Array.from({ length: 300 }, (_, i) => file(`dump/${i}.mp4`, `clip ${i}`.padEnd(40 + i, "x")));
+		await importInto(lib, big);
+		const missed = file("dump/new.mp4", "the one that was missed".padEnd(1000, "y"));
+		const { hashed, hash } = counter();
+		lib.asked.length = 0;
+		const plan = await importInto(lib, [...big, missed], { hash });
+		// 1 sizes question + 300 hashes in batches of at most 200 — a handful of requests, not 301
+		expect(lib.asked.length).toBeLessThanOrEqual(4);
+		expect(Math.max(...lib.asked.map((a) => a.sha256?.length ?? 0))).toBeLessThanOrEqual(200);
+		expect(plan.upload.map((u) => u.item.path)).toEqual(["dump/new.mp4"]);
+		expect(plan.upload[0].sha256).toBeUndefined();
+		expect(hashed).toHaveLength(300);
+		expect(plan.duplicates).toHaveLength(300);
+	});
+
+	test("same size, different bytes — hashed, and sent", async () => {
+		const lib = library();
+		await importInto(lib, [file("a.txt", "aaaa")]);
+		const plan = await importInto(lib, [file("b.txt", "bbbb")]);
+		expect(plan.upload).toHaveLength(1);
+		expect(plan.upload[0].sha256).toBe(await sha256OfBlob(new Blob(["bbbb"])));
+	});
+
+	test("the same bytes twice in one drop are sent once — the second is a repeat of the first", async () => {
+		const lib = library();
+		const plan = await importInto(lib, [file("one/x.jpg", "same bytes"), file("two/x copy.jpg", "same bytes")]);
+		expect(plan.upload).toHaveLength(1);
+		expect(plan.repeats).toHaveLength(1);
+		expect(plan.repeats[0].of).toBe(plan.upload[0].item);
+	});
+
+	test("🔴 the FIRST in the drop is the one sent, even when its hash finishes last (forge task 4407)", async () => {
+		const lib = library();
+		const drop = [file("A/song.wav", "same bytes"), file("B/song again.wav", "same bytes"), file("C/other.wav", "diff bytes")];
+		const plan = await planUploads({
+			items: drop,
+			blobOf: (d) => d.blob,
+			ask: (r) => lib.ask(r),
+			// The first file's hash is the slow one — a busy disk, or `crypto.subtle` finishing out of order.
+			hash: async (blob) => {
+				const sha = await sha256OfBlob(blob);
+				if (blob === drop[0]?.blob) await new Promise((r) => setTimeout(r, 30));
+				return sha;
+			},
+		});
+		expect(plan.upload.map((u) => u.item.path).sort()).toEqual(["A/song.wav", "C/other.wav"]);
+		expect(plan.repeats.map((r) => [r.item.path, r.of.path])).toEqual([["B/song again.wav", "A/song.wav"]]);
+	});
+
+	test("…and a first that cannot be read still lets its line move — the copy is sent too", async () => {
+		const drop = [file("A/song.wav", "same bytes"), file("B/song again.wav", "same bytes")];
+		const plan = await planUploads({
+			items: drop,
+			blobOf: (d) => d.blob,
+			ask: async (r) => ({ sizes: r.sizes ?? [], have: {} }),
+			hash: async (blob) => {
+				if (blob === drop[0]?.blob) {
+					await new Promise((r) => setTimeout(r, 30));
+					throw new Error("NotReadableError");
+				}
+				return sha256OfBlob(blob);
+			},
+		});
+		expect(plan.upload.map((u) => u.item.path).sort()).toEqual(["A/song.wav", "B/song again.wav"]);
+		expect(plan.stats.hashErrors).toBe(1);
+	});
+
+	test("a changed modification time is a memo miss — the file is hashed again", async () => {
+		const lib = library();
+		const memo = memoryHashMemo();
+		await importInto(lib, [file("a.txt", "aaaa", 1)]);
+		await importInto(lib, [file("a.txt", "aaaa", 1)], { memo });
+		const { hashed, hash } = counter();
+		await importInto(lib, [file("a.txt", "aaaa", 2)], { memo, hash });
+		expect(hashed).toEqual(["aaaa"]);
+	});
+
+	test("🔴 a folder sent again with files ADDED sends exactly the new ones", async () => {
+		const lib = library();
+		const memo = memoryHashMemo();
+		await importInto(lib, folder(), { memo });
+		await importInto(lib, folder(), { memo }); // the memo now holds all four
+		const grown = [...folder(), file("trip/e.jpg", "echo, added later"), file("trip/f.jpg", "foxtrot!")];
+		const plan = await importInto(lib, grown, { memo });
+		expect(plan.upload.map((u) => u.item.path).sort()).toEqual(["trip/e.jpg", "trip/f.jpg"]);
+		expect(plan.duplicates).toHaveLength(4);
+	});
+
+	test("🔴 image.png deleted and replaced by a different image.png is sent — the date moved", async () => {
+		const lib = library();
+		const memo = memoryHashMemo();
+		await importInto(lib, [file("image.png", "the old picture", 1000)]);
+		await importInto(lib, [file("image.png", "the old picture", 1000)], { memo });
+		const plan = await importInto(lib, [file("image.png", "the NEW picture", 2000)], { memo });
+		expect(plan.upload).toHaveLength(1);
+		expect(plan.stats.memoHits).toBe(0);
+	});
+
+	test("🔴 …and with a script's stat identity, even with the same size AND the same date", async () => {
+		const lib = library();
+		const memo = memoryHashMemo();
+		const old = { ...file("image.png", "the old picture", 1000), ino: 11 };
+		const replaced = { ...file("image.png", "the NEW picture", 1000), ino: 12 }; // same size, same date
+		const run = (d: typeof old) =>
+			importInto(lib, [d], { memo, identityOf: (x) => statIdentity({ ino: (x as typeof old).ino, ctimeMs: 5 }) });
+		await importInto(lib, [old]);
+		await run(old);
+		const plan = await run(replaced);
+		expect(plan.upload).toHaveLength(1);
+		expect(plan.stats.memoHits).toBe(0);
+	});
+
+	test("a pre-check that cannot be asked sends everything, hashes nothing, and says so", async () => {
+		const { hashed, hash } = counter();
+		const plan = await planUploads({
+			items: folder(),
+			blobOf: (d) => d.blob,
+			ask: async () => {
+				throw new Error("404");
+			},
+			hash,
+		});
+		expect(plan.upload).toHaveLength(4);
+		expect(hashed).toEqual([]);
+		expect(plan.stats.degraded).toContain("404");
+	});
+
+	test("a hash question that fails mid-drop sends the rest rather than losing them", async () => {
+		const lib = library();
+		await importInto(lib, folder());
+		let calls = 0;
+		const plan = await planUploads({
+			items: folder(),
+			blobOf: (d) => d.blob,
+			ask: async (r) => {
+				if (++calls > 1) throw new Error("503");
+				return lib.ask(r);
+			},
+		});
+		expect(plan.upload).toHaveLength(4);
+		expect(plan.stats.degraded).toContain("503");
+	});
+
+	test("verdicts stream — an upload can start before the last file is hashed", async () => {
+		const lib = library();
+		await importInto(lib, [file("old.bin", "x".repeat(100))]);
+		const order: string[] = [];
+		const slowHash = async (blob: Blob) => {
+			await new Promise((r) => setTimeout(r, blob.size === 100 ? 30 : 0));
+			return sha256OfBlob(blob);
+		};
+		await planUploads({
+			items: [file("old copy.bin", "x".repeat(100)), file("new.bin", "fresh")],
+			blobOf: (d) => d.blob,
+			ask: (r) => lib.ask(r),
+			hash: slowHash,
+			onVerdict: (v: Verdict<Dropped>) => order.push(`${v.kind}:${v.item.path}`),
+		});
+		expect(order).toEqual(["upload:new.bin", "duplicate:old copy.bin"]);
+	});
+
+	test("hashes in flight stay inside the memory budget", async () => {
+		const drop = Array.from({ length: 6 }, (_, i) => file(`f${i}`, String(i).repeat(10)));
+		let live = 0;
+		let peak = 0;
+		await planUploads({
+			items: drop,
+			blobOf: (d) => d.blob,
+			ask: async (r) => ({ sizes: r.sizes ?? [], have: {} }),
+			concurrency: 6,
+			maxInFlightBytes: 25,
+			hash: async (blob) => {
+				peak = Math.max(peak, ++live);
+				await new Promise((r) => setTimeout(r, 5));
+				live--;
+				return sha256OfBlob(blob);
+			},
+		});
+		expect(peak).toBe(2); // 10 + 10 bytes fit in 25; a third would not
+	});
+
+	test("a file that cannot be read is sent unhashed — it does not end the plan for the rest", async () => {
+		const lib = library();
+		await importInto(lib, folder());
+		const plan = await planUploads({
+			items: folder(),
+			blobOf: (d) => d.blob,
+			ask: (r) => lib.ask(r),
+			hash: async (blob) => {
+				if ((await blob.text()) === "delta") throw new Error("NotReadableError");
+				return sha256OfBlob(blob);
+			},
+		});
+		expect(plan.upload.map((u) => u.item.path)).toEqual(["trip/d.mov"]);
+		expect(plan.duplicates).toHaveLength(3);
+		expect(plan.stats.hashErrors).toBe(1);
+	});
+
+	test("an abort stops the hashing and rejects", async () => {
+		const controller = new AbortController();
+		const plan = planUploads({
+			items: [file("a", "same"), file("b", "same"), file("c", "same")],
+			blobOf: (d) => d.blob,
+			ask: async (r) => ({ sizes: r.sizes ?? [], have: {} }),
+			concurrency: 1,
+			signal: controller.signal,
+			hash: async (blob, { signal }) => {
+				controller.abort(new Error("stopped"));
+				if (signal?.aborted) throw signal.reason;
+				return sha256OfBlob(blob);
+			},
+		});
+		await expect(plan).rejects.toThrow("stopped");
+	});
+});