cursedbelt-core 2.13.3 → 2.14.1

package/src/core/upload-precheck/plan.ts

@@ -0,0 +1,345 @@
+/**
+ * Decide, for a drop of files, which ones to SEND — before sending any (forge task 3579).
+ *
+ * The order of the work is the whole design, cheapest question first:
+ *
+ * 1. **Sizes, one request.** A file whose byte size the library holds nothing of — and that no
+ *    other file in this drop shares — cannot be a copy of anything. It is uploaded at once, never
+ *    hashed here (binary-server hashes what it receives anyway). A folder of new videos is decided
+ *    almost entirely by this step.
+ * 2. **The memo.** A file this browser (or script) hashed before, at the same path with the same
+ *    size and date, takes its remembered hash — a second import of the same huge folder reads none
+ *    of its bytes.
+ * 3. **Hash the rest**, a few at a time and within a memory budget, and ask the library about the
+ *    hashes AS they finish: one request in flight, each carrying every hash completed since the
+ *    last. Uploads of the first new files begin while later ones are still being hashed.
+ * 4. **A repeat inside the drop** (the same bytes under two names) is caught by its hash without
+ *    asking anybody, and only the first IN THE DROP is sent — never whichever hash happened to
+ *    finish first. Hashes finish out of order (a busy disk, `crypto.subtle` on its own threads), so
+ *    a file's hash is held until every earlier file of its SIZE has one: only those can be its
+ *    repeat, and they started hashing before it did. music's gate went red on exactly that race
+ *    (forge task 4407): "song again.wav" sent and "song.wav" called the copy.
+ *
+ * A pre-check that FAILS never blocks an upload: the sizes question failing sends everything as
+ * it would have before the pre-check existed (each app's own after-the-fact check still stands),
+ * and is reported as `degraded`. It costs bandwidth, never a file.
+ */
+import { PRECHECK_MAX_HASHES, PRECHECK_MAX_SIZES, type PrecheckHit, type PrecheckRequest, type PrecheckResponse } from "./contract.js";
+import { type HashOptions, NATIVE_HASH_MAX_BYTES, STREAM_SLICE_BYTES, sha256OfBlob } from "./hash.js";
+import { type HashMemo, memoKey } from "./memo.js";
+
+/** Asks the app's pre-check route. Each app supplies its own fetch. */
+export type PrecheckAsk = (request: PrecheckRequest) => Promise<PrecheckResponse>;
+
+export type Verdict<T> =
+	/** Send it. `sha256` is set when it was hashed on the way (an app may pass it on). */
+	| { kind: "upload"; item: T; sha256?: string }
+	/** The library already answers for these bytes. Do not send. */
+	| { kind: "duplicate"; item: T; sha256: string; hit: PrecheckHit }
+	/** Another file in this same drop has these bytes; only that one is sent. */
+	| { kind: "repeat"; item: T; sha256: string; of: T };
+
+export interface PlanStats {
+	files: number;
+	bytes: number;
+	/** Files sent without being hashed, because their size matched nothing. */
+	sizeOnly: number;
+	/** Files whose hash came from the memo — no bytes read. */
+	memoHits: number;
+	/** Files hashed here, and how many bytes that read. */
+	hashedFiles: number;
+	hashedBytes: number;
+	/** Requests made to the pre-check route. */
+	asks: number;
+	/** Bytes the plan kept from being sent. */
+	savedBytes: number;
+	/** Files whose hash failed (an unreadable file) — sent unhashed rather than ending the plan. */
+	hashErrors: number;
+	/** Set when the pre-check could not be asked; the plan then sent what it could not rule out. */
+	degraded?: string;
+}
+
+export interface PlanResult<T> {
+	upload: Extract<Verdict<T>, { kind: "upload" }>[];
+	duplicates: Extract<Verdict<T>, { kind: "duplicate" }>[];
+	repeats: Extract<Verdict<T>, { kind: "repeat" }>[];
+	stats: PlanStats;
+}
+
+export interface PlanOptions<T> {
+	items: readonly T[];
+	/** The bytes of an item. Its `size` is the size asked about. */
+	blobOf: (item: T) => Blob;
+	/** The item's path inside the drop (or its name) — the memo's key. Without it there is no memo. */
+	pathOf?: (item: T) => string;
+	/** Modification time in ms (a `File`'s `lastModified`). Part of the memo's key. */
+	lastModifiedOf?: (item: T) => number | undefined;
+	/** A script's {@link statIdentity} of the item — part of the memo's key where it is known. */
+	identityOf?: (item: T) => string | undefined;
+	ask: PrecheckAsk;
+	memo?: HashMemo;
+	/** Defaults to {@link sha256OfBlob}. The specs count calls through it. */
+	hash?: (blob: Blob, options: HashOptions) => Promise<string>;
+	/** Files hashed at once. Default 3 — a disk is the bottleneck past that. */
+	concurrency?: number;
+	/** Bytes held in memory by hashes in flight. Default 320 MiB; one file is always allowed. */
+	maxInFlightBytes?: number;
+	/**
+	 * How long a finished hash waits for company before it is asked about (default 250 ms), unless
+	 * {@link batchSize} are already waiting or the hashing is over. Bounds a drop of 10,000 small
+	 * files to about one request per batch rather than one per file, at a quarter-second's delay.
+	 */
+	lingerMs?: number;
+	/** Hashes that are asked about at once without waiting out the linger. Default 200. */
+	batchSize?: number;
+	signal?: AbortSignal;
+	/** Each verdict as it is reached — start uploading on the first `upload`, not at the end. */
+	onVerdict?: (verdict: Verdict<T>) => void;
+	/** Hashing progress of one item, 0..1. */
+	onHashProgress?: (item: T, fraction: number) => void;
+}
+
+function chunks<V>(values: readonly V[], size: number): V[][] {
+	const out: V[][] = [];
+	for (let i = 0; i < values.length; i += size) out.push(values.slice(i, i + size));
+	return out;
+}
+
+const message = (error: unknown): string => (error instanceof Error ? error.message : String(error));
+
+export async function planUploads<T>(options: PlanOptions<T>): Promise<PlanResult<T>> {
+	const {
+		items,
+		blobOf,
+		pathOf,
+		lastModifiedOf,
+		identityOf,
+		ask,
+		memo,
+		hash = sha256OfBlob,
+		concurrency = 3,
+		maxInFlightBytes = 320 * 1024 * 1024,
+		lingerMs = 250,
+		batchSize = 200,
+		signal,
+		onVerdict,
+		onHashProgress,
+	} = options;
+
+	const result: PlanResult<T> = {
+		upload: [],
+		duplicates: [],
+		repeats: [],
+		stats: {
+			files: items.length,
+			bytes: 0,
+			sizeOnly: 0,
+			memoHits: 0,
+			hashedFiles: 0,
+			hashedBytes: 0,
+			asks: 0,
+			savedBytes: 0,
+			hashErrors: 0,
+		},
+	};
+	const decide = (verdict: Verdict<T>): void => {
+		if (verdict.kind === "upload") result.upload.push(verdict);
+		else {
+			result.stats.savedBytes += blobOf(verdict.item).size;
+			if (verdict.kind === "duplicate") result.duplicates.push(verdict);
+			else result.repeats.push(verdict);
+		}
+		onVerdict?.(verdict);
+	};
+
+	const sizeOf = (item: T): number => blobOf(item).size;
+	const inDrop = new Map<number, number>();
+	for (const item of items) {
+		const size = sizeOf(item);
+		result.stats.bytes += size;
+		inDrop.set(size, (inDrop.get(size) ?? 0) + 1);
+	}
+	if (items.length === 0) return result;
+
+	// 1 — sizes
+	const held = new Set<number>();
+	try {
+		for (const batch of chunks([...inDrop.keys()], PRECHECK_MAX_SIZES)) {
+			result.stats.asks++;
+			for (const size of (await ask({ sizes: batch })).sizes) held.add(size);
+		}
+	} catch (error) {
+		if (signal?.aborted) throw error;
+		result.stats.degraded = `pre-check unavailable (${message(error)}) — sending without it`;
+		for (const item of items) decide({ kind: "upload", item });
+		return result;
+	}
+
+	const candidates: T[] = [];
+	for (const item of items) {
+		const size = sizeOf(item);
+		if (held.has(size) || (inDrop.get(size) ?? 0) > 1) candidates.push(item);
+		else {
+			result.stats.sizeOnly++;
+			decide({ kind: "upload", item });
+		}
+	}
+	if (candidates.length === 0) return result;
+
+	// 2 — the memo
+	const keyOf = pathOf
+		? (item: T) => memoKey(pathOf(item), sizeOf(item), lastModifiedOf?.(item), identityOf?.(item))
+		: undefined;
+	const remembered = memo && keyOf ? await memo.get(candidates.map(keyOf)) : new Map<string, string>();
+
+	// 3 + 4 — hash, and ask as hashes finish
+	const firstWith = new Map<string, T>();
+	let pending: { item: T; sha256: string }[] = [];
+	let toRemember: [string, string][] = [];
+	let asking: Promise<void> | undefined;
+	let askFailed: string | undefined;
+	let hashingDone = false;
+	let linger: ReturnType<typeof setTimeout> | undefined;
+
+	const flush = async (): Promise<void> => {
+		while (pending.length > 0) {
+			const batch = pending.splice(0, PRECHECK_MAX_HASHES);
+			const remember = toRemember;
+			toRemember = [];
+			if (askFailed) {
+				for (const { item, sha256 } of batch) decide({ kind: "upload", item, sha256 });
+				continue;
+			}
+			let have: Record<string, PrecheckHit> = {};
+			try {
+				result.stats.asks++;
+				have = (await ask({ sha256: [...new Set(batch.map((b) => b.sha256))] })).have;
+			} catch (error) {
+				if (signal?.aborted) throw error;
+				askFailed = message(error);
+				result.stats.degraded = `pre-check failed mid-drop (${askFailed}) — sent the rest without it`;
+			}
+			for (const { item, sha256 } of batch) {
+				const hit = have[sha256];
+				decide(hit ? { kind: "duplicate", item, sha256, hit } : { kind: "upload", item, sha256 });
+			}
+			if (memo && remember.length > 0) await memo.set(remember);
+		}
+	};
+	const kick = (): void => {
+		if (linger !== undefined) {
+			clearTimeout(linger);
+			linger = undefined;
+		}
+		if (asking) return;
+		asking = flush().finally(() => {
+			asking = undefined;
+			if (pending.length > 0) kick();
+		});
+		// Awaited at the end; this keeps an abort mid-hashing from surfacing as unhandled first.
+		asking.catch(() => {});
+	};
+	const settle = (item: T, sha256: string): void => {
+		const first = firstWith.get(sha256);
+		if (first !== undefined) {
+			decide({ kind: "repeat", item, sha256, of: first });
+			return;
+		}
+		firstWith.set(sha256, item);
+		pending.push({ item, sha256 });
+		if (hashingDone || pending.length >= batchSize) kick();
+		else linger ??= setTimeout(kick, lingerMs);
+	};
+
+	// Step 4's drop order: each size shared inside the drop is a line, in drop order, and a hash
+	// (or a failure to read, `null`) is settled only once everything ahead of it in its line has one.
+	const lines = new Map<number, T[]>();
+	for (const item of candidates) {
+		const size = sizeOf(item);
+		if ((inDrop.get(size) ?? 0) > 1) lines.set(size, [...(lines.get(size) ?? []), item]);
+	}
+	const arrived = new Map<T, string | null>();
+	const apply = (item: T, sha256: string | null): void => {
+		if (sha256 !== null) settle(item, sha256);
+		else decide({ kind: "upload", item });
+	};
+	const arrive = (item: T, sha256: string | null): void => {
+		const line = lines.get(sizeOf(item));
+		if (!line) {
+			apply(item, sha256);
+			return;
+		}
+		arrived.set(item, sha256);
+		while (line.length > 0 && arrived.has(line[0] as T)) {
+			const head = line.shift() as T;
+			apply(head, arrived.get(head) ?? null);
+			arrived.delete(head);
+		}
+	};
+
+	const toHash: T[] = [];
+	for (const item of candidates) {
+		const known = keyOf ? remembered.get(keyOf(item)) : undefined;
+		if (known) {
+			result.stats.memoHits++;
+			arrive(item, known);
+		} else toHash.push(item);
+	}
+
+	let next = 0;
+	let inFlightBytes = 0;
+	let running = 0;
+	const footprint = (size: number): number =>
+		size <= NATIVE_HASH_MAX_BYTES ? size : STREAM_SLICE_BYTES;
+	await new Promise<void>((resolve, reject) => {
+		let failed = false;
+		const pump = (): void => {
+			if (failed) return;
+			if (next >= toHash.length && running === 0) {
+				resolve();
+				return;
+			}
+			while (next < toHash.length && running < concurrency) {
+				const item = toHash[next];
+				const blob = blobOf(item);
+				const cost = footprint(blob.size);
+				if (running > 0 && inFlightBytes + cost > maxInFlightBytes) break;
+				next++;
+				running++;
+				inFlightBytes += cost;
+				hash(blob, { signal, onProgress: onHashProgress ? (f) => onHashProgress(item, f) : undefined })
+					.then((sha256) => {
+						result.stats.hashedFiles++;
+						result.stats.hashedBytes += blob.size;
+						if (keyOf) toRemember.push([keyOf(item), sha256]);
+						arrive(item, sha256);
+					})
+					.catch((error) => {
+						// An abort ends the plan. One file that cannot be READ does not: it is sent
+						// unhashed like any file the pre-check cannot rule out, and its own upload is
+						// where the person learns it is unreadable (roms, forge task 3579).
+						if (signal?.aborted) {
+							failed = true;
+							reject(error);
+							return;
+						}
+						result.stats.hashErrors++;
+						arrive(item, null);
+					})
+					.finally(() => {
+						running--;
+						inFlightBytes -= cost;
+						pump();
+					});
+			}
+		};
+		pump();
+	}).catch((error) => {
+		if (linger !== undefined) clearTimeout(linger);
+		throw error;
+	});
+	hashingDone = true;
+	if (pending.length > 0) kick();
+	while (asking) await asking;
+	return result;
+}

package/src/core/upload-precheck/sha256.spec.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from "bun:test";
+import { Sha256 } from "./sha256.js";
+
+const subtle = async (bytes: Uint8Array<ArrayBuffer>): Promise<string> =>
+	[...new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))].map((b) => b.toString(16).padStart(2, "0")).join("");
+
+const bytesOf = (n: number): Uint8Array<ArrayBuffer> => {
+	const out = new Uint8Array(n);
+	let x = 2463534242;
+	for (let i = 0; i < n; i++) {
+		x ^= x << 13;
+		x ^= x >>> 17;
+		x ^= x << 5;
+		out[i] = x & 0xff;
+	}
+	return out;
+};
+
+describe("Sha256", () => {
+	test("the empty message and 'abc' — the FIPS vectors", () => {
+		expect(new Sha256().hex()).toBe("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+		expect(new Sha256().update(new TextEncoder().encode("abc")).hex()).toBe(
+			"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+		);
+	});
+
+	// 55/56/64 are the padding boundaries: one length byte fits, does not fit, a whole block.
+	for (const n of [1, 55, 56, 63, 64, 65, 127, 128, 1000]) {
+		test(`agrees with WebCrypto on ${n} bytes, fed whole and byte by byte`, async () => {
+			const data = bytesOf(n);
+			const whole = new Sha256().update(data).hex();
+			const drip = new Sha256();
+			for (let i = 0; i < n; i++) drip.update(data.subarray(i, i + 1));
+			expect(whole).toBe(await subtle(data));
+			expect(drip.hex()).toBe(whole);
+		});
+	}
+
+	test("a megabyte split at an odd offset — the shape a resumed download has", async () => {
+		const data = bytesOf(1024 * 1024 + 7);
+		const hasher = new Sha256();
+		hasher.update(data.subarray(0, 333_333));
+		hasher.update(data.subarray(333_333, 700_001));
+		hasher.update(data.subarray(700_001));
+		expect(hasher.hex()).toBe(await subtle(data));
+	});
+});

package/src/core/upload-precheck/sha256.ts

@@ -0,0 +1,127 @@
+/**
+ * An INCREMENTAL SHA-256 — fed chunk by chunk, for a Blob too big to hold in one buffer.
+ *
+ * 🔴 Why not `crypto.subtle.digest`: it takes the whole message in one buffer, and a PS2 disc is
+ * up to 8.5 GB, a camera video several. Born in roms (`src/ps2/sha256.ts`, the disc download's
+ * verifier) and moved here so every app's upload pre-check hashes with the same code
+ * (forge task 3579). `sha256.spec.ts` pins it against WebCrypto across every chunk boundary that
+ * matters (55/56/64 bytes, and a 1 MB split at an odd offset).
+ *
+ * Measured on this Mac under Bun, 256 MiB in 8 MiB slices: 184 MiB/s here, 195 MiB/s for
+ * `@noble/hashes`, 1,960 MiB/s for `crypto.subtle` — which is why {@link sha256OfBlob} only falls
+ * back to this past the size it is willing to read into memory at once.
+ */
+
+const K = new Uint32Array([
+	0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98,
+	0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786,
+	0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8,
+	0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+	0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819,
+	0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a,
+	0x5b9cca4f, 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7,
+	0xc67178f2,
+]);
+
+const INITIAL = [0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19];
+
+export class Sha256 {
+	private readonly h = new Uint32Array(INITIAL);
+
+	/** Back to the empty message — a download that has to start over. */
+	reset(): this {
+		this.h.set(INITIAL);
+		this.tailLength = 0;
+		this.length = 0;
+		return this;
+	}
+	private readonly w = new Uint32Array(64);
+	private readonly tail = new Uint8Array(64);
+	private tailLength = 0;
+	/** Total bytes fed, as a double — exact to 2^53, far past any disc. */
+	private length = 0;
+
+	update(data: Uint8Array): this {
+		this.length += data.length;
+		let offset = 0;
+		if (this.tailLength > 0) {
+			const take = Math.min(64 - this.tailLength, data.length);
+			this.tail.set(data.subarray(0, take), this.tailLength);
+			this.tailLength += take;
+			offset = take;
+			if (this.tailLength < 64) return this;
+			this.block(this.tail, 0);
+			this.tailLength = 0;
+		}
+		const whole = data.length - ((data.length - offset) % 64);
+		for (; offset < whole; offset += 64) this.block(data, offset);
+		if (offset < data.length) {
+			this.tail.set(data.subarray(offset), 0);
+			this.tailLength = data.length - offset;
+		}
+		return this;
+	}
+
+	/** The digest as lowercase hex. The hasher is spent afterwards. */
+	hex(): string {
+		const bits = this.length * 8;
+		const pad = new Uint8Array(this.tailLength < 56 ? 64 : 128);
+		pad.set(this.tail.subarray(0, this.tailLength));
+		pad[this.tailLength] = 0x80;
+		const view = new DataView(pad.buffer);
+		view.setUint32(pad.length - 8, Math.floor(bits / 2 ** 32));
+		view.setUint32(pad.length - 4, bits >>> 0);
+		for (let offset = 0; offset < pad.length; offset += 64) this.block(pad, offset);
+		let out = "";
+		for (const word of this.h) out += word.toString(16).padStart(8, "0");
+		return out;
+	}
+
+	private block(data: Uint8Array, offset: number): void {
+		const w = this.w;
+		for (let i = 0; i < 16; i++) {
+			const j = offset + i * 4;
+			w[i] = (data[j] << 24) | (data[j + 1] << 16) | (data[j + 2] << 8) | data[j + 3];
+		}
+		for (let i = 16; i < 64; i++) {
+			const a = w[i - 15];
+			const b = w[i - 2];
+			const s0 = ((a >>> 7) | (a << 25)) ^ ((a >>> 18) | (a << 14)) ^ (a >>> 3);
+			const s1 = ((b >>> 17) | (b << 15)) ^ ((b >>> 19) | (b << 13)) ^ (b >>> 10);
+			w[i] = (w[i - 16] + s0 + w[i - 7] + s1) | 0;
+		}
+		const h = this.h;
+		let a = h[0];
+		let b = h[1];
+		let c = h[2];
+		let d = h[3];
+		let e = h[4];
+		let f = h[5];
+		let g = h[6];
+		let hh = h[7];
+		for (let i = 0; i < 64; i++) {
+			const S1 = ((e >>> 6) | (e << 26)) ^ ((e >>> 11) | (e << 21)) ^ ((e >>> 25) | (e << 7));
+			const ch = (e & f) ^ (~e & g);
+			const t1 = (hh + S1 + ch + K[i] + w[i]) | 0;
+			const S0 = ((a >>> 2) | (a << 30)) ^ ((a >>> 13) | (a << 19)) ^ ((a >>> 22) | (a << 10));
+			const maj = (a & b) ^ (a & c) ^ (b & c);
+			const t2 = (S0 + maj) | 0;
+			hh = g;
+			g = f;
+			f = e;
+			e = (d + t1) | 0;
+			d = c;
+			c = b;
+			b = a;
+			a = (t1 + t2) | 0;
+		}
+		h[0] = (h[0] + a) | 0;
+		h[1] = (h[1] + b) | 0;
+		h[2] = (h[2] + c) | 0;
+		h[3] = (h[3] + d) | 0;
+		h[4] = (h[4] + e) | 0;
+		h[5] = (h[5] + f) | 0;
+		h[6] = (h[6] + g) | 0;
+		h[7] = (h[7] + hh) | 0;
+	}
+}