@gscdump/lakehouse 7.3.0 → 7.3.2

package/dist/append-identity.mjs

@@ -15,12 +15,72 @@ async function hashContent(content) {
 	const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(content));
 	return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
 }
+function* boundedText(parts) {
+	const maxUnits = 65536;
+	let pending = "";
+	for (const part of parts) for (let at = 0; at < part.length;) {
+		const take = Math.min(maxUnits - pending.length, part.length - at);
+		pending += part.slice(at, at + take);
+		at += take;
+		if (pending.length === maxUnits) {
+			yield pending;
+			pending = "";
+		}
+	}
+	if (pending.length) yield pending;
+}
+async function hashFragments(parts, DigestStream) {
+	const digest = new DigestStream("SHA-256");
+	const encoder = new TextEncoderStream();
+	const writer = encoder.writable.getWriter();
+	const pipe = encoder.readable.pipeTo(digest);
+	const result = digest.digest;
+	const finished = Promise.all([pipe, result]);
+	finished.catch(() => {});
+	try {
+		for (const chunk of boundedText(parts)) await writer.write(chunk);
+		await writer.close();
+		const [, hash] = await finished;
+		return Array.from(new Uint8Array(hash), (byte) => byte.toString(16).padStart(2, "0")).join("");
+	} catch (error) {
+		await writer.abort(error).catch(() => {});
+		await Promise.allSettled([pipe, result]);
+		throw error;
+	} finally {
+		writer.releaseLock();
+	}
+}
+async function hashLegacy(records, DigestStream) {
+	const rows = records.map((record) => Object.keys(record).sort().map((key) => `${key}=${String(record[key])}`).join("")).sort();
+	return DigestStream ? hashFragments(rows, DigestStream) : hashContent(rows.join(""));
+}
+function* framedParts(groups) {
+	yield "[";
+	let firstGroup = true;
+	for (const schema of Array.from(groups.keys()).sort()) {
+		if (!firstGroup) yield ",";
+		firstGroup = false;
+		yield "[";
+		yield schema;
+		yield ",[";
+		let firstRow = true;
+		for (const row of groups.get(schema).sort()) {
+			if (!firstRow) yield ",";
+			firstRow = false;
+			yield row;
+		}
+		yield "]]";
+	}
+	yield "]";
+}
 async function deriveAppendIdentity(records) {
 	if (!records.length) return {
 		_tag: "Explicit",
 		appendId: crypto.randomUUID()
 	};
-	const legacyAppendId = await hashContent(records.map((record) => Object.keys(record).sort().map((key) => `${key}=${String(record[key])}`).join("")).sort().join(""));
+	const available = crypto.DigestStream;
+	const DigestStream = typeof available === "function" && typeof TextEncoderStream === "function" ? available : void 0;
+	const legacyAppendId = await hashLegacy(records, DigestStream);
 	const groups = /* @__PURE__ */ new Map();
 	for (const record of records) {
 		if (Object.getPrototypeOf(record) !== Object.prototype && Object.getPrototypeOf(record) !== null) throw new TypeError("Append records contain an unsupported value.");
@@ -31,6 +91,15 @@ async function deriveAppendIdentity(records) {
 		if (group) group.push(row);
 		else groups.set(schema, [row]);
 	}
+	if (DigestStream) {
+		const contentHash = await hashFragments(framedParts(groups), DigestStream);
+		groups.clear();
+		return {
+			_tag: "Content",
+			appendId: `content-v2:${contentHash}`,
+			legacyAppendId
+		};
+	}
 	const content = `[${Array.from(groups, ([schema, rows]) => `[${schema},[${rows.sort().join(",")}]]`).sort().join(",")}]`;
 	groups.clear();
 	return {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "7.3.0",
+  "version": "7.3.2",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",