@gscdump/engine 4.2.1 → 4.2.3

package/dist/iceberg/append-sink.mjs

@@ -29,37 +29,36 @@ function toIntPartitionSiteId(value) {
 	if (siteId < INT32_MIN || siteId > INT32_MAX) throw new TypeError(`toRecords: int partition site_id must fit Iceberg INT, got '${String(value)}'`);
 	return siteId;
 }
-function dedupeByIdentity(table, records) {
-	if (records.length < 2) return records;
-	const key = gscDataset(table, "int").tableSpec.identityColumns;
-	const seen = /* @__PURE__ */ new Map();
-	for (const rec of records) {
-		let identity = "";
-		for (let index = 0; index < key.length; index++) {
-			if (index > 0) identity += "\0";
-			identity += `${rec[key[index]] ?? ""}`;
-		}
-		seen.set(identity, rec);
-	}
-	return seen.size === records.length ? records : [...seen.values()];
+function compareIdentityValue(av, bv) {
+	if (av === bv) return 0;
+	if (typeof av === "number" && typeof bv === "number") return av - bv;
+	const as = `${av ?? ""}`;
+	const bs = `${bv ?? ""}`;
+	if (as === bs) return 0;
+	return as < bs ? -1 : 1;
 }
-function sortByClusterKey(table, records) {
-	const cols = gscDataset(table, "int").tableSpec.clusterKey ?? [];
-	if (cols.length === 0 || records.length < 2) return records;
-	return records.slice().sort((a, b) => {
-		for (const col of cols) {
-			const av = a[col];
-			const bv = b[col];
-			if (av === bv) continue;
-			if (av == null) return -1;
-			if (bv == null) return 1;
-			if (typeof av === "number" && typeof bv === "number") return av - bv;
-			const as = String(av);
-			const bs = String(bv);
-			if (as !== bs) return as < bs ? -1 : 1;
+function clusterAndDedupe(table, records) {
+	if (records.length < 2) return records;
+	const spec = gscDataset(table, "int").tableSpec;
+	const cluster = spec.clusterKey ?? [];
+	for (const col of cluster) if (!spec.identityColumns.includes(col)) throw new TypeError(`clusterAndDedupe: clusterKey column '${col}' is not an identity column of '${table}'. Dedupe would silently key on cluster plus identity.`);
+	const order = [...cluster, ...spec.identityColumns.filter((col) => !cluster.includes(col))];
+	const compare = (a, b) => {
+		for (const col of order) {
+			const c = compareIdentityValue(a[col], b[col]);
+			if (c !== 0) return c;
 		}
 		return 0;
-	});
+	};
+	records.sort(compare);
+	let write = 0;
+	for (let read = 0; read < records.length; read++) {
+		const rec = records[read];
+		if (read + 1 < records.length && compare(rec, records[read + 1]) === 0) continue;
+		records[write++] = rec;
+	}
+	records.length = write;
+	return records;
 }
 function toRecords(slice, rows, encoding) {
 	const siteVal = encoding === "int" ? toIntPartitionSiteId(slice.ctx.siteId) : slice.ctx.siteId ?? "";
@@ -115,7 +114,16 @@ function createIcebergAppendSink(options) {
 			}
 			for (const [table, records] of buffers) {
 				if (records.length === 0) continue;
-				const deduped = sortByClusterKey(table, dedupeByIdentity(table, records));
+				let deduped;
+				try {
+					deduped = clusterAndDedupe(table, records);
+				} catch (cause) {
+					failed.push({
+						table,
+						error: engineErrors.sinkTableFlushFailed(table, cause)
+					});
+					continue;
+				}
 				await icebergAppendRetrying({
 					catalog: conn.catalog,
 					namespace: conn.namespace,

package/dist/ingest.mjs

@@ -242,14 +242,20 @@ const STORED_KEY_COLUMNS = Object.fromEntries(Object.keys(TABLE_DIMS).map((table
 function sumByStoredKey(table, rows) {
 	const keyColumns = STORED_KEY_COLUMNS[table];
 	const grouped = /* @__PURE__ */ new Map();
+	const owned = /* @__PURE__ */ new Set();
 	for (const row of rows) {
 		const r = row;
 		const key = JSON.stringify(keyColumns.map((col) => r[col] ?? null));
-		const prior = grouped.get(key);
+		let prior = grouped.get(key);
 		if (!prior) {
-			grouped.set(key, { ...r });
+			grouped.set(key, r);
 			continue;
 		}
+		if (!owned.has(prior)) {
+			prior = { ...prior };
+			owned.add(prior);
+			grouped.set(key, prior);
+		}
 		for (const col of Object.keys(r)) if (ADDITIVE_COLUMN_RE.test(col)) prior[col] = Number(prior[col] ?? 0) + Number(r[col] ?? 0);
 	}
 	return [...grouped.values()];
@@ -299,7 +305,14 @@ function createRowAccumulator(options = {}) {
 		},
 		drain() {
 			const out = /* @__PURE__ */ new Map();
-			for (const [table, byDate] of buckets) out.set(table, new Map([...byDate].map(([date, sourceRows]) => [date, sumByStoredKey(table, sourceRows.values())])));
+			for (const [table, byDate] of buckets) {
+				const summed = /* @__PURE__ */ new Map();
+				for (const [date, sourceRows] of byDate) {
+					summed.set(date, sumByStoredKey(table, sourceRows.values()));
+					byDate.delete(date);
+				}
+				out.set(table, summed);
+			}
 			buckets = /* @__PURE__ */ new Map();
 			latestDate.clear();
 			total = 0;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "4.2.1",
+  "version": "4.2.3",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^4.2.1",
-    "@gscdump/lakehouse": "^4.2.1",
+    "@gscdump/contracts": "^4.2.3",
+    "@gscdump/lakehouse": "^4.2.3",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^4.2.1",
+    "gscdump": "^4.2.3",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },