@gscdump/engine 0.31.9 → 0.32.1

package/dist/adapters/hyparquet.mjs

@@ -1,6 +1,6 @@
 import { SCHEMAS, TABLE_METADATA, dedupeByNaturalKey } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "../_chunks/libs/hyparquet.mjs";
-import { ByteWriter$1 as ByteWriter, parquetWriteRows } from "../_chunks/libs/hyparquet-writer.mjs";
+import { ByteWriter, parquetWriteRows } from "../_chunks/libs/hyparquet-writer.mjs";
 const ROW_GROUP_SIZE = 25e3;
 function basicTypeFor(colType) {
 	if (colType === "VARCHAR") return "STRING";

package/dist/rollups.d.mts

@@ -123,6 +123,15 @@ interface RollupEnvelope<T = unknown> {
 interface ParquetRollupPointer {
   parquetKey: string;
   rowCount: number;
+  /**
+   * MULTI-FILE rollup: when set, the rollup is the UNION of these parquet keys
+   * (disjoint by the grain's partition column, e.g. `date` for the resumable
+   * `query_canonical_daily` build). Readers MUST union all keys; `parquetKey`
+   * stays populated (the first part) for single-file readers. Avoids a JS
+   * merge/re-encode of the whole rollup — the scaling bottleneck for a
+   * cross-invocation resumable build.
+   */
+  parquetKeys?: string[];
 }
 declare function rollupKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string;
 declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string;

package/dist/rollups.mjs

@@ -1,6 +1,6 @@
 import "./_chunks/layout.mjs";
 import { engineErrors } from "./errors.mjs";
-import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
+import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { createIndexingMetadataStore, createQueryDimStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx, searchType) {
@@ -750,7 +750,6 @@ function mapDailyRow(r) {
 		sum_position: Number(r.sum_position)
 	};
 }
-const CANONICAL_DAILY_STAGING_STEM = "query_canonical_daily__staging";
 async function rebuildCanonicalDailyResumable(opts) {
 	const { engine, ctx, dataSource, searchType, builtAt, windowOffset, deadlineMs } = opts;
 	const sType = searchType !== void 0 ? { searchType } : {};
@@ -800,8 +799,8 @@ async function rebuildCanonicalDailyResumable(opts) {
 		}
 	}
 	const nextWindowOffset = i;
-	const stagingKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_STAGING_STEM}__w${windowOffset}`, builtAt, searchType);
-	await dataSource.write(stagingKey, encodeRowsToParquetFlex(batchRows, {
+	const partKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w${windowOffset}`, builtAt, searchType);
+	await dataSource.write(partKey, encodeRowsToParquetFlex(batchRows, {
 		columns: cols,
 		sortKey
 	}));
@@ -812,36 +811,30 @@ async function rebuildCanonicalDailyResumable(opts) {
 		windowsBuilt: nextWindowOffset - windowOffset,
 		rowsWritten: batchRows.length
 	};
-	const stagingPrefix = rollupParquetKey(ctx, `${CANONICAL_DAILY_STAGING_STEM}__`, builtAt, searchType).replace(/__v\d+\.parquet$/, "");
-	const stagingKeys = (await dataSource.list(stagingPrefix.replace(/[^/]*$/, ""))).filter((k) => k.includes(`${CANONICAL_DAILY_STAGING_STEM}__w`) && k.includes(`__v${builtAt}.parquet`));
-	const merged = [];
-	for (const k of stagingKeys) merged.push(...await decodeParquetToRows(await dataSource.read(k), { columns: cols.map((c) => c.name) }));
-	const parquetKey = rollupParquetKey(ctx, CANONICAL_DAILY_ROLLUP_FINAL_ID, builtAt, searchType);
-	await dataSource.write(parquetKey, encodeRowsToParquetFlex(merged, {
-		columns: cols,
-		sortKey
-	}));
+	const partPrefixDir = rollupParquetKey(ctx, CANONICAL_DAILY_PART_STEM, builtAt, searchType).replace(/[^/]*$/, "");
+	const partKeys = (await dataSource.list(partPrefixDir)).filter((k) => k.includes(`${CANONICAL_DAILY_PART_STEM}__w`) && k.endsWith(`__v${builtAt}.parquet`)).sort();
 	const envelope = {
 		version: 1,
 		id: CANONICAL_DAILY_ROLLUP_FINAL_ID,
 		builtAt,
 		windowDays: queryCanonicalDailyRollup.windowDays,
 		payload: {
-			parquetKey,
-			rowCount: merged.length
+			parquetKey: partKeys[0] ?? partKey,
+			parquetKeys: partKeys,
+			rowCount: 0
 		}
 	};
 	await dataSource.write(rollupKey(ctx, CANONICAL_DAILY_ROLLUP_FINAL_ID, builtAt, searchType), new TextEncoder().encode(JSON.stringify(envelope)));
-	await dataSource.delete(stagingKeys);
 	return {
 		done: true,
 		nextWindowOffset,
 		windowsTotal,
 		windowsBuilt: nextWindowOffset - windowOffset,
-		rowsWritten: merged.length
+		rowsWritten: batchRows.length
 	};
 }
 const CANONICAL_DAILY_ROLLUP_FINAL_ID = "query_canonical_daily";
+const CANONICAL_DAILY_PART_STEM = "query_canonical_daily__part";
 const indexingMetadataRollup = {
 	id: "indexing_metadata",
 	windowDays: null,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.31.9",
+  "version": "0.32.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.31.9",
-    "@gscdump/contracts": "0.31.9"
+    "gscdump": "0.32.1",
+    "@gscdump/contracts": "0.32.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
@@ -190,7 +190,7 @@
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.26.1",
     "hyparquet-writer": "^0.16.1",
-    "icebird": "^0.8.10",
+    "icebird": "^0.8.11",
     "tsx": "^4.22.4",
     "unstorage": "^1.17.5",
     "vitest": "^4.1.9"