@gscdump/engine 0.36.4 → 0.37.1

package/dist/_chunks/coerce.mjs

@@ -0,0 +1,15 @@
+import { coerceBigIntToNumber } from "@gscdump/lakehouse";
+function coerceRow(row) {
+	let mutated = null;
+	for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
+		if (!mutated) mutated = { ...row };
+		mutated[k] = coerceBigIntToNumber(v);
+	}
+	return mutated ?? row;
+}
+function coerceRows(rows) {
+	const out = Array.from({ length: rows.length });
+	for (let i = 0; i < rows.length; i++) out[i] = coerceRow(rows[i]);
+	return out;
+}
+export { coerceRow, coerceRows };

package/dist/_chunks/engine.mjs

@@ -1,9 +1,11 @@
+import { coerceRows } from "./coerce.mjs";
 import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
 import { SCHEMAS, TABLE_METADATA, currentSchemaVersion, dateColumnsFor, dedupeByNaturalKey } from "./schema.mjs";
 import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from "./compaction.mjs";
 import { dateReplaceClause as dateReplaceClause$1 } from "../sql-fragments.mjs";
 import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
 const BUFFER_READ_BATCH_SIZE = 8;
@@ -23,7 +25,7 @@ async function registerBufferedFiles(db, files, dataSource, registered, signal)
 async function encodeBytes(db, table, rows) {
 	const inName = db.makeTempPath("json");
 	const outName = db.makeTempPath("parquet");
-	const jsonBytes = new TextEncoder().encode(JSON.stringify(rows));
+	const jsonBytes = encodeJsonBigintSafe(coerceRows(rows));
 	const registered = [];
 	await db.registerFileBuffer(inName, jsonBytes);
 	registered.push(inName);

package/dist/_chunks/entities.mjs

@@ -1,4 +1,5 @@
 import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
 function isMissingKeyError(e) {
 	if (typeof e !== "object" || e === null) return false;
 	if (e.code === "ENOENT") return true;
@@ -90,7 +91,7 @@ function createQueryDimStore({ dataSource }) {
 				normalizerVersion: records[0]?.normalizer_version ?? 0,
 				intentVersion: records[0]?.intent_version ?? 0
 			};
-			await dataSource.write(queryDimMetaKey(ctx), new TextEncoder().encode(JSON.stringify(meta)));
+			await dataSource.write(queryDimMetaKey(ctx), encodeJsonBigintSafe(meta));
 			return {
 				parquetKey,
 				rowCount: records.length
@@ -311,11 +312,10 @@ function createInspectionStore(opts) {
 				byMonth.get(month).push(r);
 			}
 			for (const [yearMonth, batch] of byMonth) {
-				const shard = {
+				const bytes = encodeJsonBigintSafe({
 					version: 1,
 					records: batch
-				};
-				const bytes = new TextEncoder().encode(JSON.stringify(shard));
+				});
 				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
 				await ds.write(inspectionHistoryShardKey(ctx, yearMonth, batchId), bytes);
 			}
@@ -581,7 +581,7 @@ function createSitemapStore(opts) {
 		return JSON.parse(new TextDecoder().decode(bytes));
 	}
 	async function writeJson(key, value) {
-		await ds.write(key, new TextEncoder().encode(JSON.stringify(value)));
+		await ds.write(key, encodeJsonBigintSafe(value));
 	}
 	return {
 		async writeSnapshot(ctx, records) {
@@ -931,7 +931,7 @@ function createIndexingMetadataStore(opts) {
 			const key = indexingMetadataIndexKey(ctx);
 			const index = await readIndex(key);
 			for (const r of records) index.records[hash(r.url)] = r;
-			await ds.write(key, new TextEncoder().encode(JSON.stringify(index)));
+			await ds.write(key, encodeJsonBigintSafe(index));
 		},
 		async loadIndex(ctx) {
 			return readIndex(indexingMetadataIndexKey(ctx));
@@ -954,7 +954,7 @@ function createEmptyTypesStore(opts) {
 		return JSON.parse(new TextDecoder().decode(bytes));
 	}
 	async function writeDoc(key, doc) {
-		await ds.write(key, new TextEncoder().encode(JSON.stringify(doc)));
+		await ds.write(key, encodeJsonBigintSafe(doc));
 	}
 	return {
 		async load(ctx) {

package/dist/_chunks/sink.d.mts

@@ -1,8 +1,8 @@
 import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { EngineError } from "./errors.mjs";
 import { icebergCreateTable } from "./libs/icebird.mjs";
-import { Result } from "gscdump/result";
 import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, IcebergCatalogConfig, IcebergColumn, IcebergColumn as IcebergColumn$1, IcebergColumnType, IcebergConnection, IcebergConnection as IcebergConnection$1, IcebergListedDataFile, IcebergListedDataFile as IcebergListedDataFile$1, IcebergPartitionField, IcebergPartitionField as IcebergPartitionField$1, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableSpec, IcebergTableSpec as IcebergTableSpec$1, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1, QueryProfiler, catalogCacheScope, connectIcebergCatalog, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables } from "@gscdump/lakehouse";
+import { Result } from "gscdump/result";
 import { icebergAppendRetrying, isCommitRateLimited } from "@gscdump/lakehouse/unsafe-raw";
 import { TableName } from "@gscdump/contracts";
 /** The 9 fact tables that exist as global Iceberg tables. */

package/dist/_chunks/source.mjs

@@ -1,20 +1,8 @@
+import { coerceRows } from "./coerce.mjs";
 import "./layout.mjs";
 import { engineErrors } from "../errors.mjs";
 import { assertDimensionsSupported, getFilterDimensions, pgResolverAdapter, resolveToSQL } from "./resolver.mjs";
 import { runAnalyzerFromSource } from "./dispatch.mjs";
-function coerceRow(row) {
-	let mutated = null;
-	for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
-		if (!mutated) mutated = { ...row };
-		mutated[k] = Number(v);
-	}
-	return mutated ?? row;
-}
-function coerceRows(rows) {
-	const out = Array.from({ length: rows.length });
-	for (let i = 0; i < rows.length; i++) out[i] = coerceRow(rows[i]);
-	return out;
-}
 var AttachedTableMissingError = class extends Error {
 	missing;
 	engineError;
@@ -166,4 +154,4 @@ async function queryComparisonRows(source, current, previous) {
 		previous: previousRows
 	};
 }
-export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, coerceRow, coerceRows, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery };
+export { AttachedTableMissingError, ENGINE_QUERY_CAPABILITIES, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryComparisonRows, queryRows, rewriteForTableSource, runAnalyzerWithEngine, typedQuery };

package/dist/iceberg/index.mjs

@@ -2,8 +2,8 @@ import { TABLE_METADATA } from "../_chunks/schema.mjs";
 import { engineErrors } from "../errors.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
 import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
-import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, catalogCacheScope, connectIcebergCatalog, dropIcebergTables as dropIcebergTables$1, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "@gscdump/lakehouse";
+import { err, ok } from "gscdump/result";
 import { icebergAppendRetrying, isCommitRateLimited } from "@gscdump/lakehouse/unsafe-raw";
 function icebergSchemaFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	return {

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { ENGINE_QUERY_CAPABILITIES, coerceRow, coerceRows, createSqlQuerySource } from "./_chunks/source.mjs";
+import { coerceRow, coerceRows } from "./_chunks/coerce.mjs";
 import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./_chunks/layout.mjs";
 import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
 import { enumeratePartitions } from "./_chunks/compaction.mjs";
@@ -11,6 +11,7 @@ import "./planner.mjs";
 import { createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter } from "./_chunks/resolver.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
+import { ENGINE_QUERY_CAPABILITIES, createSqlQuerySource } from "./_chunks/source.mjs";
 import { err, ok, unwrapResult } from "gscdump/result";
 const NOOP_RESULT = {
 	flushed: 0,

package/dist/rollups.mjs

@@ -2,6 +2,7 @@ import "./_chunks/layout.mjs";
 import { engineErrors } from "./errors.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { createIndexingMetadataStore, createQueryDimStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx, searchType) {
 	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
@@ -68,14 +69,13 @@ async function rebuildRollups(opts) {
 					parquetKey,
 					rowCount: rows.length
 				};
-				const envelope = {
+				const envelopeBytes = encodeJsonBigintSafe({
 					version: 1,
 					id: def.id,
 					builtAt,
 					windowDays: def.windowDays,
 					payload: pointer
-				};
-				const envelopeBytes = new TextEncoder().encode(JSON.stringify(envelope));
+				});
 				const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
 				await opts.dataSource.write(key, envelopeBytes);
 				results.push({
@@ -88,15 +88,13 @@ async function rebuildRollups(opts) {
 				});
 				continue;
 			}
-			const envelope = {
+			const bytes = encodeJsonBigintSafe({
 				version: 1,
 				id: def.id,
 				builtAt,
 				windowDays: def.windowDays,
 				payload
-			};
-			const json = JSON.stringify(envelope);
-			const bytes = new TextEncoder().encode(json);
+			});
 			const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
 			await opts.dataSource.write(key, bytes);
 			results.push({
@@ -872,7 +870,7 @@ async function rebuildCanonicalDailyResumable(opts) {
 			rowCount: 0
 		}
 	};
-	await dataSource.write(rollupKey(ctx, CANONICAL_DAILY_ROLLUP_FINAL_ID, builtAt, searchType), new TextEncoder().encode(JSON.stringify(envelope)));
+	await dataSource.write(rollupKey(ctx, CANONICAL_DAILY_ROLLUP_FINAL_ID, builtAt, searchType), encodeJsonBigintSafe(envelope));
 	return {
 		done: true,
 		nextWindowOffset,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.36.4",
+  "version": "0.37.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.36.4",
-    "@gscdump/lakehouse": "0.36.4",
-    "gscdump": "0.36.4"
+    "@gscdump/contracts": "0.37.1",
+    "gscdump": "0.37.1",
+    "@gscdump/lakehouse": "0.37.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
@@ -191,8 +191,8 @@
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.26.2",
     "hyparquet-writer": "^0.16.1",
-    "icebird": "^0.8.12",
-    "tsx": "^4.22.5",
+    "icebird": "^0.8.13",
+    "tsx": "^4.23.0",
     "unstorage": "^1.17.5",
     "vitest": "^4.1.9"
   },