@gscdump/lakehouse 0.36.4 → 0.37.1

package/dist/_chunks/catalog.mjs

@@ -1,4 +1,16 @@
 import { cachingResolver, icebergAppend, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./libs/icebird.mjs";
+function bigintJsonReplacer(_key, value) {
+	return typeof value === "bigint" ? value.toString() : value;
+}
+function stringifyBigintSafe(value, space) {
+	return JSON.stringify(value, bigintJsonReplacer, space);
+}
+function encodeJsonBigintSafe(value) {
+	return new TextEncoder().encode(stringifyBigintSafe(value));
+}
+function coerceBigIntToNumber(value) {
+	return typeof value === "bigint" ? Number(value) : value;
+}
 async function cacheGet(cache, key, now) {
 	const boxed = await cache.storage.getItem(key).catch(() => null);
 	if (!boxed || typeof boxed.exp !== "number" || boxed.exp <= now) return void 0;
@@ -297,7 +309,7 @@ async function loadSnapshotId(conn, namespace, table, cache, now) {
 	if (cache) {
 		await cachePut(cache, snapshotRefKey(scope, namespace, table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
 		if (snapshotId != null) {
-			if (JSON.stringify(metadata).length <= MAX_CACHED_METADATA_BYTES) await cachePut(cache, metadataRefKey(scope, namespace, table, snapshotId), metadata, METADATA_TTL_MS, now);
+			if (stringifyBigintSafe(metadata).length <= MAX_CACHED_METADATA_BYTES) await cachePut(cache, metadataRefKey(scope, namespace, table, snapshotId), metadata, METADATA_TTL_MS, now);
 		}
 	}
 	return {
@@ -373,4 +385,4 @@ async function resolveIcebergDataFiles(conn, opts) {
 	}
 	return out;
 }
-export { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe };

package/dist/_chunks/libs/icebird.mjs

@@ -527,7 +527,7 @@ function icebergLatestVersion({ tableUrl, resolver, lister }) {
 			return versions[versions.length - 1];
 		});
 	}).catch((err) => {
-		throw new Error(`failed to determine latest iceberg version: ${err.message}`);
+		throw new Error(`failed to determine latest iceberg version of ${tableUrl}: ${err.message}`);
 	});
 }
 async function resolveMetadata({ tableUrl, metadataFileName, resolver, lister }) {

package/dist/index.d.mts

@@ -1,5 +1,43 @@
 import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
+/**
+ * BigInt at JSON boundaries — the single place this policy lives.
+ *
+ * int64 reaches serialization from two origins:
+ *  1. DuckDB `SUM`/`COUNT` aggregates — already coerced to `number` at the
+ *     source-factory boundary (`@gscdump/engine` `coerceRows`), so rows are
+ *     BigInt-free by the time they cross the query boundary.
+ *  2. icebird / REST-catalog metadata (`current-snapshot-id`, `sequence-number`,
+ *     `snapshot-id`) and other identity ids — these never pass through a source,
+ *     so they arrive at envelope / cache / log serializers still as BigInt.
+ *
+ * A plain `JSON.stringify` throws `Do not know how to serialize a BigInt`, which
+ * is how a stray identity int64 turns into a request 500. Any boundary that can
+ * see origin (2) must serialize through here.
+ *
+ * TWO policies, because the correct target type depends on what the value is:
+ *
+ *  - Identity / precision values (snapshot ids, large row ids) can exceed 2^53
+ *    and must round-trip losslessly → serialize as a decimal STRING. Use
+ *    {@link stringifyBigintSafe} / {@link bigintJsonReplacer} / {@link encodeJsonBigintSafe}
+ *    for storage / transport / log serialization. Where values are already
+ *    number-coerced (origin 1), the replacer is a no-op backstop.
+ *
+ *  - Analytics aggregates that you want as real numbers downstream → coerce with
+ *    {@link coerceBigIntToNumber}. Lossy above 2^53 by design; click / impression
+ *    columns never reach that range. This is the scalar behind engine `coerceRows`.
+ */
+/** `JSON.stringify` replacer: BigInt → decimal string (lossless). Pass-through otherwise. */
+declare function bigintJsonReplacer(_key: string, value: unknown): unknown;
+/** `JSON.stringify` that never throws on BigInt; identity ids stay lossless as strings. */
+declare function stringifyBigintSafe(value: unknown, space?: string | number): string;
+/** UTF-8 JSON bytes, BigInt-safe — the payload shape for `DataSource.write`. */
+declare function encodeJsonBigintSafe(value: unknown): Uint8Array;
+/**
+ * BigInt → number for analytics aggregates (< 2^53). Lossy above; acceptable for
+ * click / impression sums. Non-BigInt values pass through untouched.
+ */
+declare function coerceBigIntToNumber<T>(value: T): T | number;
 /**
  * Closed identity shapes (ADR-0021 amendments 2 + 4). `'site-int'` is the
  * team-scoped numeric Catalog Site Id (nuxtseo ADR-0091) stored under the
@@ -156,4 +194,4 @@ interface RunPyIcebergWriterOptions {
 }
 declare function resolvePyIcebergPython(override?: string): string;
 declare function runPyIcebergWriter<T extends PyIcebergWriterResult>(options: RunPyIcebergWriterOptions): Promise<T>;
-export { type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type PyIcebergWriterResult, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, type RunPyIcebergWriterOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, toIcebergDayCount };
+export { type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type PyIcebergWriterResult, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, type RunPyIcebergWriterOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, stringifyBigintSafe, toIcebergDayCount };

package/dist/index.mjs

@@ -1,5 +1,5 @@
+import { ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe } from "./_chunks/catalog.mjs";
 import { icebergCreateTable } from "./_chunks/libs/icebird.mjs";
-import { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
 import process from "node:process";
 const INT32_MIN = -2147483648;
 const INT32_MAX = 2147483647;
@@ -113,9 +113,6 @@ function toIcebergDayCount(value) {
 	if (Number.isNaN(ms)) throw new TypeError(`toIcebergDayCount: invalid date string '${value}'`);
 	return Math.floor(ms / DAY_MILLIS);
 }
-function coerceJsonSafe(value) {
-	return typeof value === "bigint" ? Number(value) : value;
-}
 function asInt32(value) {
 	if (typeof value !== "string" && typeof value !== "number" && typeof value !== "bigint") return null;
 	const n = Number(value);
@@ -141,7 +138,7 @@ function buildRowProcessor(def, tableSpec) {
 			}
 		}
 		for (const name of dimNames) out[name] = def.dims[name].toPartitionValue(String(row[name]));
-		for (const col of def.columns) out[col.name] = coerceJsonSafe(row[col.name]);
+		for (const col of def.columns) out[col.name] = coerceBigIntToNumber(row[col.name]);
 		return out;
 	}
 	function dedupe(rows) {
@@ -353,4 +350,4 @@ async function runPyIcebergWriter(options) {
 	});
 }
 const DEFAULT_PARTITION_KEY_ENCODING = "int";
-export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, toIcebergDayCount };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, stringifyBigintSafe, toIcebergDayCount };

package/dist/unsafe-raw.mjs

@@ -1,5 +1,5 @@
-import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
 import { icebergAppendRetrying, isCommitRateLimited } from "./_chunks/catalog.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
 const HOUR_MS = 3600 * 1e3;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "0.36.4",
+  "version": "0.37.1",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",
@@ -48,7 +48,7 @@
   "devDependencies": {
     "@types/node": "^26.1.0",
     "hyparquet": "^1.26.2",
-    "icebird": "^0.8.12",
+    "icebird": "^0.8.13",
     "unstorage": "^1.17.5",
     "vitest": "^4.1.9"
   },