@gscdump/lakehouse 1.0.1 → 1.0.3

package/README.md

@@ -15,6 +15,9 @@ Node.js 22 or newer is required.
 
 - `@gscdump/lakehouse` — connect to a catalog, define datasets, derive table
   specifications, resolve data files, and use bigint-safe serialization.
+- `@gscdump/lakehouse/bigint`: bigint-safe JSON helpers without catalog or
+  Node dependencies.
+- `@gscdump/lakehouse/schema`: lightweight shared schema constants.
 - `@gscdump/lakehouse/maintenance` — stable commit classification and orphan
   cleanup workflows.
 - `@gscdump/lakehouse/provisioning` — adopt, allocate, or provision R2 Data
@@ -25,6 +28,7 @@ Node.js 22 or newer is required.
 
 ```ts
 import { connectIcebergCatalog, listIcebergTables } from '@gscdump/lakehouse'
+import { encodeJsonBigintSafe } from '@gscdump/lakehouse/bigint'
 import { isCommitRateLimited } from '@gscdump/lakehouse/maintenance'
 
 const connection = await connectIcebergCatalog(config)

package/dist/_chunks/catalog.mjs

@@ -1,16 +1,5 @@
+import { stringifyBigintSafe } from "../bigint.mjs";
 import { cachingResolver, icebergAppend, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./libs/icebird.mjs";
-function bigintJsonReplacer(_key, value) {
-	return typeof value === "bigint" ? value.toString() : value;
-}
-function stringifyBigintSafe(value, space) {
-	return JSON.stringify(value, bigintJsonReplacer, space);
-}
-function encodeJsonBigintSafe(value) {
-	return new TextEncoder().encode(stringifyBigintSafe(value));
-}
-function coerceBigIntToNumber(value) {
-	return typeof value === "bigint" ? Number(value) : value;
-}
 function reportCatalogCacheError(cache, operation, key, error) {
 	try {
 		if (cache.onError) cache.onError(operation, key, error);
@@ -410,4 +399,4 @@ async function resolveIcebergDataFiles(conn, opts) {
 	}
 	return out;
 }
-export { ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe };
+export { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };

package/dist/bigint.d.mts

@@ -0,0 +1,38 @@
+/**
+ * BigInt at JSON boundaries — the single place this policy lives.
+ *
+ * int64 reaches serialization from two origins:
+ *  1. DuckDB `SUM`/`COUNT` aggregates — already coerced to `number` at the
+ *     source-factory boundary (`@gscdump/engine` `coerceRows`), so rows are
+ *     BigInt-free by the time they cross the query boundary.
+ *  2. icebird / REST-catalog metadata (`current-snapshot-id`, `sequence-number`,
+ *     `snapshot-id`) and other identity ids — these never pass through a source,
+ *     so they arrive at envelope / cache / log serializers still as BigInt.
+ *
+ * A plain `JSON.stringify` throws `Do not know how to serialize a BigInt`, which
+ * is how a stray identity int64 turns into a request 500. Any boundary that can
+ * see origin (2) must serialize through here.
+ *
+ * TWO policies, because the correct target type depends on what the value is:
+ *
+ *  - Identity / precision values (snapshot ids, large row ids) can exceed 2^53
+ *    and must round-trip losslessly → serialize as a decimal STRING. Use
+ *    {@link stringifyBigintSafe} / {@link bigintJsonReplacer} / {@link encodeJsonBigintSafe}
+ *    for storage / transport / log serialization. Where values are already
+ *    number-coerced (origin 1), the replacer is a no-op backstop.
+ *
+ *  - Analytics aggregates that you want as real numbers downstream → coerce with
+ *    {@link coerceBigIntToNumber}. Lossy above 2^53 by design; click / impression
+ *    columns never reach that range. This is the scalar behind engine `coerceRows`.
+ */
+/** `JSON.stringify` replacer: BigInt → decimal string (lossless). Pass-through otherwise. */
+declare function bigintJsonReplacer(_key: string, value: unknown): unknown;
+/** `JSON.stringify` that never throws on BigInt; identity ids stay lossless as strings. */
+declare function stringifyBigintSafe(value: unknown, space?: string | number): string;
+declare function encodeJsonBigintSafe(value: unknown): Uint8Array;
+/**
+ * BigInt → number for analytics aggregates (< 2^53). Lossy above; acceptable for
+ * click / impression sums. Non-BigInt values pass through untouched.
+ */
+declare function coerceBigIntToNumber<T>(value: T): T | number;
+export { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe };

package/dist/bigint.mjs

@@ -0,0 +1,15 @@
+function bigintJsonReplacer(_key, value) {
+	return typeof value === "bigint" ? value.toString() : value;
+}
+function stringifyBigintSafe(value, space) {
+	return JSON.stringify(value, bigintJsonReplacer, space);
+}
+let jsonEncoder;
+function encodeJsonBigintSafe(value) {
+	jsonEncoder ??= new TextEncoder();
+	return jsonEncoder.encode(stringifyBigintSafe(value));
+}
+function coerceBigIntToNumber(value) {
+	return typeof value === "bigint" ? Number(value) : value;
+}
+export { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe };

package/dist/index.d.mts

@@ -1,43 +1,6 @@
+import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
 import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
-/**
- * BigInt at JSON boundaries — the single place this policy lives.
- *
- * int64 reaches serialization from two origins:
- *  1. DuckDB `SUM`/`COUNT` aggregates — already coerced to `number` at the
- *     source-factory boundary (`@gscdump/engine` `coerceRows`), so rows are
- *     BigInt-free by the time they cross the query boundary.
- *  2. icebird / REST-catalog metadata (`current-snapshot-id`, `sequence-number`,
- *     `snapshot-id`) and other identity ids — these never pass through a source,
- *     so they arrive at envelope / cache / log serializers still as BigInt.
- *
- * A plain `JSON.stringify` throws `Do not know how to serialize a BigInt`, which
- * is how a stray identity int64 turns into a request 500. Any boundary that can
- * see origin (2) must serialize through here.
- *
- * TWO policies, because the correct target type depends on what the value is:
- *
- *  - Identity / precision values (snapshot ids, large row ids) can exceed 2^53
- *    and must round-trip losslessly → serialize as a decimal STRING. Use
- *    {@link stringifyBigintSafe} / {@link bigintJsonReplacer} / {@link encodeJsonBigintSafe}
- *    for storage / transport / log serialization. Where values are already
- *    number-coerced (origin 1), the replacer is a no-op backstop.
- *
- *  - Analytics aggregates that you want as real numbers downstream → coerce with
- *    {@link coerceBigIntToNumber}. Lossy above 2^53 by design; click / impression
- *    columns never reach that range. This is the scalar behind engine `coerceRows`.
- */
-/** `JSON.stringify` replacer: BigInt → decimal string (lossless). Pass-through otherwise. */
-declare function bigintJsonReplacer(_key: string, value: unknown): unknown;
-/** `JSON.stringify` that never throws on BigInt; identity ids stay lossless as strings. */
-declare function stringifyBigintSafe(value: unknown, space?: string | number): string;
-/** UTF-8 JSON bytes, BigInt-safe — the payload shape for `DataSource.write`. */
-declare function encodeJsonBigintSafe(value: unknown): Uint8Array;
-/**
- * BigInt → number for analytics aggregates (< 2^53). Lossy above; acceptable for
- * click / impression sums. Non-BigInt values pass through untouched.
- */
-declare function coerceBigIntToNumber<T>(value: T): T | number;
 /**
  * Closed identity shapes (ADR-0021 amendments 2 + 4). `'site-int'` is the
  * team-scoped numeric Catalog Site Id (nuxtseo ADR-0091) stored under the

package/dist/index.mjs

@@ -1,5 +1,7 @@
-import { ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe } from "./_chunks/catalog.mjs";
+import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
 import { icebergCreateTable } from "./_chunks/libs/icebird.mjs";
+import { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING } from "./schema.mjs";
 const INT32_MIN = -2147483648;
 const INT32_MAX = 2147483647;
 const DAY_MILLIS = 864e5;
@@ -325,5 +327,4 @@ function defineIcebergDataset(def) {
 		resolveDataFiles
 	};
 }
-const DEFAULT_PARTITION_KEY_ENCODING = "int";
 export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/maintenance.mjs

@@ -1,5 +1,5 @@
-import { isCommitRateLimited } from "./_chunks/catalog.mjs";
 import { icebergManifests, restCatalogListTables, restCatalogLoadTable } from "./_chunks/libs/icebird.mjs";
+import { isCommitRateLimited } from "./_chunks/catalog.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
 const DEFAULT_IO_CONCURRENCY = 4;

package/dist/schema.d.mts

@@ -0,0 +1,2 @@
+import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
+export { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding };

package/dist/schema.mjs

@@ -0,0 +1,2 @@
+const DEFAULT_PARTITION_KEY_ENCODING = "int";
+export { DEFAULT_PARTITION_KEY_ENCODING };

package/dist/unsafe-raw.mjs

@@ -1,3 +1,3 @@
-import { icebergAppendRetrying } from "./_chunks/catalog.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
+import { icebergAppendRetrying } from "./_chunks/catalog.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",
@@ -26,6 +26,16 @@
       "import": "./dist/index.mjs",
       "default": "./dist/index.mjs"
     },
+    "./bigint": {
+      "types": "./dist/bigint.d.mts",
+      "import": "./dist/bigint.mjs",
+      "default": "./dist/bigint.mjs"
+    },
+    "./schema": {
+      "types": "./dist/schema.d.mts",
+      "import": "./dist/schema.mjs",
+      "default": "./dist/schema.mjs"
+    },
     "./maintenance": {
       "types": "./dist/maintenance.d.mts",
       "import": "./dist/maintenance.mjs",