@gscdump/engine 0.23.2 → 0.24.0

package/dist/_chunks/sink.d.mts

@@ -1,9 +1,23 @@
 import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { icebergAppend, restCatalogConnect, s3SignedResolver } from "icebird";
 import { TableName } from "@gscdump/contracts";
-/** The 6 fact tables that exist as global Iceberg tables. */
+/**
+ * S3-compatible credentials for the Iceberg warehouse object store (R2 in prod,
+ * MinIO in the POC). The single definition shared by every catalog/writer/sink
+ * that signs warehouse object access — keep this contract in one place so the
+ * credential shape cannot drift between the icebird and PyIceberg paths.
+ */
+interface IcebergS3Config {
+  /** S3 endpoint host (POC MinIO: `localhost:9100`; prod: the R2 S3 endpoint). */
+  endpoint: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  /** Defaults to `'auto'` (R2's region). */
+  region?: string;
+}
+/** The 9 fact tables that exist as global Iceberg tables. */
 type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;
-/** The 6 Iceberg table names, in canonical order. */
+/** The 9 Iceberg table names, in canonical order. */
 declare const ICEBERG_TABLES: readonly IcebergTableName[];
 /**
  * Iceberg-native column type. Superset-mapped from the engine `ColumnType`;
@@ -35,7 +49,7 @@ interface IcebergTableSpec {
   table: IcebergTableName;
   columns: readonly IcebergColumn[];
   /**
-   * Partition spec — shared by all 5 tables: identity(site_id),
+   * Partition spec — shared by every table: identity(site_id),
    * identity(search_type), month(date).
    */
   partitionSpec: readonly IcebergPartitionField[];
@@ -66,7 +80,7 @@ declare const ICEBERG_PARTITION_COLUMNS: readonly IcebergColumn[];
  * catalog. Iceberg still guarantees ids are stable once a table exists.
  */
 declare const ICEBERG_FIELD_ID_BASE = 3;
-/** Shared partition spec — identical across all 5 tables. */
+/** Shared partition spec — identical across every table. */
 declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
 /**
  * Derive the full Iceberg table spec for a table from the engine `SCHEMAS`
@@ -78,7 +92,7 @@ declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
  * source of truth — do not hand-list columns elsewhere.
  */
 declare function icebergTableSpec(table: IcebergTableName): IcebergTableSpec;
-/** All 5 Iceberg table specs, keyed by table name. */
+/** All Iceberg table specs, keyed by table name. */
 declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
 /** icebird's lowercase Iceberg primitive types (subset we use). */
 type IcebergPrimitiveType = 'string' | 'int' | 'long' | 'double' | 'date';
@@ -107,15 +121,6 @@ interface IcebergPartitionSpec {
   'spec-id': number;
   'fields': IcebergPartitionSpecField[];
 }
-/** S3-compatible credentials for the R2 warehouse. */
-interface IcebergS3Config {
-  /** R2 S3 endpoint, e.g. `https://<account>.r2.cloudflarestorage.com`. */
-  endpoint: string;
-  accessKeyId: string;
-  secretAccessKey: string;
-  /** Defaults to `'auto'` (R2's region). */
-  region?: string;
-}
 /** Everything needed to talk to the R2 Data Catalog. */
 interface IcebergCatalogConfig {
   /** REST catalog URI, e.g. `https://catalog.cloudflarestorage.com/<acct>/<warehouse>`. */

package/dist/adapters/duckdb-node.mjs

@@ -67,6 +67,11 @@ function createNodeDuckDBHandle(opts = {}) {
 	};
 }
 function resetNodeDuckDB() {
+	const pending = singleton;
 	singleton = null;
+	pending?.then(({ db, conn }) => {
+		conn.close();
+		db.reset();
+	}).catch(() => {});
 }
 export { createNodeDuckDBHandle, resetNodeDuckDB };

package/dist/adapters/r2-manifest.mjs

@@ -1,5 +1,11 @@
 import { inferLegacyTier, inferSearchType } from "../_chunks/storage.mjs";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
+const CAS_BACKOFF_BASE_MS = 5;
+const CAS_BACKOFF_CAP_MS = 250;
+async function casBackoff(attempt) {
+	const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
+	await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
+}
 function defaultSnapshotId() {
 	return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
 }
@@ -52,7 +58,7 @@ function createR2ManifestStore(opts) {
 	const { bucket, userId } = opts;
 	const newSnapshotId = opts.newSnapshotId ?? defaultSnapshotId;
 	const now = opts.now ?? (() => Date.now());
-	const maxRetries = opts.maxRetries ?? 8;
+	const maxRetries = opts.maxRetries ?? 16;
 	const onEvent = opts.onEvent;
 	async function readShard(siteId, table) {
 		const head = await bucket.get(headKey(userId, siteId, table));
@@ -112,6 +118,7 @@ function createR2ManifestStore(opts) {
 				attempt
 			});
 			attempt++;
+			if (attempt < maxRetries) await casBackoff(attempt);
 		}
 		throw new Error(`R2 manifest CAS exceeded ${maxRetries} retries for ${siteId}/${table}`);
 	}

package/dist/index.mjs

@@ -301,18 +301,32 @@ function createIngestAccumulator(opts) {
 }
 const DAY_MILLIS = 864e5;
 function toIcebergDate(value) {
-	if (typeof value === "string") return Math.floor(Date.parse(`${value}T00:00:00Z`) / DAY_MILLIS);
-	if (value instanceof Date) return Math.floor(value.getTime() / DAY_MILLIS);
+	if (typeof value === "string") {
+		const ms = Date.parse(`${value}T00:00:00Z`);
+		if (Number.isNaN(ms)) throw new TypeError(`toIcebergDate: invalid date string '${value}'`);
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	if (value instanceof Date) {
+		const ms = value.getTime();
+		if (Number.isNaN(ms)) throw new TypeError("toIcebergDate: invalid Date (NaN)");
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	return value;
+}
+function coerceJsonSafe(value) {
+	if (typeof value === "bigint") return Number(value);
 	return value;
 }
 function toRecords(slice, rows) {
 	const siteId = slice.ctx.siteId ?? "";
-	return rows.map((row) => ({
-		...row,
-		date: toIcebergDate(row.date),
-		site_id: siteId,
-		search_type: slice.searchType
-	}));
+	return rows.map((row) => {
+		const out = {};
+		for (const k in row) out[k] = coerceJsonSafe(row[k]);
+		out.date = toIcebergDate(out.date);
+		out.site_id = siteId;
+		out.search_type = slice.searchType;
+		return out;
+	});
 }
 function createIcebergAppendSink(options) {
 	let connection;

package/dist/sink-node.d.mts

@@ -1,16 +1,8 @@
-import { LocalIcebergSinkOptions, Sink } from "./_chunks/sink.mjs";
-/** S3-compatible credentials for the warehouse (POC: MinIO). */
-interface LocalIcebergS3Config {
-  /** S3 endpoint host (POC MinIO: `localhost:9100`). */
-  endpoint: string;
-  accessKeyId: string;
-  secretAccessKey: string;
-  region?: string;
-}
+import { IcebergS3Config, LocalIcebergSinkOptions, Sink } from "./_chunks/sink.mjs";
 /** Full `LocalIcebergSink` options — extends the frozen contract options. */
 interface LocalIcebergSinkFullOptions extends LocalIcebergSinkOptions {
   /** S3 credentials for the warehouse. Defaults to the POC MinIO creds. */
-  s3?: LocalIcebergS3Config;
+  s3?: IcebergS3Config;
   /** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
   python?: string;
   /** Override the writer-script path. Defaults to `scripts/iceberg-writer.py`. */
@@ -28,4 +20,4 @@ interface LocalIcebergSink extends Sink {
  * use this sink must skip when the stack is unreachable.
  */
 declare function createLocalIcebergSink(options: LocalIcebergSinkFullOptions): LocalIcebergSink;
-export { type LocalIcebergS3Config, type LocalIcebergSink, type LocalIcebergSinkFullOptions, createLocalIcebergSink };
+export { type LocalIcebergSink, type LocalIcebergSinkFullOptions, createLocalIcebergSink };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.23.2",
+  "version": "0.24.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -177,11 +177,11 @@
     }
   },
   "dependencies": {
-    "drizzle-orm": "^0.45.2",
+    "drizzle-orm": "1.0.0-rc.3",
     "icebird": "^0.8.6",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.23.2",
-    "gscdump": "0.23.2"
+    "@gscdump/contracts": "0.24.0",
+    "gscdump": "0.24.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",