@gscdump/engine 0.28.3 → 0.30.0

package/dist/_chunks/schema2.mjs

@@ -10,6 +10,15 @@ const ICEBERG_TABLES = [
 	"search_appearance_queries",
 	"search_appearance_page_queries"
 ];
+const SEARCH_TYPE_INT = {
+	web: 1,
+	image: 2,
+	video: 3,
+	news: 4,
+	discover: 5,
+	googleNews: 6
+};
+const INT_SEARCH_TYPE = Object.fromEntries(Object.entries(SEARCH_TYPE_INT).map(([k, v]) => [v, k]));
 const ICEBERG_PARTITION_COLUMNS = [{
 	name: "site_id",
 	type: "STRING",
@@ -21,6 +30,20 @@ const ICEBERG_PARTITION_COLUMNS = [{
 	required: true,
 	fieldId: 2
 }];
+function icebergPartitionColumns(encoding = "string") {
+	if (encoding === "string") return ICEBERG_PARTITION_COLUMNS;
+	return [{
+		name: "site_id",
+		type: "INT",
+		required: true,
+		fieldId: 1
+	}, {
+		name: "search_type",
+		type: "INT",
+		required: true,
+		fieldId: 2
+	}];
+}
 const ICEBERG_FIELD_ID_BASE = 3;
 const ICEBERG_PARTITION_SPEC = [
 	{
@@ -48,7 +71,7 @@ function mapColumnType(t) {
 		case "DATE": return "DATE";
 	}
 }
-function icebergTableSpec(table) {
+function icebergTableSpec(table, encoding = "string") {
 	const base = SCHEMAS[table];
 	const dataColumns = base.columns.map((col, i) => ({
 		name: col.name,
@@ -58,7 +81,7 @@ function icebergTableSpec(table) {
 	}));
 	return {
 		table,
-		columns: [...ICEBERG_PARTITION_COLUMNS, ...dataColumns],
+		columns: [...icebergPartitionColumns(encoding), ...dataColumns],
 		partitionSpec: ICEBERG_PARTITION_SPEC,
 		identityColumns: [
 			"site_id",
@@ -68,6 +91,10 @@ function icebergTableSpec(table) {
 	};
 }
 const ICEBERG_SCHEMAS = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t)]));
+const ICEBERG_SCHEMAS_INT = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "int")]));
+function icebergSchemasFor(encoding = "string") {
+	return encoding === "int" ? ICEBERG_SCHEMAS_INT : ICEBERG_SCHEMAS;
+}
 const ICEBERG_TABLE_SET = new Set(ICEBERG_TABLES);
 function isIcebergTable(table) {
 	return ICEBERG_TABLE_SET.has(table);
@@ -76,4 +103,4 @@ function assertIcebergTable(table) {
 	if (!isIcebergTable(table)) throw new Error(`Unknown Iceberg table '${table}'. Expected one of: ${ICEBERG_TABLES.join(", ")}`);
 	return table;
 }
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, icebergTableSpec, isIcebergTable };
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };

package/dist/_chunks/sink.d.mts

@@ -50,6 +50,33 @@ interface IcebergColumn {
    */
   fieldId: number;
 }
+/**
+ * Partition-key encoding for the two identity columns (`site_id`, `search_type`).
+ *
+ * - `'string'` (default, legacy): both columns are STRING. Correct, but R2 SQL's
+ *   string min/max statistics are truncated in predicate pushdown, so a bare
+ *   `WHERE site_id='<uuid>'` UNDERCOUNTS — callers must CONCAT(col,'') to stay
+ *   correct, which defeats partition pruning.
+ * - `'int'`: BOTH `site_id` and `search_type` are INT. Integer statistics are
+ *   fixed-width and never truncated, so `WHERE site_id=<n>` is both correct AND
+ *   prunes (empirically confirmed 2026-06-19, gscdump.com probe-int64-partition;
+ *   INT equality proven via the search_type column in the engine e2e canary). A
+ *   small INT site_id is ample (≪ 2.1B sites) — no LONG/BigInt needed. The caller
+ *   maps the UUID `site_id` ↔ int (app-owned, per-tenant serial) and uses
+ *   {@link SEARCH_TYPE_INT} for `search_type` (engine-owned, fixed enum).
+ *
+ * New per-team catalogs are provisioned `'int'`; existing catalogs stay
+ * `'string'`. Purely additive: `'string'` is the default everywhere so existing
+ * tables, writers, and readers are unchanged.
+ */
+type PartitionKeyEncoding = 'string' | 'int';
+/**
+ * Stable `search_type` enum → int map for `'int'`-encoded catalogs. Engine-owned
+ * and FROZEN: never renumber or reuse an id (it's the on-disk partition value).
+ */
+declare const SEARCH_TYPE_INT: Record<SearchType, number>;
+/** Reverse of {@link SEARCH_TYPE_INT} — int → `search_type`, for read-result mapping. */
+declare const INT_SEARCH_TYPE: Record<number, SearchType>;
 /** Iceberg partition transform applied to a source column. */
 type IcebergPartitionTransform = 'identity' | 'month';
 interface IcebergPartitionField {
@@ -82,6 +109,15 @@ interface IcebergTableSpec {
  * contiguously from id 3 (see `ICEBERG_FIELD_ID_BASE`).
  */
 declare const ICEBERG_PARTITION_COLUMNS: readonly IcebergColumn[];
+/**
+ * The two partition-identity columns for a given {@link PartitionKeyEncoding}.
+ * `'string'` returns {@link ICEBERG_PARTITION_COLUMNS} verbatim; `'int'` swaps
+ * BOTH to INT — `site_id` (the app's small `user_sites.int_id`; ≪ 2.1B sites, so
+ * INT is ample) and `search_type` (its fixed enum code). Integer identity columns
+ * avoid R2 SQL's truncated-string-stats equality undercount and restore pruning.
+ * Field ids are unchanged (1, 2) — only the column types differ.
+ */
+declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): readonly IcebergColumn[];
 /**
  * First field id used for per-table (non-partition) columns — immediately
  * after the two partition-identity columns (`site_id`=1, `search_type`=2).
@@ -105,9 +141,13 @@ declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
  * CONTRACT NOTE: implementation agents must treat the RETURNED VALUE as the
  * source of truth — do not hand-list columns elsewhere.
  */
-declare function icebergTableSpec(table: IcebergTableName): IcebergTableSpec;
-/** All Iceberg table specs, keyed by table name. */
+declare function icebergTableSpec(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergTableSpec;
+/** All Iceberg table specs (legacy `'string'` encoding), keyed by table name. */
 declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+/** All Iceberg table specs in `'int'` encoding (INT site_id + INT search_type). */
+declare const ICEBERG_SCHEMAS_INT: Record<IcebergTableName, IcebergTableSpec>;
+/** Table specs for the given encoding (`'string'` default). */
+declare function icebergSchemasFor(encoding?: PartitionKeyEncoding): Record<IcebergTableName, IcebergTableSpec>;
 /** True when `table` is one of the canonical {@link ICEBERG_TABLES}. */
 declare function isIcebergTable(table: string): table is IcebergTableName;
 /**
@@ -172,14 +212,14 @@ interface IcebergConnection {
  * `ICEBERG_SCHEMAS` contract. Field ids are advisory — R2 Data Catalog
  * re-assigns them on `createTable` (see `ICEBERG_FIELD_ID_BASE`).
  */
-declare function icebergSchemaFor(table: IcebergTableName): IcebergSchema;
+declare function icebergSchemaFor(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergSchema;
 /**
  * Build the icebird `PartitionSpec` for one of the 5 fact tables: the locked
  * spec `identity(site_id) + identity(search_type) + month(date)`. Each
  * partition field's `source-id` is resolved to the real column field id from
  * {@link icebergSchemaFor}.
  */
-declare function icebergPartitionSpecFor(table: IcebergTableName): IcebergPartitionSpec;
+declare function icebergPartitionSpecFor(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergPartitionSpec;
 /** Options for {@link connectIcebergCatalog}. */
 interface ConnectIcebergOptions {
   /**
@@ -264,7 +304,7 @@ declare function ensureIcebergNamespace(conn: IcebergConnection): Promise<void>;
  * than thrown so a partial run is observable; "table already exists" surfaces
  * as a failed result. Used by the app's one-off provisioning script.
  */
-declare function createIcebergTables(conn: IcebergConnection, tables?: readonly IcebergTableName[]): Promise<IcebergTableOpResult[]>;
+declare function createIcebergTables(conn: IcebergConnection, tables?: readonly IcebergTableName[], encoding?: PartitionKeyEncoding): Promise<IcebergTableOpResult[]>;
 /**
  * List the table names currently in the catalog namespace.
  *
@@ -284,10 +324,16 @@ interface IcebergListedDataFile {
 }
 interface ListIcebergDataFilesOptions {
   table: IcebergTableName;
-  /** Partition identity column. */
-  siteId: string;
-  /** Partition identity column. */
-  searchType: string;
+  /** Partition identity column. `number` for `'int'`-encoded catalogs. */
+  siteId: string | number;
+  /** Partition identity column. `number` (int code) for `'int'`-encoded catalogs. */
+  searchType: string | number;
+  /**
+   * Partition-key encoding of the catalog. `'int'` changes how manifest-summary
+   * bounds are decoded (int bytes vs UTF-8) and how the per-file partition value
+   * is compared. Defaults to `'string'`.
+   */
+  encoding?: PartitionKeyEncoding;
   /**
    * Inclusive date range. Every month touched by `[start, end]` is scanned;
    * `month(date)` is the third partition transform.
@@ -437,6 +483,14 @@ interface IcebergAppendSinkOptions extends SinkOptions {
    * uses the defaults; tests inject a synchronous `sleep`.
    */
   commitRetry?: CommitRetryOptions;
+  /**
+   * Partition-key encoding (default `'string'`). `'int'` writes BOTH `site_id`
+   * and `search_type` as INT — the caller MUST pass the numeric `site_id` (a
+   * numeric string is fine; it's `Number()`-coerced) in `slice.ctx.siteId`. A
+   * small INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
+   * {@link import('./iceberg/schema').PartitionKeyEncoding}.
+   */
+  encoding?: PartitionKeyEncoding;
 }
 /** `LocalIcebergSink` options — points at the local Iceberg REST catalog. */
 interface LocalIcebergSinkOptions extends SinkOptions {
@@ -447,4 +501,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/_chunks/storage.d.mts

@@ -1,3 +1,4 @@
+import { ParquetQueryFilter } from "hyparquet";
 import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 import { Grain, Grain as Grain$1, Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
 /**
@@ -397,6 +398,15 @@ interface QueryExecuteOptions {
    * the page_queries schema, not the analyzer's primary `table`.
    */
   placeholderTables?: Record<string, TableName>;
+  /**
+   * Per-placeholder row-group pushdown filter, derived from the query's
+   * structured filter (see `extractParquetPushdown`). A pure-JS decode executor
+   * MAY pass it to the parquet reader to prune row groups and shrink the rows
+   * it materialises before the SQL WHERE re-applies. Pure optimization: the
+   * filter is a superset of the final predicate, so an executor that ignores it
+   * (e.g. native DuckDB, which pushes from the SQL itself) stays correct.
+   */
+  pushdownFilters?: Record<string, ParquetQueryFilter>;
   dataSource: DataSource;
   table: TableName;
   signal?: AbortSignal;
@@ -475,6 +485,12 @@ interface RunSQLOptions {
    * Undefined keeps the legacy cross-type union.
    */
   searchType?: SearchType;
+  /**
+   * Per-placeholder parquet pushdown filter, forwarded verbatim to the
+   * executor. Keyed by fileSet name (matching `fileSets`). See
+   * `QueryExecuteOptions.pushdownFilters` and `extractParquetPushdown`.
+   */
+  pushdownFilters?: Record<string, ParquetQueryFilter>;
   /**
    * Optional read-path profiler. `runSQL` emits `manifest.list` +
    * `executor.execute` spans and forwards it into the executor for the

package/dist/_chunks/types.d.mts

@@ -30,8 +30,13 @@ interface ResolverOptions<TableKey extends string = string> {
   adapter: ResolverAdapter<TableKey>;
   /** Optional site scope. Required for multi-tenant D1; omitted for parquet. */
   siteId?: string | number;
-  /** Optional searchType scope. Required for multi-tenant Iceberg; omitted for parquet. */
-  searchType?: string;
+  /**
+   * Optional searchType scope. Required for multi-tenant Iceberg; omitted for
+   * parquet. `number` is the int-encoded code (`SEARCH_TYPE_INT`) for catalogs
+   * whose `search_type` partition column is INT — bound bare (unquoted) so the
+   * int partition prunes; `string` for the default string-encoded catalogs.
+   */
+  searchType?: string | number;
 }
 interface ResolvedSQL {
   sql: string;

package/dist/adapters/duckdb-node.mjs

@@ -8,6 +8,7 @@ import { tmpdir } from "node:os";
 import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
 const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
 let singleton = null;
+let singletonOpts = null;
 function bundles() {
 	return {
 		mvp: {
@@ -29,11 +30,19 @@ async function initialize(opts) {
 		conn: db.connect()
 	};
 }
+function getSingleton(opts) {
+	if (!singleton) {
+		singleton = initialize(opts);
+		singletonOpts = opts;
+	}
+	return singleton;
+}
 function createNodeDuckDBHandle(opts = {}) {
-	if (!singleton) singleton = initialize(opts);
+	if (singleton && opts.verbose !== void 0 && opts.verbose !== (singletonOpts?.verbose ?? false)) console.warn(`[gscdump] createNodeDuckDBHandle: ignoring verbose=${opts.verbose} — a shared DuckDB instance was already initialized with verbose=${singletonOpts?.verbose ?? false}. Call resetNodeDuckDB() before re-initializing to change it.`);
+	getSingleton(opts);
 	return {
 		async query(sql, params) {
-			const { conn } = await singleton;
+			const { conn } = await getSingleton(opts);
 			if (!params || params.length === 0) return arrowToRows(conn.query(sql));
 			const stmt = conn.prepare(sql);
 			try {
@@ -43,15 +52,15 @@ function createNodeDuckDBHandle(opts = {}) {
 			}
 		},
 		async registerFileBuffer(name, bytes) {
-			const { db } = await singleton;
+			const { db } = await getSingleton(opts);
 			db.registerFileBuffer(name, bytes);
 		},
 		async copyFileToBuffer(name) {
-			const { db } = await singleton;
+			const { db } = await getSingleton(opts);
 			return db.copyFileToBuffer(name);
 		},
 		async dropFiles(names) {
-			const { db } = await singleton;
+			const { db } = await getSingleton(opts);
 			for (const name of names) {
 				try {
 					db.dropFile(name);
@@ -69,9 +78,12 @@ function createNodeDuckDBHandle(opts = {}) {
 function resetNodeDuckDB() {
 	const pending = singleton;
 	singleton = null;
+	singletonOpts = null;
 	pending?.then(({ db, conn }) => {
 		conn.close();
 		db.reset();
-	}).catch(() => {});
+	}).catch((err) => {
+		console.warn("[gscdump] resetNodeDuckDB: failed to release DuckDB instance", err);
+	});
 }
 export { createNodeDuckDBHandle, resetNodeDuckDB };

package/dist/adapters/hyparquet.d.mts

@@ -24,10 +24,18 @@ interface DecodeParquetOptions {
    * per row group — pruning groups whose column statistics can't match and
    * materialising only matching rows — so a filtered decode of a large file
    * holds at most one row group plus the matches in memory, never the whole
-   * file. Use this whenever the caller needs a sub-slice of a big parquet
-   * (e.g. one feedpath out of a site-wide sitemap-urls index).
+   * file. Use when a caller needs a sub-slice of a big parquet keyed on a
+   * clustered column (a row group's min/max stats only prune if the predicate
+   * column is the physical sort key — see `sortKey`/`clusterKey`).
    */
   filter?: ParquetQueryFilter;
+  /**
+   * Project a subset of columns. hyparquet only fetches + decodes the named
+   * column chunks, so a read that needs 2 of 14 columns skips the other 12's
+   * pages entirely. Omit to read every column. Names not present in the file
+   * are ignored by the reader.
+   */
+  columns?: readonly string[];
 }
 declare function decodeParquetToRows(bytes: Uint8Array, opts?: DecodeParquetOptions): Promise<Row[]>;
 interface HyparquetCodecOptions {

package/dist/adapters/hyparquet.mjs

@@ -1,14 +1,83 @@
 import { SCHEMAS, TABLE_METADATA, dedupeByNaturalKey } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "hyparquet";
-import { parquetWriteBuffer } from "hyparquet-writer";
+import { ByteWriter, parquetWriteRows } from "hyparquet-writer";
 const ROW_GROUP_SIZE = 25e3;
 function basicTypeFor(colType) {
-	if (colType === "VARCHAR" || colType === "DATE") return "STRING";
+	if (colType === "VARCHAR") return "STRING";
 	if (colType === "BIGINT") return "INT64";
 	if (colType === "INTEGER") return "INT32";
 	if (colType === "DOUBLE") return "DOUBLE";
+	if (colType === "DATE") return "INT32";
 	throw new Error(`unsupported column type for parquet encoding: ${colType}`);
 }
+const EPOCH_DAY_MS = 864e5;
+function toEpochDays(value) {
+	if (value === null || value === void 0) return null;
+	if (typeof value === "number") return value;
+	if (value instanceof Date) {
+		const ms = value.getTime();
+		if (Number.isNaN(ms)) throw new TypeError("encodeRowsToParquet: invalid Date for DATE column");
+		return Math.floor(ms / EPOCH_DAY_MS);
+	}
+	if (typeof value === "string") {
+		const ms = Date.parse(`${value}T00:00:00Z`);
+		if (Number.isNaN(ms)) throw new TypeError(`encodeRowsToParquet: invalid date string '${value}'`);
+		return Math.floor(ms / EPOCH_DAY_MS);
+	}
+	throw new TypeError(`encodeRowsToParquet: unsupported DATE value '${String(value)}'`);
+}
+function isoFromDate(d) {
+	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
+}
+function buildWriteSchema(columns) {
+	const schema = [{
+		name: "root",
+		num_children: columns.length
+	}];
+	for (const col of columns) {
+		const repetition_type = col.nullable ? "OPTIONAL" : "REQUIRED";
+		switch (col.type) {
+			case "DATE":
+				schema.push({
+					name: col.name,
+					type: "INT32",
+					converted_type: "DATE",
+					repetition_type
+				});
+				break;
+			case "VARCHAR":
+				schema.push({
+					name: col.name,
+					type: "BYTE_ARRAY",
+					converted_type: "UTF8",
+					repetition_type
+				});
+				break;
+			case "INTEGER":
+				schema.push({
+					name: col.name,
+					type: "INT32",
+					repetition_type
+				});
+				break;
+			case "BIGINT":
+				schema.push({
+					name: col.name,
+					type: "INT64",
+					repetition_type
+				});
+				break;
+			case "DOUBLE":
+				schema.push({
+					name: col.name,
+					type: "DOUBLE",
+					repetition_type
+				});
+				break;
+		}
+	}
+	return schema;
+}
 function coerceValue(value, type) {
 	if (value === null || value === void 0) return null;
 	if (type === "STRING") return typeof value === "string" ? value : String(value);
@@ -50,65 +119,86 @@ function sortRowsByClusterKey(table, rows) {
 	});
 	return copy;
 }
+function encodeOrderedRows(rows, columns, rowGroupSize) {
+	const schema = buildWriteSchema(columns);
+	const isDate = columns.map((col) => col.type === "DATE");
+	const types = columns.map((col) => basicTypeFor(col.type));
+	const columnSpecs = columns.map((col) => ({
+		name: col.name,
+		nullable: col.nullable,
+		columnIndex: true
+	}));
+	function* coercedRows() {
+		for (const r of rows) {
+			const out = {};
+			for (let c = 0; c < columns.length; c++) {
+				const name = columns[c].name;
+				out[name] = isDate[c] ? toEpochDays(r[name]) : coerceValue(r[name], types[c]);
+			}
+			yield out;
+		}
+	}
+	const writer = new ByteWriter();
+	parquetWriteRows({
+		writer,
+		rows: coercedRows(),
+		columns: columnSpecs,
+		schema,
+		rowGroupSize
+	});
+	return new Uint8Array(writer.getBuffer());
+}
 function encodeRowsToParquet(table, rows) {
 	const schema = SCHEMAS[table];
-	const sorted = sortRowsByClusterKey(table, rows);
-	const buffer = parquetWriteBuffer({
-		columnData: schema.columns.map((col) => {
-			const type = basicTypeFor(col.type);
-			const data = sorted.map((r) => coerceValue(r[col.name], type));
-			return {
-				name: col.name,
-				data,
-				type,
-				nullable: col.nullable,
-				columnIndex: true
-			};
-		}),
-		rowGroupSize: ROW_GROUP_SIZE
-	});
-	return new Uint8Array(buffer);
+	return encodeOrderedRows(sortRowsByClusterKey(table, rows), schema.columns, ROW_GROUP_SIZE);
 }
 function encodeRowsToParquetFlex(rows, opts) {
 	const { columns, sortKey = [], rowGroupSize = ROW_GROUP_SIZE } = opts;
-	const sorted = sortKey.length === 0 || rows.length <= 1 ? rows : [...rows].sort((a, b) => {
+	return encodeOrderedRows(sortKey.length === 0 || rows.length <= 1 ? rows : [...rows].sort((a, b) => {
 		for (const col of sortKey) {
 			const cmp = compareValues(a[col], b[col]);
 			if (cmp !== 0) return cmp;
 		}
 		return 0;
-	});
-	const buffer = parquetWriteBuffer({
-		columnData: columns.map((col) => {
-			const type = basicTypeFor(col.type);
-			const data = sorted.map((r) => coerceValue(r[col.name], type));
-			return {
-				name: col.name,
-				data,
-				type,
-				nullable: col.nullable,
-				columnIndex: true
-			};
-		}),
-		rowGroupSize
-	});
-	return new Uint8Array(buffer);
+	}), columns, rowGroupSize);
 }
 function asyncBufferFromBytes(bytes) {
-	const ab = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+	const base = bytes.byteOffset;
+	const buf = bytes.buffer;
 	return {
-		byteLength: ab.byteLength,
+		byteLength: bytes.byteLength,
 		slice(start, end) {
-			return ab.slice(start, end);
+			const from = base + start;
+			const to = end === void 0 ? base + bytes.byteLength : base + end;
+			return buf.slice(from, to);
 		}
 	};
 }
 async function decodeParquetToRows(bytes, opts = {}) {
 	if (bytes.byteLength === 0) return [];
-	return await parquetReadObjects({
+	return normalizeDecodedDates(await parquetReadObjects({
 		file: asyncBufferFromBytes(bytes),
-		...opts.filter ? { filter: opts.filter } : {}
-	});
+		...opts.columns ? { columns: [...opts.columns] } : {},
+		...opts.filter ? {
+			filter: opts.filter,
+			useBloomFilters: true
+		} : {}
+	}));
+}
+function normalizeDecodedDates(rows) {
+	if (rows.length === 0) return rows;
+	const dateCols = [];
+	const first = rows[0];
+	for (const k in first) if (first[k] instanceof Date) dateCols.push(k);
+	if (dateCols.length === 0) return rows;
+	for (const row of rows) {
+		const r = row;
+		for (const k of dateCols) {
+			const v = r[k];
+			if (v instanceof Date) r[k] = isoFromDate(v);
+		}
+	}
+	return rows;
 }
 function createHyparquetCodec(options = {}) {
 	return {
@@ -135,7 +225,7 @@ function createHyparquetCodec(options = {}) {
 			const allRows = [];
 			for (const key of inputKeys) {
 				const rows = await decodeParquetToRows(await dataSource.read(key));
-				allRows.push(...rows);
+				for (let i = 0; i < rows.length; i++) allRows.push(rows[i]);
 			}
 			const rows = dedupeByNaturalKey(ctx.table, allRows);
 			const bytes = encodeRowsToParquet(ctx.table, rows);

package/dist/adapters/node.mjs

@@ -74,7 +74,7 @@ function snapshotAlias(fileName) {
 	if (!m?.[1]) throw new TypeError(`snapshotAlias: unrecognised filename ${JSON.stringify(fileName)}`);
 	return `cold_${m[1].replace("-", "_")}`;
 }
-const SNAPSHOT_TYPE_ERROR_KINDS = new Set([
+const SNAPSHOT_TYPE_ERROR_KINDS = /* @__PURE__ */ new Set([
 	"invalid-snapshot-filename",
 	"unsupported-snapshot-index-version",
 	"invalid-schema-identifier",

package/dist/errors.mjs

@@ -168,7 +168,7 @@ const engineErrors = {
 		};
 	}
 };
-const ENGINE_ERROR_KINDS = new Set([
+const ENGINE_ERROR_KINDS = /* @__PURE__ */ new Set([
 	"analyzer-not-found",
 	"analyzer-capability-missing",
 	"invalid-sql-literal",

package/dist/iceberg/index.d.mts

@@ -1,4 +1,4 @@
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
 import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
 type IcebergAppendSink = Sink;
 /**
@@ -10,4 +10,4 @@ type IcebergAppendSink = Sink;
  * with no rows never touches the network.
  */
 declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
-export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
+export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };