@gscdump/engine 0.32.12 → 0.33.0

package/dist/_chunks/schema.d.mts

@@ -145,21 +145,6 @@ declare const queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
       identity: undefined;
       generated: undefined;
     }>;
-    query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-      name: string;
-      tableName: "queries";
-      dataType: "string";
-      data: string;
-      driverParam: string;
-      notNull: false;
-      hasDefault: false;
-      isPrimaryKey: false;
-      isAutoincrement: false;
-      hasRuntimeDefault: false;
-      enumValues: undefined;
-      identity: undefined;
-      generated: undefined;
-    }>;
     date: import("drizzle-orm/pg-core").PgBuildColumn<"queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
       name: string;
       tableName: "queries";
@@ -339,21 +324,6 @@ declare const page_queries: import("drizzle-orm/pg-core").PgTableWithColumns<{
       identity: undefined;
       generated: undefined;
     }>;
-    query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"page_queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-      name: string;
-      tableName: "page_queries";
-      dataType: "string";
-      data: string;
-      driverParam: string;
-      notNull: false;
-      hasDefault: false;
-      isPrimaryKey: false;
-      isAutoincrement: false;
-      hasRuntimeDefault: false;
-      enumValues: undefined;
-      identity: undefined;
-      generated: undefined;
-    }>;
     date: import("drizzle-orm/pg-core").PgBuildColumn<"page_queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
       name: string;
       tableName: "page_queries";
@@ -863,21 +833,6 @@ declare const search_appearance_queries: import("drizzle-orm/pg-core").PgTableWi
       identity: undefined;
       generated: undefined;
     }>;
-    query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-      name: string;
-      tableName: "search_appearance_queries";
-      dataType: "string";
-      data: string;
-      driverParam: string;
-      notNull: false;
-      hasDefault: false;
-      isPrimaryKey: false;
-      isAutoincrement: false;
-      hasRuntimeDefault: false;
-      enumValues: undefined;
-      identity: undefined;
-      generated: undefined;
-    }>;
     date: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
       name: string;
       tableName: "search_appearance_queries";
@@ -990,21 +945,6 @@ declare const search_appearance_page_queries: import("drizzle-orm/pg-core").PgTa
       identity: undefined;
       generated: undefined;
     }>;
-    query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_page_queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-      name: string;
-      tableName: "search_appearance_page_queries";
-      dataType: "string";
-      data: string;
-      driverParam: string;
-      notNull: false;
-      hasDefault: false;
-      isPrimaryKey: false;
-      isAutoincrement: false;
-      hasRuntimeDefault: false;
-      enumValues: undefined;
-      identity: undefined;
-      generated: undefined;
-    }>;
     date: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_page_queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
       name: string;
       tableName: "search_appearance_page_queries";
@@ -1267,21 +1207,6 @@ declare const drizzleSchema: {
         identity: undefined;
         generated: undefined;
       }>;
-      query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-        name: string;
-        tableName: "queries";
-        dataType: "string";
-        data: string;
-        driverParam: string;
-        notNull: false;
-        hasDefault: false;
-        isPrimaryKey: false;
-        isAutoincrement: false;
-        hasRuntimeDefault: false;
-        enumValues: undefined;
-        identity: undefined;
-        generated: undefined;
-      }>;
       date: import("drizzle-orm/pg-core").PgBuildColumn<"queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
         name: string;
         tableName: "queries";
@@ -1461,21 +1386,6 @@ declare const drizzleSchema: {
         identity: undefined;
         generated: undefined;
       }>;
-      query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"page_queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-        name: string;
-        tableName: "page_queries";
-        dataType: "string";
-        data: string;
-        driverParam: string;
-        notNull: false;
-        hasDefault: false;
-        isPrimaryKey: false;
-        isAutoincrement: false;
-        hasRuntimeDefault: false;
-        enumValues: undefined;
-        identity: undefined;
-        generated: undefined;
-      }>;
       date: import("drizzle-orm/pg-core").PgBuildColumn<"page_queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
         name: string;
         tableName: "page_queries";
@@ -1969,21 +1879,6 @@ declare const drizzleSchema: {
         identity: undefined;
         generated: undefined;
       }>;
-      query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-        name: string;
-        tableName: "search_appearance_queries";
-        dataType: "string";
-        data: string;
-        driverParam: string;
-        notNull: false;
-        hasDefault: false;
-        isPrimaryKey: false;
-        isAutoincrement: false;
-        hasRuntimeDefault: false;
-        enumValues: undefined;
-        identity: undefined;
-        generated: undefined;
-      }>;
       date: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
         name: string;
         tableName: "search_appearance_queries";
@@ -2096,21 +1991,6 @@ declare const drizzleSchema: {
         identity: undefined;
         generated: undefined;
       }>;
-      query_canonical: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_page_queries", import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>, {
-        name: string;
-        tableName: "search_appearance_page_queries";
-        dataType: "string";
-        data: string;
-        driverParam: string;
-        notNull: false;
-        hasDefault: false;
-        isPrimaryKey: false;
-        isAutoincrement: false;
-        hasRuntimeDefault: false;
-        enumValues: undefined;
-        identity: undefined;
-        generated: undefined;
-      }>;
       date: import("drizzle-orm/pg-core").PgBuildColumn<"search_appearance_page_queries", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgDateStringBuilder>, {
         name: string;
         tableName: "search_appearance_page_queries";

package/dist/_chunks/schema.mjs

@@ -14,7 +14,6 @@ const pages = pgTable("pages", {
 });
 const queries = pgTable("queries", {
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -26,7 +25,6 @@ const countries = pgTable("countries", {
 const page_queries = pgTable("page_queries", {
 	url: varchar("url").notNull(),
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -60,7 +58,6 @@ const search_appearance_pages = pgTable("search_appearance_pages", {
 const search_appearance_queries = pgTable("search_appearance_queries", {
 	searchAppearance: varchar("searchAppearance").notNull(),
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -68,7 +65,6 @@ const search_appearance_page_queries = pgTable("search_appearance_page_queries",
 	searchAppearance: varchar("searchAppearance").notNull(),
 	url: varchar("url").notNull(),
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -99,7 +95,7 @@ const TABLE_METADATA = {
 	queries: {
 		sortKey: ["date", "query"],
 		clusterKey: ["query", "date"],
-		version: 2
+		version: 3
 	},
 	countries: {
 		sortKey: ["date", "country"],
@@ -117,7 +113,7 @@ const TABLE_METADATA = {
 			"query",
 			"date"
 		],
-		version: 2
+		version: 3
 	},
 	dates: {
 		sortKey: ["date"],
@@ -153,7 +149,7 @@ const TABLE_METADATA = {
 			"query",
 			"date"
 		],
-		version: 1
+		version: 2
 	},
 	search_appearance_page_queries: {
 		sortKey: [
@@ -168,7 +164,7 @@ const TABLE_METADATA = {
 			"query",
 			"date"
 		],
-		version: 1
+		version: 2
 	},
 	hourly_pages: {
 		sortKey: [

package/dist/_chunks/schema2.mjs

@@ -10,6 +10,7 @@ const ICEBERG_TABLES = [
 	"search_appearance_queries",
 	"search_appearance_page_queries"
 ];
+const DEFAULT_PARTITION_KEY_ENCODING = "int";
 const SEARCH_TYPE_INT = {
 	web: 1,
 	image: 2,
@@ -30,7 +31,7 @@ const ICEBERG_PARTITION_COLUMNS = [{
 	required: true,
 	fieldId: 2
 }];
-function icebergPartitionColumns(encoding = "string") {
+function icebergPartitionColumns(encoding = "int") {
 	if (encoding === "string") return ICEBERG_PARTITION_COLUMNS;
 	return [{
 		name: "site_id",
@@ -71,7 +72,7 @@ function mapColumnType(t) {
 		case "DATE": return "DATE";
 	}
 }
-function icebergTableSpec(table, encoding = "string") {
+function icebergTableSpec(table, encoding = "int") {
 	const base = SCHEMAS[table];
 	const dataColumns = base.columns.map((col, i) => ({
 		name: col.name,
@@ -90,10 +91,11 @@ function icebergTableSpec(table, encoding = "string") {
 		]
 	};
 }
-const ICEBERG_SCHEMAS = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t)]));
+const ICEBERG_SCHEMAS_STRING = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "string")]));
 const ICEBERG_SCHEMAS_INT = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "int")]));
-function icebergSchemasFor(encoding = "string") {
-	return encoding === "int" ? ICEBERG_SCHEMAS_INT : ICEBERG_SCHEMAS;
+const ICEBERG_SCHEMAS = ICEBERG_SCHEMAS_INT;
+function icebergSchemasFor(encoding = "int") {
+	return encoding === "int" ? ICEBERG_SCHEMAS_INT : ICEBERG_SCHEMAS_STRING;
 }
 const ICEBERG_TABLE_SET = new Set(ICEBERG_TABLES);
 function isIcebergTable(table) {
@@ -103,4 +105,4 @@ function assertIcebergTable(table) {
 	if (!isIcebergTable(table)) throw new Error(`Unknown Iceberg table '${table}'. Expected one of: ${ICEBERG_TABLES.join(", ")}`);
 	return table;
 }
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };

package/dist/_chunks/sink.d.mts

@@ -53,7 +53,7 @@ interface IcebergColumn {
 /**
  * Partition-key encoding for the two identity columns (`site_id`, `search_type`).
  *
- * - `'string'` (default, legacy): both columns are STRING. Correct, but R2 SQL's
+ * - `'string'` (legacy): both columns are STRING. Correct, but R2 SQL's
  *   string min/max statistics are truncated in predicate pushdown, so a bare
  *   `WHERE site_id='<uuid>'` UNDERCOUNTS — callers must CONCAT(col,'') to stay
  *   correct, which defeats partition pruning.
@@ -66,10 +66,12 @@ interface IcebergColumn {
  *   {@link SEARCH_TYPE_INT} for `search_type` (engine-owned, fixed enum).
  *
  * New per-team catalogs are provisioned `'int'`; existing catalogs stay
- * `'string'`. Purely additive: `'string'` is the default everywhere so existing
- * tables, writers, and readers are unchanged.
+ * `'string'`. Callers that read or write legacy catalogs must pass
+ * `encoding: 'string'` explicitly.
  */
 type PartitionKeyEncoding = 'string' | 'int';
+/** Default for new Iceberg/R2 Data Catalog tables. */
+declare const DEFAULT_PARTITION_KEY_ENCODING: PartitionKeyEncoding;
 /**
  * Stable `search_type` enum → int map for `'int'`-encoded catalogs. Engine-owned
  * and FROZEN: never renumber or reuse an id (it's the on-disk partition value).
@@ -142,11 +144,13 @@ declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
  * source of truth — do not hand-list columns elsewhere.
  */
 declare function icebergTableSpec(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergTableSpec;
-/** All Iceberg table specs (legacy `'string'` encoding), keyed by table name. */
-declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+/** All Iceberg table specs in legacy `'string'` encoding, keyed by table name. */
+declare const ICEBERG_SCHEMAS_STRING: Record<IcebergTableName, IcebergTableSpec>;
 /** All Iceberg table specs in `'int'` encoding (INT site_id + INT search_type). */
 declare const ICEBERG_SCHEMAS_INT: Record<IcebergTableName, IcebergTableSpec>;
-/** Table specs for the given encoding (`'string'` default). */
+/** All Iceberg table specs for the default new-catalog encoding. */
+declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+/** Table specs for the given encoding (`'int'` default). */
 declare function icebergSchemasFor(encoding?: PartitionKeyEncoding): Record<IcebergTableName, IcebergTableSpec>;
 /** True when `table` is one of the canonical {@link ICEBERG_TABLES}. */
 declare function isIcebergTable(table: string): table is IcebergTableName;
@@ -385,7 +389,8 @@ interface ListIcebergDataFilesOptions {
   /**
    * Partition-key encoding of the catalog. `'int'` changes how manifest-summary
    * bounds are decoded (int bytes vs UTF-8) and how the per-file partition value
-   * is compared. Defaults to `'string'`.
+   * is compared. Defaults to `'int'` for new catalogs; pass `'string'` for
+   * legacy catalogs.
    */
   encoding?: PartitionKeyEncoding;
   /**
@@ -542,10 +547,11 @@ interface IcebergAppendSinkOptions extends SinkOptions {
    */
   commitRetry?: CommitRetryOptions;
   /**
-   * Partition-key encoding (default `'string'`). `'int'` writes BOTH `site_id`
-   * and `search_type` as INT — the caller MUST pass the numeric `site_id` (a
-   * numeric string is fine; it's `Number()`-coerced) in `slice.ctx.siteId`. A
-   * small INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
+   * Partition-key encoding (default `'int'` for new catalogs). `'int'` writes
+   * BOTH `site_id` and `search_type` as INT — the caller MUST pass the numeric
+   * `site_id` (a numeric string is fine; it's `Number()`-coerced) in
+   * `slice.ctx.siteId`. Pass `'string'` explicitly for legacy catalogs. A small
+   * INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
    * {@link import('./iceberg/schema').PartitionKeyEncoding}.
    */
   encoding?: PartitionKeyEncoding;
@@ -559,4 +565,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/_chunks/types.d.mts

@@ -10,6 +10,9 @@ interface ResolverAdapter<TableKey extends string = string> {
   dimColumn: (dim: Dimension, tableKey: TableKey) => string;
   isMetricDimension: (dim: string) => dim is Metric;
   tableRef: (tableKey: TableKey) => SQL;
+  fromSql: (tableKey: TableKey, options?: {
+    queryCanonical?: boolean;
+  }) => SQL;
   dateColRef: (tableKey: TableKey) => SQL;
   urlToPathExpr: (col: string) => string;
   siteIdColRef?: (tableKey: TableKey) => SQL;

package/dist/adapters/hyparquet.mjs

@@ -1,4 +1,4 @@
-import { SCHEMAS, TABLE_METADATA, dedupeByNaturalKey } from "../_chunks/schema.mjs";
+import { SCHEMAS, TABLE_METADATA } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "../_chunks/libs/hyparquet.mjs";
 import { ByteWriter, parquetWriteRows } from "../_chunks/libs/hyparquet-writer.mjs";
 const ROW_GROUP_SIZE = 25e3;
@@ -119,6 +119,9 @@ function sortRowsByClusterKey(table, rows) {
 	});
 	return copy;
 }
+function naturalKeyFor(table, row) {
+	return TABLE_METADATA[table].sortKey.map((col) => `${row[col] ?? ""}`).join("\0");
+}
 function encodeOrderedRows(rows, columns, rowGroupSize) {
 	const schema = buildWriteSchema(columns);
 	const isDate = columns.map((col) => col.type === "DATE");
@@ -222,12 +225,12 @@ function createHyparquetCodec(options = {}) {
 					rowCount: 0
 				};
 			}
-			const allRows = [];
+			const byNaturalKey = /* @__PURE__ */ new Map();
 			for (const key of inputKeys) {
 				const rows = await decodeParquetToRows(await dataSource.read(key));
-				for (let i = 0; i < rows.length; i++) allRows.push(rows[i]);
+				for (let i = 0; i < rows.length; i++) byNaturalKey.set(naturalKeyFor(ctx.table, rows[i]), rows[i]);
 			}
-			const rows = dedupeByNaturalKey(ctx.table, allRows);
+			const rows = [...byNaturalKey.values()];
 			const bytes = encodeRowsToParquet(ctx.table, rows);
 			await dataSource.write(outputKey, bytes);
 			return {

package/dist/adapters/r2-manifest.mjs

@@ -4,10 +4,26 @@ import { err, ok, unwrapResult } from "gscdump/result";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 const CAS_BACKOFF_BASE_MS = 5;
 const CAS_BACKOFF_CAP_MS = 250;
+const SHARD_IO_CONCURRENCY = 8;
 async function casBackoff(attempt) {
 	const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
 	await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
 }
+async function mapWithConcurrency(items, concurrency, fn) {
+	if (items.length === 0) return [];
+	const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency)));
+	const results = Array.from({ length: items.length }, () => void 0);
+	let nextIndex = 0;
+	async function worker() {
+		while (true) {
+			const index = nextIndex++;
+			if (index >= items.length) return;
+			results[index] = await fn(items[index], index);
+		}
+	}
+	await Promise.all(Array.from({ length: workerCount }, worker));
+	return results;
+}
 function defaultSnapshotId() {
 	return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
 }
@@ -155,16 +171,15 @@ function createR2ManifestStore(opts) {
 		return (await listShards()).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
 	}
 	async function readEntriesAcrossShards(filter, includeRetired) {
-		const shards = await shardsForFilter(filter);
-		const all = [];
-		for (const { siteId, table } of shards) {
+		return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 			const { snapshot } = await readShard(siteId, table);
+			const entries = [];
 			for (const entry of snapshot.entries) {
 				if (!includeRetired && entry.retiredAt !== void 0) continue;
-				if (matchesEntryFilter(entry, filter)) all.push(entry);
+				if (matchesEntryFilter(entry, filter)) entries.push(entry);
 			}
-		}
-		return all;
+			return entries;
+		})).flat();
 	}
 	function groupBySiteTable(entries) {
 		const out = /* @__PURE__ */ new Map();
@@ -195,7 +210,7 @@ function createR2ManifestStore(opts) {
 		}
 		for (const e of newEntries) bucket(e, "new");
 		if (superseding) for (const e of superseding) bucket(e, "super");
-		for (const [shardKey, { newEntries: news, superseding: supers }] of byShard) {
+		await mapWithConcurrency([...byShard], SHARD_IO_CONCURRENCY, async ([shardKey, { newEntries: news, superseding: supers }]) => {
 			const [siteId, table] = shardKey.split("\0");
 			await mutateShard(siteId, table, (snap) => {
 				const byObjectKey = new Map(snap.entries.map((e) => [e.objectKey, e]));
@@ -209,7 +224,7 @@ function createR2ManifestStore(opts) {
 				for (const n of news) byObjectKey.set(n.objectKey, n);
 				snap.entries = Array.from(byObjectKey.values());
 			});
-		}
+		});
 	}
 	return {
 		async listLive(filter) {
@@ -226,32 +241,29 @@ function createR2ManifestStore(opts) {
 			return registerVersionsImpl(entries, superseding);
 		},
 		async listRetired(olderThan) {
-			const shards = await listShards();
-			const out = [];
-			for (const { siteId, table } of shards) {
+			return (await mapWithConcurrency(await listShards(), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				for (const e of snapshot.entries) if (e.retiredAt !== void 0 && e.retiredAt <= olderThan) out.push(e);
-			}
-			return out;
+				const retired = [];
+				for (const e of snapshot.entries) if (e.retiredAt !== void 0 && e.retiredAt <= olderThan) retired.push(e);
+				return retired;
+			})).flat();
 		},
 		async delete(toDelete) {
-			const grouped = groupBySiteTable(toDelete);
-			for (const [shardKey, entries] of grouped) {
+			await mapWithConcurrency([...groupBySiteTable(toDelete)], SHARD_IO_CONCURRENCY, async ([shardKey, entries]) => {
 				const [siteId, table] = shardKey.split("\0");
 				await mutateShard(siteId, table, (snap) => {
 					const drop = new Set(entries.map((e) => e.objectKey));
 					snap.entries = snap.entries.filter((e) => !drop.has(e.objectKey));
 				});
-			}
+			});
 		},
 		async getWatermarks(filter) {
-			const shards = await shardsForFilter(filter);
-			const out = [];
-			for (const { siteId, table } of shards) {
+			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter)) out.push(w);
-			}
-			return out;
+				const watermarks = [];
+				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter)) watermarks.push(w);
+				return watermarks;
+			})).flat();
 		},
 		async bumpWatermark(scope, date, at) {
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires watermarks to carry siteId");
@@ -282,13 +294,12 @@ function createR2ManifestStore(opts) {
 			});
 		},
 		async getSyncStates(filter) {
-			const shards = await shardsForFilter(filter);
-			const out = [];
-			for (const { siteId, table } of shards) {
+			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter)) out.push(s);
-			}
-			return out;
+				const states = [];
+				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter)) states.push(s);
+				return states;
+			})).flat();
 		},
 		async setSyncState(scope, state, detail) {
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");

package/dist/entities.d.mts

@@ -34,8 +34,8 @@ interface QueryDimDeps {
 }
 /**
  * Pure: distinct raw queries → dimension records. De-dupes, drops empties, and
- * folds an empty/whitespace canonical back to the raw query so the key is
- * total (matches the read path's `COALESCE(NULLIF(query_canonical, ''), query)`).
+ * folds an empty/whitespace canonical back to the raw query so the dimension
+ * is total for read-time joins.
  */
 declare function buildQueryDimRecords(queries: Iterable<string>, deps: QueryDimDeps): QueryDimRecord[];
 interface QueryDimStore {

package/dist/iceberg/index.d.mts

@@ -1,4 +1,4 @@
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
 import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
 type IcebergAppendSink = Sink;
 /**
@@ -10,4 +10,4 @@ type IcebergAppendSink = Sink;
  * with no rows never touches the network.
  */
 declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
-export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
+export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };

package/dist/iceberg/index.mjs

@@ -1,6 +1,6 @@
 import { TABLE_METADATA } from "../_chunks/schema.mjs";
 import { engineErrors } from "../errors.mjs";
-import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
 import { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "../_chunks/libs/icebird.mjs";
 import { err, ok } from "gscdump/result";
 async function cacheGet(cache, key, now) {
@@ -36,7 +36,7 @@ function decodeInt(bytes) {
 	if (u == null) return null;
 	return new DataView(u.buffer, u.byteOffset, u.byteLength).getInt32(0, true);
 }
-function buildPartitionFilter(siteId, searchType, wantedMonths, encoding = "string") {
+function buildPartitionFilter(siteId, searchType, wantedMonths, encoding = "int") {
 	return (partitions) => {
 		const parts = partitions;
 		if (!parts || parts.length === 0) return true;
@@ -79,7 +79,7 @@ const ICEBERG_TYPE_MAP = {
 	DOUBLE: "double",
 	DATE: "date"
 };
-function icebergSchemaFor(table, encoding = "string") {
+function icebergSchemaFor(table, encoding = "int") {
 	return {
 		"type": "struct",
 		"schema-id": 0,
@@ -91,7 +91,7 @@ function icebergSchemaFor(table, encoding = "string") {
 		}))
 	};
 }
-function icebergPartitionSpecFor(table, encoding = "string") {
+function icebergPartitionSpecFor(table, encoding = "int") {
 	const fields = icebergSchemasFor(encoding)[table].columns;
 	const fieldId = (name) => {
 		const col = fields.find((c) => c.name === name);
@@ -108,7 +108,7 @@ function icebergPartitionSpecFor(table, encoding = "string") {
 		}))
 	};
 }
-function icebergSortOrderFor(table, encoding = "string") {
+function icebergSortOrderFor(table, encoding = "int") {
 	const fields = icebergSchemasFor(encoding)[table].columns;
 	const fieldId = (name) => {
 		const col = fields.find((c) => c.name === name);
@@ -230,7 +230,7 @@ async function appendAlreadyLanded(args, appendId) {
 async function ensureIcebergNamespace(conn) {
 	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
 }
-async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "string") {
+async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "int") {
 	const results = [];
 	for (const table of tables) await icebergCreateTable({
 		catalog: conn.catalog,
@@ -344,7 +344,7 @@ async function listIcebergDataFiles(conn, opts) {
 		if (snapshotId == null || !metadata) return [];
 	}
 	const endWalk = profiler?.start("iceberg.walk");
-	const partitionFilter = buildPartitionFilter(opts.siteId, opts.searchType, wantedMonths, opts.encoding ?? "string");
+	const partitionFilter = buildPartitionFilter(opts.siteId, opts.searchType, wantedMonths, opts.encoding ?? "int");
 	const manifests = await icebergManifests({
 		metadata,
 		resolver: conn.resolver,
@@ -466,7 +466,7 @@ function toRecords(slice, rows, encoding) {
 }
 function createIcebergAppendSink(options) {
 	let connection;
-	const encoding = options.encoding ?? "string";
+	const encoding = options.encoding ?? "int";
 	const buffers = /* @__PURE__ */ new Map();
 	function connect() {
 		connection ??= connectIcebergCatalog(options.catalog);
@@ -530,4 +530,4 @@ function createIcebergAppendSink(options) {
 		}
 	};
 }
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };