@gscdump/engine 0.32.12 → 0.33.3

package/dist/_chunks/schema.mjs

@@ -14,7 +14,6 @@ const pages = pgTable("pages", {
 });
 const queries = pgTable("queries", {
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -26,7 +25,6 @@ const countries = pgTable("countries", {
 const page_queries = pgTable("page_queries", {
 	url: varchar("url").notNull(),
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -60,7 +58,6 @@ const search_appearance_pages = pgTable("search_appearance_pages", {
 const search_appearance_queries = pgTable("search_appearance_queries", {
 	searchAppearance: varchar("searchAppearance").notNull(),
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -68,7 +65,6 @@ const search_appearance_page_queries = pgTable("search_appearance_page_queries",
 	searchAppearance: varchar("searchAppearance").notNull(),
 	url: varchar("url").notNull(),
 	query: varchar("query").notNull(),
-	query_canonical: varchar("query_canonical"),
 	date: dateCol(),
 	...metricCols()
 });
@@ -99,7 +95,7 @@ const TABLE_METADATA = {
 	queries: {
 		sortKey: ["date", "query"],
 		clusterKey: ["query", "date"],
-		version: 2
+		version: 3
 	},
 	countries: {
 		sortKey: ["date", "country"],
@@ -117,7 +113,7 @@ const TABLE_METADATA = {
 			"query",
 			"date"
 		],
-		version: 2
+		version: 3
 	},
 	dates: {
 		sortKey: ["date"],
@@ -153,7 +149,7 @@ const TABLE_METADATA = {
 			"query",
 			"date"
 		],
-		version: 1
+		version: 2
 	},
 	search_appearance_page_queries: {
 		sortKey: [
@@ -168,7 +164,7 @@ const TABLE_METADATA = {
 			"query",
 			"date"
 		],
-		version: 1
+		version: 2
 	},
 	hourly_pages: {
 		sortKey: [

package/dist/_chunks/schema2.mjs

@@ -10,6 +10,7 @@ const ICEBERG_TABLES = [
 	"search_appearance_queries",
 	"search_appearance_page_queries"
 ];
+const DEFAULT_PARTITION_KEY_ENCODING = "int";
 const SEARCH_TYPE_INT = {
 	web: 1,
 	image: 2,
@@ -30,7 +31,7 @@ const ICEBERG_PARTITION_COLUMNS = [{
 	required: true,
 	fieldId: 2
 }];
-function icebergPartitionColumns(encoding = "string") {
+function icebergPartitionColumns(encoding = "int") {
 	if (encoding === "string") return ICEBERG_PARTITION_COLUMNS;
 	return [{
 		name: "site_id",
@@ -71,7 +72,7 @@ function mapColumnType(t) {
 		case "DATE": return "DATE";
 	}
 }
-function icebergTableSpec(table, encoding = "string") {
+function icebergTableSpec(table, encoding = "int") {
 	const base = SCHEMAS[table];
 	const dataColumns = base.columns.map((col, i) => ({
 		name: col.name,
@@ -90,10 +91,11 @@ function icebergTableSpec(table, encoding = "string") {
 		]
 	};
 }
-const ICEBERG_SCHEMAS = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t)]));
+const ICEBERG_SCHEMAS_STRING = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "string")]));
 const ICEBERG_SCHEMAS_INT = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "int")]));
-function icebergSchemasFor(encoding = "string") {
-	return encoding === "int" ? ICEBERG_SCHEMAS_INT : ICEBERG_SCHEMAS;
+const ICEBERG_SCHEMAS = ICEBERG_SCHEMAS_INT;
+function icebergSchemasFor(encoding = "int") {
+	return encoding === "int" ? ICEBERG_SCHEMAS_INT : ICEBERG_SCHEMAS_STRING;
 }
 const ICEBERG_TABLE_SET = new Set(ICEBERG_TABLES);
 function isIcebergTable(table) {
@@ -103,4 +105,4 @@ function assertIcebergTable(table) {
 	if (!isIcebergTable(table)) throw new Error(`Unknown Iceberg table '${table}'. Expected one of: ${ICEBERG_TABLES.join(", ")}`);
 	return table;
 }
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };

package/dist/_chunks/sink.d.mts

@@ -53,7 +53,7 @@ interface IcebergColumn {
 /**
  * Partition-key encoding for the two identity columns (`site_id`, `search_type`).
  *
- * - `'string'` (default, legacy): both columns are STRING. Correct, but R2 SQL's
+ * - `'string'` (legacy): both columns are STRING. Correct, but R2 SQL's
  *   string min/max statistics are truncated in predicate pushdown, so a bare
  *   `WHERE site_id='<uuid>'` UNDERCOUNTS — callers must CONCAT(col,'') to stay
  *   correct, which defeats partition pruning.
@@ -66,10 +66,12 @@ interface IcebergColumn {
  *   {@link SEARCH_TYPE_INT} for `search_type` (engine-owned, fixed enum).
  *
  * New per-team catalogs are provisioned `'int'`; existing catalogs stay
- * `'string'`. Purely additive: `'string'` is the default everywhere so existing
- * tables, writers, and readers are unchanged.
+ * `'string'`. Callers that read or write legacy catalogs must pass
+ * `encoding: 'string'` explicitly.
  */
 type PartitionKeyEncoding = 'string' | 'int';
+/** Default for new Iceberg/R2 Data Catalog tables. */
+declare const DEFAULT_PARTITION_KEY_ENCODING: PartitionKeyEncoding;
 /**
  * Stable `search_type` enum → int map for `'int'`-encoded catalogs. Engine-owned
  * and FROZEN: never renumber or reuse an id (it's the on-disk partition value).
@@ -142,11 +144,13 @@ declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
  * source of truth — do not hand-list columns elsewhere.
  */
 declare function icebergTableSpec(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergTableSpec;
-/** All Iceberg table specs (legacy `'string'` encoding), keyed by table name. */
-declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+/** All Iceberg table specs in legacy `'string'` encoding, keyed by table name. */
+declare const ICEBERG_SCHEMAS_STRING: Record<IcebergTableName, IcebergTableSpec>;
 /** All Iceberg table specs in `'int'` encoding (INT site_id + INT search_type). */
 declare const ICEBERG_SCHEMAS_INT: Record<IcebergTableName, IcebergTableSpec>;
-/** Table specs for the given encoding (`'string'` default). */
+/** All Iceberg table specs for the default new-catalog encoding. */
+declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+/** Table specs for the given encoding (`'int'` default). */
 declare function icebergSchemasFor(encoding?: PartitionKeyEncoding): Record<IcebergTableName, IcebergTableSpec>;
 /** True when `table` is one of the canonical {@link ICEBERG_TABLES}. */
 declare function isIcebergTable(table: string): table is IcebergTableName;
@@ -385,7 +389,8 @@ interface ListIcebergDataFilesOptions {
   /**
    * Partition-key encoding of the catalog. `'int'` changes how manifest-summary
    * bounds are decoded (int bytes vs UTF-8) and how the per-file partition value
-   * is compared. Defaults to `'string'`.
+   * is compared. Defaults to `'int'` for new catalogs; pass `'string'` for
+   * legacy catalogs.
    */
   encoding?: PartitionKeyEncoding;
   /**
@@ -542,10 +547,11 @@ interface IcebergAppendSinkOptions extends SinkOptions {
    */
   commitRetry?: CommitRetryOptions;
   /**
-   * Partition-key encoding (default `'string'`). `'int'` writes BOTH `site_id`
-   * and `search_type` as INT — the caller MUST pass the numeric `site_id` (a
-   * numeric string is fine; it's `Number()`-coerced) in `slice.ctx.siteId`. A
-   * small INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
+   * Partition-key encoding (default `'int'` for new catalogs). `'int'` writes
+   * BOTH `site_id` and `search_type` as INT — the caller MUST pass the numeric
+   * `site_id` (a numeric string is fine; it's `Number()`-coerced) in
+   * `slice.ctx.siteId`. Pass `'string'` explicitly for legacy catalogs. A small
+   * INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
    * {@link import('./iceberg/schema').PartitionKeyEncoding}.
    */
   encoding?: PartitionKeyEncoding;
@@ -559,4 +565,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/_chunks/types.d.mts

@@ -10,6 +10,9 @@ interface ResolverAdapter<TableKey extends string = string> {
   dimColumn: (dim: Dimension, tableKey: TableKey) => string;
   isMetricDimension: (dim: string) => dim is Metric;
   tableRef: (tableKey: TableKey) => SQL;
+  fromSql: (tableKey: TableKey, options?: {
+    queryCanonical?: boolean;
+  }) => SQL;
   dateColRef: (tableKey: TableKey) => SQL;
   urlToPathExpr: (col: string) => string;
   siteIdColRef?: (tableKey: TableKey) => SQL;

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
+import { manifestEntryKey, matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter, mergeSyncState, syncStateKey, watermarkKey } from "../_chunks/manifest-store-utils.mjs";
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";
@@ -29,13 +29,13 @@ function createFilesystemDataSource(opts) {
 			}));
 		},
 		async list(prefix) {
-			const full = resolve(root, prefix);
+			const full = pathFor(prefix);
 			const out = [];
 			await walk(full, out);
 			return out.map((p) => p.slice(root.length + 1));
 		},
 		async *streamList(prefix) {
-			const full = resolve(root, prefix);
+			const full = pathFor(prefix);
 			for await (const p of walkStream(full)) yield p.slice(root.length + 1);
 		},
 		async head(key) {
@@ -71,57 +71,6 @@ async function walk(dir, out) {
 		else out.push(p);
 	}
 }
-function watermarkKey(w) {
-	return `${w.userId}|${w.siteId ?? ""}|${w.table}`;
-}
-function matchesWatermarkFilter(w, filter) {
-	if (w.userId !== filter.userId) return false;
-	if (filter.siteId !== void 0 && w.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && w.table !== filter.table) return false;
-	return true;
-}
-function syncStateKey(s) {
-	return `${s.userId}|${s.siteId ?? ""}|${s.table}|${s.date}|${inferSearchType(s)}`;
-}
-function matchesSyncStateFilter(s, filter) {
-	if (s.userId !== filter.userId) return false;
-	if (filter.siteId !== void 0 && s.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && s.table !== filter.table) return false;
-	if (filter.state !== void 0 && s.state !== filter.state) return false;
-	if (filter.searchType !== void 0 && inferSearchType(s) !== filter.searchType) return false;
-	return true;
-}
-function mergeSyncState(existing, scope, state, detail) {
-	const at = detail?.at ?? Date.now();
-	const attemptsBump = state === "inflight" ? 1 : 0;
-	if (!existing) return {
-		userId: scope.userId,
-		siteId: scope.siteId,
-		table: scope.table,
-		date: scope.date,
-		state,
-		updatedAt: at,
-		attempts: attemptsBump,
-		error: detail?.error,
-		...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
-	};
-	return {
-		...existing,
-		state,
-		updatedAt: at,
-		attempts: existing.attempts + attemptsBump,
-		error: state === "done" ? void 0 : detail?.error ?? existing.error
-	};
-}
-function matchesFilter(entry, filter) {
-	if (entry.userId !== filter.userId) return false;
-	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && entry.table !== filter.table) return false;
-	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
-	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
-	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
-	return true;
-}
 function lockFileFor(locksDir, scope) {
 	return join(locksDir, `${`${scope.userId}|${scope.siteId ?? ""}|${scope.table}|${scope.partition}`.replace(/[^\w.-]/g, "_")}.lock`);
 }
@@ -166,30 +115,27 @@ function createFilesystemManifestStore(opts) {
 		while (queue.length > 0) await queue.shift()().catch(() => {});
 		running = false;
 	}
-	function entryKey(e) {
-		return e.objectKey;
-	}
 	async function registerVersionsImpl(newEntries, superseding) {
 		const data = await load();
 		const supersededAt = newEntries[0]?.createdAt ?? Date.now();
-		const byKey = new Map(data.entries.map((e) => [entryKey(e), e]));
+		const byKey = new Map(data.entries.map((e) => [manifestEntryKey(e), e]));
 		if (superseding) for (const s of superseding) {
-			const existing = byKey.get(entryKey(s));
-			if (existing && existing.retiredAt === void 0) byKey.set(entryKey(s), {
+			const existing = byKey.get(manifestEntryKey(s));
+			if (existing && existing.retiredAt === void 0) byKey.set(manifestEntryKey(s), {
 				...existing,
 				retiredAt: supersededAt
 			});
 		}
-		for (const e of newEntries) byKey.set(entryKey(e), e);
+		for (const e of newEntries) byKey.set(manifestEntryKey(e), e);
 		data.entries = Array.from(byKey.values());
 		await save(data);
 	}
 	return {
 		async listLive(filter) {
-			return (await load()).entries.filter((e) => e.retiredAt === void 0 && matchesFilter(e, filter));
+			return (await load()).entries.filter((e) => e.retiredAt === void 0 && matchesManifestEntryFilter(e, filter));
 		},
 		async listAll(filter) {
-			return (await load()).entries.filter((e) => matchesFilter(e, filter));
+			return (await load()).entries.filter((e) => matchesManifestEntryFilter(e, filter));
 		},
 		async registerVersion(entry, superseding) {
 			return enqueue(() => registerVersionsImpl([entry], superseding));
@@ -203,8 +149,8 @@ function createFilesystemManifestStore(opts) {
 		async delete(toDelete) {
 			return enqueue(async () => {
 				const data = await load();
-				const toDeleteKeys = new Set(toDelete.map(entryKey));
-				data.entries = data.entries.filter((e) => !toDeleteKeys.has(entryKey(e)));
+				const toDeleteKeys = new Set(toDelete.map(manifestEntryKey));
+				data.entries = data.entries.filter((e) => !toDeleteKeys.has(manifestEntryKey(e)));
 				await save(data);
 			});
 		},

package/dist/adapters/hyparquet.mjs

@@ -1,4 +1,4 @@
-import { SCHEMAS, TABLE_METADATA, dedupeByNaturalKey } from "../_chunks/schema.mjs";
+import { SCHEMAS, TABLE_METADATA } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "../_chunks/libs/hyparquet.mjs";
 import { ByteWriter, parquetWriteRows } from "../_chunks/libs/hyparquet-writer.mjs";
 const ROW_GROUP_SIZE = 25e3;
@@ -119,10 +119,16 @@ function sortRowsByClusterKey(table, rows) {
 	});
 	return copy;
 }
+function naturalKeyFor(table, row) {
+	return TABLE_METADATA[table].sortKey.map((col) => `${row[col] ?? ""}`).join("\0");
+}
 function encodeOrderedRows(rows, columns, rowGroupSize) {
 	const schema = buildWriteSchema(columns);
-	const isDate = columns.map((col) => col.type === "DATE");
-	const types = columns.map((col) => basicTypeFor(col.type));
+	const codecs = columns.map((col) => ({
+		name: col.name,
+		isDate: col.type === "DATE",
+		type: basicTypeFor(col.type)
+	}));
 	const columnSpecs = columns.map((col) => ({
 		name: col.name,
 		nullable: col.nullable,
@@ -131,10 +137,7 @@ function encodeOrderedRows(rows, columns, rowGroupSize) {
 	function* coercedRows() {
 		for (const r of rows) {
 			const out = {};
-			for (let c = 0; c < columns.length; c++) {
-				const name = columns[c].name;
-				out[name] = isDate[c] ? toEpochDays(r[name]) : coerceValue(r[name], types[c]);
-			}
+			for (const codec of codecs) out[codec.name] = codec.isDate ? toEpochDays(r[codec.name]) : coerceValue(r[codec.name], codec.type);
 			yield out;
 		}
 	}
@@ -222,12 +225,12 @@ function createHyparquetCodec(options = {}) {
 					rowCount: 0
 				};
 			}
-			const allRows = [];
+			const byNaturalKey = /* @__PURE__ */ new Map();
 			for (const key of inputKeys) {
 				const rows = await decodeParquetToRows(await dataSource.read(key));
-				for (let i = 0; i < rows.length; i++) allRows.push(rows[i]);
+				for (let i = 0; i < rows.length; i++) byNaturalKey.set(naturalKeyFor(ctx.table, rows[i]), rows[i]);
 			}
-			const rows = dedupeByNaturalKey(ctx.table, allRows);
+			const rows = [...byNaturalKey.values()];
 			const bytes = encodeRowsToParquet(ctx.table, rows);
 			await dataSource.write(outputKey, bytes);
 			return {

package/dist/adapters/r2-manifest.mjs

@@ -1,13 +1,30 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
+import { inferSearchType } from "../_chunks/layout.mjs";
 import { engineErrorToException, engineErrors } from "../errors.mjs";
+import { matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter } from "../_chunks/manifest-store-utils.mjs";
 import { err, ok, unwrapResult } from "gscdump/result";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 const CAS_BACKOFF_BASE_MS = 5;
 const CAS_BACKOFF_CAP_MS = 250;
+const SHARD_IO_CONCURRENCY = 8;
 async function casBackoff(attempt) {
 	const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
 	await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
 }
+async function mapWithConcurrency(items, concurrency, fn) {
+	if (items.length === 0) return [];
+	const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency)));
+	const results = Array.from({ length: items.length }, () => void 0);
+	let nextIndex = 0;
+	async function worker() {
+		while (true) {
+			const index = nextIndex++;
+			if (index >= items.length) return;
+			results[index] = await fn(items[index], index);
+		}
+	}
+	await Promise.all(Array.from({ length: workerCount }, worker));
+	return results;
+}
 function defaultSnapshotId() {
 	return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
 }
@@ -36,26 +53,6 @@ function shardScopesFromEntries(entries) {
 	}
 	return out;
 }
-function matchesEntryFilter(entry, filter) {
-	if (filter.siteId !== void 0 && entry.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && entry.table !== filter.table) return false;
-	if (filter.partitions && !filter.partitions.includes(entry.partition)) return false;
-	if (filter.tier !== void 0 && inferLegacyTier(entry) !== filter.tier) return false;
-	if (filter.searchType !== void 0 && inferSearchType(entry) !== filter.searchType) return false;
-	return true;
-}
-function matchesWatermarkFilter(w, filter) {
-	if (filter.siteId !== void 0 && w.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && w.table !== filter.table) return false;
-	return true;
-}
-function matchesSyncStateFilter(s, filter) {
-	if (filter.siteId !== void 0 && s.siteId !== filter.siteId) return false;
-	if (filter.table !== void 0 && s.table !== filter.table) return false;
-	if (filter.state !== void 0 && s.state !== filter.state) return false;
-	if (filter.searchType !== void 0 && inferSearchType(s) !== filter.searchType) return false;
-	return true;
-}
 function createR2ManifestStore(opts) {
 	const { bucket, userId } = opts;
 	const newSnapshotId = opts.newSnapshotId ?? defaultSnapshotId;
@@ -154,17 +151,20 @@ function createR2ManifestStore(opts) {
 		}];
 		return (await listShards()).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
 	}
+	function assertScopedUser(got, op) {
+		if (got !== userId) throw new Error(`${op}: R2 manifest store is scoped to userId=${userId}, got ${got}`);
+	}
 	async function readEntriesAcrossShards(filter, includeRetired) {
-		const shards = await shardsForFilter(filter);
-		const all = [];
-		for (const { siteId, table } of shards) {
+		assertScopedUser(filter.userId, includeRetired ? "listAll" : "listLive");
+		return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 			const { snapshot } = await readShard(siteId, table);
+			const entries = [];
 			for (const entry of snapshot.entries) {
 				if (!includeRetired && entry.retiredAt !== void 0) continue;
-				if (matchesEntryFilter(entry, filter)) all.push(entry);
+				if (matchesManifestEntryFilter(entry, filter, { ignoreUserId: true })) entries.push(entry);
 			}
-		}
-		return all;
+			return entries;
+		})).flat();
 	}
 	function groupBySiteTable(entries) {
 		const out = /* @__PURE__ */ new Map();
@@ -180,6 +180,7 @@ function createR2ManifestStore(opts) {
 		const supersededAt = newEntries[0]?.createdAt ?? now();
 		const byShard = /* @__PURE__ */ new Map();
 		function bucket(entry, kind) {
+			assertScopedUser(entry.userId, "registerVersions");
 			if (entry.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId");
 			const key = `${entry.siteId}\0${entry.table}`;
 			let bag = byShard.get(key);
@@ -195,7 +196,7 @@ function createR2ManifestStore(opts) {
 		}
 		for (const e of newEntries) bucket(e, "new");
 		if (superseding) for (const e of superseding) bucket(e, "super");
-		for (const [shardKey, { newEntries: news, superseding: supers }] of byShard) {
+		await mapWithConcurrency([...byShard], SHARD_IO_CONCURRENCY, async ([shardKey, { newEntries: news, superseding: supers }]) => {
 			const [siteId, table] = shardKey.split("\0");
 			await mutateShard(siteId, table, (snap) => {
 				const byObjectKey = new Map(snap.entries.map((e) => [e.objectKey, e]));
@@ -209,7 +210,7 @@ function createR2ManifestStore(opts) {
 				for (const n of news) byObjectKey.set(n.objectKey, n);
 				snap.entries = Array.from(byObjectKey.values());
 			});
-		}
+		});
 	}
 	return {
 		async listLive(filter) {
@@ -226,34 +227,33 @@ function createR2ManifestStore(opts) {
 			return registerVersionsImpl(entries, superseding);
 		},
 		async listRetired(olderThan) {
-			const shards = await listShards();
-			const out = [];
-			for (const { siteId, table } of shards) {
+			return (await mapWithConcurrency(await listShards(), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				for (const e of snapshot.entries) if (e.retiredAt !== void 0 && e.retiredAt <= olderThan) out.push(e);
-			}
-			return out;
+				const retired = [];
+				for (const e of snapshot.entries) if (e.retiredAt !== void 0 && e.retiredAt <= olderThan) retired.push(e);
+				return retired;
+			})).flat();
 		},
 		async delete(toDelete) {
-			const grouped = groupBySiteTable(toDelete);
-			for (const [shardKey, entries] of grouped) {
+			await mapWithConcurrency([...groupBySiteTable(toDelete)], SHARD_IO_CONCURRENCY, async ([shardKey, entries]) => {
 				const [siteId, table] = shardKey.split("\0");
 				await mutateShard(siteId, table, (snap) => {
 					const drop = new Set(entries.map((e) => e.objectKey));
 					snap.entries = snap.entries.filter((e) => !drop.has(e.objectKey));
 				});
-			}
+			});
 		},
 		async getWatermarks(filter) {
-			const shards = await shardsForFilter(filter);
-			const out = [];
-			for (const { siteId, table } of shards) {
+			assertScopedUser(filter.userId, "getWatermarks");
+			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter)) out.push(w);
-			}
-			return out;
+				const watermarks = [];
+				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter, { ignoreUserId: true })) watermarks.push(w);
+				return watermarks;
+			})).flat();
 		},
 		async bumpWatermark(scope, date, at) {
+			assertScopedUser(scope.userId, "bumpWatermark");
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires watermarks to carry siteId");
 			const ts = at ?? now();
 			await mutateShard(scope.siteId, scope.table, (snap) => {
@@ -282,15 +282,16 @@ function createR2ManifestStore(opts) {
 			});
 		},
 		async getSyncStates(filter) {
-			const shards = await shardsForFilter(filter);
-			const out = [];
-			for (const { siteId, table } of shards) {
+			assertScopedUser(filter.userId, "getSyncStates");
+			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
 				const { snapshot } = await readShard(siteId, table);
-				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter)) out.push(s);
-			}
-			return out;
+				const states = [];
+				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter, { ignoreUserId: true })) states.push(s);
+				return states;
+			})).flat();
 		},
 		async setSyncState(scope, state, detail) {
+			assertScopedUser(scope.userId, "setSyncState");
 			if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");
 			const at = detail?.at ?? now();
 			const scopeSearchType = inferSearchType(scope);

package/dist/entities.d.mts

@@ -34,8 +34,8 @@ interface QueryDimDeps {
 }
 /**
  * Pure: distinct raw queries → dimension records. De-dupes, drops empties, and
- * folds an empty/whitespace canonical back to the raw query so the key is
- * total (matches the read path's `COALESCE(NULLIF(query_canonical, ''), query)`).
+ * folds an empty/whitespace canonical back to the raw query so the dimension
+ * is total for read-time joins.
  */
 declare function buildQueryDimRecords(queries: Iterable<string>, deps: QueryDimDeps): QueryDimRecord[];
 interface QueryDimStore {

package/dist/iceberg/index.d.mts

@@ -1,4 +1,4 @@
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
 import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
 type IcebergAppendSink = Sink;
 /**
@@ -10,4 +10,4 @@ type IcebergAppendSink = Sink;
  * with no rows never touches the network.
  */
 declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
-export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
+export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };