@gscdump/engine 0.30.0 → 0.31.1

package/dist/_chunks/engine.mjs

@@ -1,5 +1,5 @@
 import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
-import { SCHEMAS, currentSchemaVersion, dateColumnsFor, dedupeByNaturalKey } from "./schema.mjs";
+import { SCHEMAS, TABLE_METADATA, currentSchemaVersion, dateColumnsFor, dedupeByNaturalKey } from "./schema.mjs";
 import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from "./compaction.mjs";
 import { dateReplaceClause as dateReplaceClause$1 } from "../sql-fragments.mjs";
 import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mjs";
@@ -100,11 +100,14 @@ function createDuckDBCodec(factory) {
 		}
 	};
 }
+const quoteCol = (c) => `"${c.replace(/"/g, "\"\"")}"`;
 function dedupedMergeSql(table, fileListSql) {
 	const base = `SELECT * FROM read_parquet([${fileListSql}], union_by_name = true)`;
-	const key = SCHEMAS[table].sortKey;
-	if (key.length === 0) return base;
-	return `${base} QUALIFY row_number() OVER (PARTITION BY ${key.map((c) => `"${c.replace(/"/g, "\"\"")}"`).join(", ")}) = 1`;
+	const sortKey = SCHEMAS[table].sortKey;
+	const clusterKey = TABLE_METADATA[table].clusterKey;
+	const dedup = sortKey.length === 0 ? base : `${base} QUALIFY row_number() OVER (PARTITION BY ${sortKey.map(quoteCol).join(", ")}) = 1`;
+	if (clusterKey.length === 0) return dedup;
+	return `${dedup} ORDER BY ${clusterKey.map(quoteCol).join(", ")}`;
 }
 function rewriteEmptyFileSets(sql, placeholders, defaultTable, placeholderTables) {
 	let out = sql;

package/dist/_chunks/entities.mjs

@@ -16,6 +16,105 @@ async function readOptional(ds, key, signal) {
 		throw e;
 	});
 }
+const QUERY_DIM_COLUMNS = [
+	{
+		name: "query",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "query_canonical",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "intent_code",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "normalizer_version",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "intent_version",
+		type: "INTEGER",
+		nullable: false
+	}
+];
+function queryDimPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/query_dim` : `u_${ctx.userId}/entities/query_dim`;
+}
+function queryDimParquetKey(ctx) {
+	return `${queryDimPrefix(ctx)}/index.parquet`;
+}
+function queryDimMetaKey(ctx) {
+	return `${queryDimPrefix(ctx)}/index.json`;
+}
+function buildQueryDimRecords(queries, deps) {
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	for (const raw of queries) {
+		const query = String(raw);
+		if (query.trim() === "" || seen.has(query)) continue;
+		seen.add(query);
+		const canonical = deps.normalizeQuery(query);
+		out.push({
+			query,
+			query_canonical: canonical === "" ? query : canonical,
+			intent_code: deps.classifyIntentCode(query),
+			normalizer_version: deps.normalizerVersion,
+			intent_version: deps.intentVersion
+		});
+	}
+	return out;
+}
+function createQueryDimStore({ dataSource }) {
+	async function exists(key, prefix) {
+		return (await dataSource.list(prefix)).includes(key);
+	}
+	return {
+		parquetKey: queryDimParquetKey,
+		async write(ctx, records, builtAt) {
+			const parquetKey = queryDimParquetKey(ctx);
+			const bytes = encodeRowsToParquetFlex(records, {
+				columns: QUERY_DIM_COLUMNS,
+				sortKey: ["query"]
+			});
+			await dataSource.write(parquetKey, bytes);
+			const meta = {
+				version: 1,
+				builtAt,
+				rowCount: records.length,
+				normalizerVersion: records[0]?.normalizer_version ?? 0,
+				intentVersion: records[0]?.intent_version ?? 0
+			};
+			await dataSource.write(queryDimMetaKey(ctx), new TextEncoder().encode(JSON.stringify(meta)));
+			return {
+				parquetKey,
+				rowCount: records.length
+			};
+		},
+		async loadMeta(ctx) {
+			const key = queryDimMetaKey(ctx);
+			if (!await exists(key, `${queryDimPrefix(ctx)}/`)) return null;
+			const bytes = await dataSource.read(key);
+			return JSON.parse(new TextDecoder().decode(bytes));
+		},
+		async loadRecords(ctx) {
+			const key = queryDimParquetKey(ctx);
+			if (!await exists(key, `${queryDimPrefix(ctx)}/`)) return [];
+			return (await decodeParquetToRows(await dataSource.read(key))).map((r) => ({
+				query: String(r.query),
+				query_canonical: String(r.query_canonical),
+				intent_code: Number(r.intent_code),
+				normalizer_version: Number(r.normalizer_version),
+				intent_version: Number(r.intent_version)
+			}));
+		}
+	};
+}
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 function inspectionIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
@@ -893,4 +992,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/_chunks/libs/icebird.d.mts

@@ -133,7 +133,7 @@ interface Snapshot {
   'sequence-number': number;
   'timestamp-ms': number;
   'manifest-list': string;
-  manifests?: Manifest$1[];
+  manifests?: Manifest[];
   summary: {
     // spec: "value of these fields should be of string type"
     operation: string; // 'spark.app.id'?: string
@@ -192,7 +192,7 @@ interface MetadataLog {
   'timestamp-ms': number;
   'metadata-file': string;
 }
-interface Manifest$1 {
+interface Manifest {
   manifest_path: string;
   manifest_length: bigint;
   partition_spec_id: number;

package/dist/_chunks/resolver.mjs

@@ -297,8 +297,10 @@ function createSqlFragments(config) {
 			if (isMetricDimension(f.dimension)) continue;
 			if (f.dimension === "date") continue;
 			if (f.operator === "topLevel") continue;
-			const cRef = colRef(tableKey, dimColumn(f.dimension, tableKey));
-			const matchExpr = f.dimension === "page" ? dimExprSql(f.dimension, tableKey) : cRef;
+			const dim = f.dimension;
+			const cRef = colRef(tableKey, dimColumn(dim, tableKey));
+			const matchExpr = dim === "page" || dim === "queryCanonical" ? dimExprSql(dim, tableKey) : cRef;
+			const patternExpr = dim === "queryCanonical" ? matchExpr : cRef;
 			switch (f.operator) {
 				case "equals":
 					preds.push(sql`${matchExpr} = ${f.expression}`);
@@ -307,16 +309,16 @@ function createSqlFragments(config) {
 					preds.push(sql`${matchExpr} != ${f.expression}`);
 					break;
 				case "contains":
-					preds.push(sql`${cRef} LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					preds.push(sql`${patternExpr} LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
 					break;
 				case "notContains":
-					preds.push(sql`${cRef} NOT LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
+					preds.push(sql`${patternExpr} NOT LIKE ${`%${escapeLike(f.expression)}%`} ESCAPE '\\'`);
 					break;
 				case "includingRegex":
-					preds.push(regexPredicate(cRef, f.expression, false));
+					preds.push(regexPredicate(patternExpr, f.expression, false));
 					break;
 				case "excludingRegex":
-					preds.push(regexPredicate(cRef, f.expression, true));
+					preds.push(regexPredicate(patternExpr, f.expression, true));
 					break;
 			}
 		}

package/dist/_chunks/sink.d.mts

@@ -185,6 +185,18 @@ interface IcebergPartitionSpec {
   'spec-id': number;
   'fields': IcebergPartitionSpecField[];
 }
+/** A field in an icebird `SortOrder`. */
+interface IcebergSortOrderField {
+  'source-id': number;
+  'transform': 'identity';
+  'direction': 'asc' | 'desc';
+  'null-order': 'nulls-first' | 'nulls-last';
+}
+/** An icebird `SortOrder` (Iceberg write-order). */
+interface IcebergSortOrder {
+  'order-id': number;
+  'fields': IcebergSortOrderField[];
+}
 /** Everything needed to talk to the R2 Data Catalog. */
 interface IcebergCatalogConfig {
   /** REST catalog URI, e.g. `https://catalog.cloudflarestorage.com/<acct>/<warehouse>`. */
@@ -220,6 +232,21 @@ declare function icebergSchemaFor(table: IcebergTableName, encoding?: PartitionK
  * {@link icebergSchemaFor}.
  */
 declare function icebergPartitionSpecFor(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergPartitionSpec;
+/**
+ * Build the icebird `SortOrder` for a fact table from its `clusterKey`
+ * (dimension-first, then `date`) — e.g. `pages` → sort by `url`, then `date`.
+ *
+ * Declared so any sort-aware compaction (a self-run `icebergRewrite`, or R2
+ * managed compaction if/when it honors sort order) re-clusters merged files the
+ * same way the append path already orders them ({@link sortByClusterKey} in
+ * `append-sink.ts`). R2's managed compaction currently only bin-packs small
+ * files without re-sorting, so this is forward-looking: it costs nothing today
+ * (the table simply carries the metadata) and means a future sort-aware pass
+ * produces globally clustered files for free, maximizing row-group skipping on
+ * the DuckDB-over-R2 read path. clusterKey columns are all non-null, so the
+ * null ordering is moot; `identity`/`asc` mirrors the physical write order.
+ */
+declare function icebergSortOrderFor(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergSortOrder;
 /** Options for {@link connectIcebergCatalog}. */
 interface ConnectIcebergOptions {
   /**
@@ -501,4 +528,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/entities.d.mts

@@ -1,6 +1,60 @@
 import { DataSource } from "./_chunks/storage.mjs";
 import { ScheduleState } from "./schedule.mjs";
 import { ColumnDef, TenantCtx } from "@gscdump/contracts";
+interface QueryDimRecord {
+  query: string;
+  /** Lexical canonical, never empty: NULL/'' folds to the raw query. */
+  query_canonical: string;
+  /** Packed search-intent code (see `@gscdump/analysis` `encodeIntent`). */
+  intent_code: number;
+  normalizer_version: number;
+  intent_version: number;
+}
+/** JSON sidecar: versions + freshness, readable without decoding the parquet. */
+interface QueryDimMeta {
+  version: 1;
+  builtAt: number;
+  rowCount: number;
+  normalizerVersion: number;
+  intentVersion: number;
+}
+declare function queryDimParquetKey(ctx: TenantCtx): string;
+declare function queryDimMetaKey(ctx: TenantCtx): string;
+/**
+ * Injected derivation. `engine` never imports `@gscdump/analysis`; the host
+ * passes `normalizeQuery` / `classifyIntentCode` (e.g. `encodeIntent ∘
+ * classifyQueryIntent`) plus their version constants.
+ */
+interface QueryDimDeps {
+  normalizeQuery: (query: string) => string;
+  normalizerVersion: number;
+  /** Returns the packed intent code for a raw query. */
+  classifyIntentCode: (query: string) => number;
+  intentVersion: number;
+}
+/**
+ * Pure: distinct raw queries → dimension records. De-dupes, drops empties, and
+ * folds an empty/whitespace canonical back to the raw query so the key is
+ * total (matches the read path's `COALESCE(NULLIF(query_canonical, ''), query)`).
+ */
+declare function buildQueryDimRecords(queries: Iterable<string>, deps: QueryDimDeps): QueryDimRecord[];
+interface QueryDimStore {
+  parquetKey: (ctx: TenantCtx) => string;
+  /** Write the parquet + JSON sidecar. Last-write-wins; no history. */
+  write: (ctx: TenantCtx, records: readonly QueryDimRecord[], builtAt: number) => Promise<{
+    parquetKey: string;
+    rowCount: number;
+  }>;
+  /** Read the sidecar (versions + freshness), or null on first build. */
+  loadMeta: (ctx: TenantCtx) => Promise<QueryDimMeta | null>;
+  /** Decode the dimension rows (test/inspection; reads JOIN the parquet by key). */
+  loadRecords: (ctx: TenantCtx) => Promise<QueryDimRecord[]>;
+}
+declare function createQueryDimStore({
+  dataSource
+}: {
+  dataSource: DataSource;
+}): QueryDimStore;
 /**
  * GSC URL inspection result fields we persist. Mirrors the
  *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON
@@ -442,4 +496,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/entities.mjs

@@ -1,2 +1,2 @@
-import { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
-export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+import { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
+export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/iceberg/index.d.mts

@@ -1,4 +1,4 @@
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
 import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
 type IcebergAppendSink = Sink;
 /**
@@ -10,4 +10,4 @@ type IcebergAppendSink = Sink;
  * with no rows never touches the network.
  */
 declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
-export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
+export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, type PartitionKeyEncoding, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };

package/dist/iceberg/index.mjs

@@ -1,3 +1,4 @@
+import { TABLE_METADATA } from "../_chunks/schema.mjs";
 import { engineErrors } from "../errors.mjs";
 import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
 import { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "../_chunks/libs/icebird.mjs";
@@ -107,6 +108,23 @@ function icebergPartitionSpecFor(table, encoding = "string") {
 		}))
 	};
 }
+function icebergSortOrderFor(table, encoding = "string") {
+	const fields = icebergSchemasFor(encoding)[table].columns;
+	const fieldId = (name) => {
+		const col = fields.find((c) => c.name === name);
+		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"order-id": 1,
+		"fields": TABLE_METADATA[table].clusterKey.map((col) => ({
+			"source-id": fieldId(col),
+			"transform": "identity",
+			"direction": "asc",
+			"null-order": "nulls-last"
+		}))
+	};
+}
 const CATALOG_CONFIG_TTL_MS = 3600 * 1e3;
 function catalogConfigKey(config) {
 	return `gsc-catalog-cfg\0${config.catalogUri}\0${config.warehouse}`;
@@ -187,7 +205,8 @@ async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "st
 		namespace: conn.namespace,
 		table,
 		schema: icebergSchemaFor(table, encoding),
-		partitionSpec: icebergPartitionSpecFor(table, encoding)
+		partitionSpec: icebergPartitionSpecFor(table, encoding),
+		sortOrder: icebergSortOrderFor(table, encoding)
 	}).then(() => results.push({
 		table,
 		outcome: ok(void 0)
@@ -369,6 +388,24 @@ function dedupeByIdentity(table, records) {
 	}
 	return seen.size === records.length ? records : [...seen.values()];
 }
+function sortByClusterKey(table, records) {
+	const cols = TABLE_METADATA[table].clusterKey;
+	if (cols.length === 0 || records.length < 2) return records;
+	return records.slice().sort((a, b) => {
+		for (const col of cols) {
+			const av = a[col];
+			const bv = b[col];
+			if (av === bv) continue;
+			if (av == null) return -1;
+			if (bv == null) return 1;
+			if (typeof av === "number" && typeof bv === "number") return av - bv;
+			const as = String(av);
+			const bs = String(bv);
+			if (as !== bs) return as < bs ? -1 : 1;
+		}
+		return 0;
+	});
+}
 function toRecords(slice, rows, encoding) {
 	const siteVal = encoding === "int" ? toIntPartitionSiteId(slice.ctx.siteId) : slice.ctx.siteId ?? "";
 	const searchVal = encoding === "int" ? SEARCH_TYPE_INT[slice.searchType] : slice.searchType;
@@ -423,7 +460,7 @@ function createIcebergAppendSink(options) {
 			}
 			for (const [table, records] of buffers) {
 				if (records.length === 0) continue;
-				const deduped = dedupeByIdentity(table, records);
+				const deduped = sortByClusterKey(table, dedupeByIdentity(table, records));
 				await icebergAppendRetrying({
 					catalog: conn.catalog,
 					namespace: conn.namespace,
@@ -447,4 +484,4 @@ function createIcebergAppendSink(options) {
 		}
 	};
 }
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };

package/dist/rollups.d.mts

@@ -244,6 +244,12 @@ declare function runWindowed(opts: {
     start: string;
     end: string;
   }) => string;
+  /**
+   * Extra named file sets merged into every window's `runSQL` (alongside the
+   * windowed `FILES`). Use to JOIN a non-windowed sidecar (e.g. the query
+   * dimension parquet via `{ QUERY_DIM: { keys: [...] } }`) inside `sqlFor`.
+   */
+  extraFileSets?: Record<string, FileSetRef>;
 }): Promise<Row$1[]>;
 /**
  * Daily totals across the full history. One row per (date, table) with

package/dist/rollups.mjs

@@ -1,7 +1,7 @@
 import "./_chunks/layout.mjs";
 import { engineErrors } from "./errors.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
+import { createIndexingMetadataStore, createQueryDimStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx, searchType) {
 	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
@@ -237,10 +237,13 @@ async function runWindowed(opts) {
 		const result = await opts.engine.runSQL({
 			ctx: opts.ctx,
 			table: opts.table,
-			fileSets: { FILES: {
-				table: opts.table,
-				partitions: w.partitions
-			} },
+			fileSets: {
+				FILES: {
+					table: opts.table,
+					partitions: w.partitions
+				},
+				...opts.extraFileSets
+			},
 			sql: opts.sqlFor(w),
 			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
 		});
@@ -638,23 +641,41 @@ const queryCanonicalDailyRollup = {
 		}
 	],
 	parquetSortKey: ["date", "query_canonical"],
-	async build({ engine, ctx, searchType }) {
+	async build({ engine, ctx, dataSource, searchType }) {
+		const dimStore = createQueryDimStore({ dataSource });
+		const useDim = await dimStore.loadMeta(ctx) !== null;
+		const canonExpr = useDim ? `COALESCE(qd.query_canonical, NULLIF(q.query_canonical, ''), q.query)` : `COALESCE(NULLIF(query_canonical, ''), query)`;
 		return (await runWindowed({
 			engine,
 			ctx,
 			table: "queries",
 			...searchType !== void 0 ? { searchType } : {},
-			sqlFor: (w) => `
-        SELECT
-          COALESCE(NULLIF(query_canonical, ''), query) AS query_canonical,
-          CAST(date AS VARCHAR) AS date,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${w.start}' AND date <= '${w.end}'
-        GROUP BY COALESCE(NULLIF(query_canonical, ''), query), date
-      `
+			...useDim ? { extraFileSets: { QUERY_DIM: {
+				table: "queries",
+				keys: [dimStore.parquetKey(ctx)]
+			} } } : {},
+			sqlFor: useDim ? (w) => `
+          SELECT
+            ${canonExpr} AS query_canonical,
+            CAST(q.date AS VARCHAR) AS date,
+            SUM(q.clicks)::BIGINT AS clicks,
+            SUM(q.impressions)::BIGINT AS impressions,
+            SUM(q.sum_position)::DOUBLE AS sum_position
+          FROM read_parquet({{FILES}}, union_by_name = true) q
+          LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query
+          WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'
+          GROUP BY ${canonExpr}, q.date
+        ` : (w) => `
+          SELECT
+            ${canonExpr} AS query_canonical,
+            CAST(date AS VARCHAR) AS date,
+            SUM(clicks)::BIGINT AS clicks,
+            SUM(impressions)::BIGINT AS impressions,
+            SUM(sum_position)::DOUBLE AS sum_position
+          FROM read_parquet({{FILES}}, union_by_name = true)
+          WHERE date >= '${w.start}' AND date <= '${w.end}'
+          GROUP BY ${canonExpr}, date
+        `
 		})).map((r) => ({
 			query_canonical: String(r.query_canonical),
 			date: String(r.date),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.30.0",
+  "version": "0.31.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -191,8 +191,8 @@
     "hyparquet": "^1.26.1",
     "hyparquet-writer": "^0.16.1",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.30.0",
-    "gscdump": "0.30.0"
+    "@gscdump/contracts": "0.31.1",
+    "gscdump": "0.31.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
@@ -208,6 +208,7 @@
     "build": "obuild",
     "typecheck": "tsc --noEmit",
     "test": "vitest",
+    "benchmark-store": "tsx scripts/benchmark-store.mts",
     "r2-harness": "tsx scripts/r2-contention-harness.ts",
     "backfill-audit": "tsx scripts/backfill-audit.ts"
   }