@gscdump/engine 0.20.3 → 0.21.1

package/dist/index.mjs

@@ -1,13 +1,132 @@
 import { n as coerceRows, t as coerceRow } from "./_chunks/coerce.mjs";
-import { a as dimensionToColumn, d as devices, f as drizzleSchema, g as pages, h as page_keywords, l as TABLE_METADATA, m as keywords, n as allTables, o as inferTable, p as hourly_pages, r as currentSchemaVersion, t as SCHEMAS, u as countries } from "./_chunks/schema.mjs";
+import { a as dimensionToColumn, d as dates, f as drizzleSchema, g as queries, h as pages, l as TABLE_METADATA, m as page_queries, n as allTables, o as inferTable, p as hourly_pages, r as currentSchemaVersion, t as SCHEMAS, u as countries } from "./_chunks/schema.mjs";
 import { a as inferSearchType, c as objectKey, i as inferLegacyTier, n as dayPartition, r as hourPartition, t as DEFAULT_SEARCH_TYPE } from "./_chunks/storage.mjs";
 import { a as RAW_DAILY_COMPACT_THRESHOLD, c as dedupeOverlappingTiers, i as substituteNamedFiles, l as enumeratePartitions, r as resolveParquetSQL, s as countRawDailies, t as FILES_PLACEHOLDER, u as splitOverlappingTiers } from "./_chunks/parquet-plan.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
 import { a as createDuckDBCodec, i as canonicalEmptyParquetSchema, n as createStorageEngine, o as createDuckDBExecutor, r as gcOrphansImpl, t as MAX_DAY_BYTES } from "./_chunks/engine.mjs";
-import { createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
+import { a as ICEBERG_TABLES, i as ICEBERG_SCHEMAS, n as ICEBERG_PARTITION_COLUMNS, o as icebergTableSpec, r as ICEBERG_PARTITION_SPEC, t as ICEBERG_FIELD_ID_BASE } from "./_chunks/iceberg-schema.mjs";
+import { assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import "./planner.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, s3SignedResolver } from "icebird";
+const ICEBERG_TYPE_MAP = {
+	STRING: "string",
+	INT: "int",
+	LONG: "long",
+	DOUBLE: "double",
+	DATE: "date"
+};
+function icebergSchemaFor(table) {
+	return {
+		"type": "struct",
+		"schema-id": 0,
+		"fields": ICEBERG_SCHEMAS[table].columns.map((col) => ({
+			id: col.fieldId,
+			name: col.name,
+			required: col.required,
+			type: ICEBERG_TYPE_MAP[col.type]
+		}))
+	};
+}
+function icebergPartitionSpecFor(table) {
+	const fields = ICEBERG_SCHEMAS[table].columns;
+	const fieldId = (name) => {
+		const col = fields.find((c) => c.name === name);
+		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"spec-id": 0,
+		"fields": ICEBERG_PARTITION_SPEC.map((p, i) => ({
+			"source-id": fieldId(p.sourceColumn),
+			"field-id": 1e3 + i,
+			"name": p.name,
+			"transform": p.transform
+		}))
+	};
+}
+async function connectIcebergCatalog(config) {
+	return {
+		catalog: await restCatalogConnect({
+			url: config.catalogUri,
+			warehouse: config.warehouse,
+			requestInit: { headers: { Authorization: `Bearer ${config.catalogToken}` } }
+		}),
+		resolver: s3SignedResolver({
+			accessKeyId: config.s3.accessKeyId,
+			secretAccessKey: config.s3.secretAccessKey,
+			region: config.s3.region ?? "auto",
+			endpoint: config.s3.endpoint,
+			pathStyle: true
+		}),
+		namespace: config.namespace
+	};
+}
+function isCommitRateLimited(err) {
+	if (err && typeof err === "object" && err.status === 429) return true;
+	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+	return msg.includes("429") || msg.includes("too many commits") || msg.includes("rate limit");
+}
+function defaultCommitSleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function icebergAppendRetrying(args, options = {}) {
+	const maxAttempts = options.maxAttempts ?? 6;
+	const baseDelayMs = options.baseDelayMs ?? 1e3;
+	const maxDelayMs = options.maxDelayMs ?? 2e4;
+	const sleep = options.sleep ?? defaultCommitSleep;
+	const random = options.random ?? Math.random;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		const err = await icebergAppend(args).then(() => void 0, (e) => e);
+		if (err === void 0) return;
+		if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
+		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
+		await sleep(Math.floor(random() * ceiling));
+	}
+}
+async function ensureIcebergNamespace(conn) {
+	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
+}
+async function createIcebergTables(conn, tables = ICEBERG_TABLES) {
+	const results = [];
+	for (const table of tables) await icebergCreateTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		schema: icebergSchemaFor(table),
+		partitionSpec: icebergPartitionSpecFor(table)
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (e) => results.push({
+		table,
+		ok: false,
+		error: String(e)
+	}));
+	return results;
+}
+async function listIcebergTables(conn) {
+	return restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name).sort(), () => []);
+}
+async function dropIcebergTables(conn, tables) {
+	const targets = tables ?? await restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name), () => []);
+	const results = [];
+	for (const table of targets) await icebergDropTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		purgeRequested: true
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (e) => results.push({
+		table,
+		ok: false,
+		error: String(e)
+	}));
+	return results;
+}
 const NOOP_RESULT = {
 	flushed: 0,
 	recovered: 0,
@@ -123,38 +242,177 @@ function createIngestAccumulator(opts) {
 		}
 	};
 }
+const DAY_MILLIS = 864e5;
+function toIcebergDate(value) {
+	if (typeof value === "string") return Math.floor(Date.parse(`${value}T00:00:00Z`) / DAY_MILLIS);
+	if (value instanceof Date) return Math.floor(value.getTime() / DAY_MILLIS);
+	return value;
+}
+function toRecords(slice, rows) {
+	const siteId = slice.ctx.siteId ?? "";
+	return rows.map((row) => ({
+		...row,
+		date: toIcebergDate(row.date),
+		site_id: siteId,
+		search_type: slice.searchType
+	}));
+}
+function createIcebergAppendSink(options) {
+	let connection;
+	const buffers = /* @__PURE__ */ new Map();
+	function connect() {
+		connection ??= connectIcebergCatalog(options.catalog);
+		return connection;
+	}
+	return {
+		capabilities: { appendOnly: true },
+		async emit(slice, rows) {
+			if (rows.length === 0) return { rowCount: 0 };
+			const records = toRecords(slice, rows);
+			const buffer = buffers.get(slice.table);
+			if (buffer) for (let i = 0; i < records.length; i++) buffer.push(records[i]);
+			else buffers.set(slice.table, records);
+			return { rowCount: records.length };
+		},
+		async close() {
+			const flushed = [];
+			const failed = [];
+			if (buffers.size === 0) return {
+				flushed,
+				failed
+			};
+			const conn = await connect().then((c) => c, (err) => {
+				connection = void 0;
+				return { error: String(err) };
+			});
+			if ("error" in conn) {
+				for (const [table, records] of buffers) if (records.length > 0) failed.push({
+					table,
+					error: conn.error
+				});
+				buffers.clear();
+				return {
+					flushed,
+					failed
+				};
+			}
+			for (const [table, records] of buffers) {
+				if (records.length === 0) continue;
+				await icebergAppendRetrying({
+					catalog: conn.catalog,
+					namespace: conn.namespace,
+					table,
+					resolver: conn.resolver,
+					records
+				}, options.commitRetry).then(() => {
+					flushed.push(table);
+				}, (err) => {
+					failed.push({
+						table,
+						error: String(err)
+					});
+				});
+			}
+			buffers.clear();
+			return {
+				flushed,
+				failed
+			};
+		}
+	};
+}
+const KEY_SEP = "\0";
+function partitionKey(slice) {
+	return [
+		slice.table,
+		slice.ctx.siteId ?? "",
+		slice.searchType,
+		slice.date
+	].join(KEY_SEP);
+}
+function tableOfKey(key) {
+	return key.slice(0, key.indexOf(KEY_SEP));
+}
+function withIdentity(slice, rows) {
+	return rows.map((r) => ({
+		...r,
+		site_id: slice.ctx.siteId ?? "",
+		search_type: slice.searchType
+	}));
+}
+function createInMemorySink() {
+	const partitions = /* @__PURE__ */ new Map();
+	let closed = false;
+	function allRows() {
+		return [...partitions.values()].flat();
+	}
+	return {
+		capabilities: { appendOnly: true },
+		async emit(slice, rows) {
+			const key = partitionKey(slice);
+			const stored = withIdentity(slice, rows);
+			const existing = partitions.get(key);
+			if (existing) existing.push(...stored);
+			else partitions.set(key, stored);
+			return { rowCount: stored.length };
+		},
+		async close() {
+			closed = true;
+			return {
+				flushed: [...new Set([...partitions.keys()].map((k) => tableOfKey(k)))],
+				failed: []
+			};
+		},
+		get rows() {
+			return allRows();
+		},
+		get closed() {
+			return closed;
+		},
+		rowsFor(table) {
+			return [...partitions.entries()].filter(([k]) => tableOfKey(k) === table).flatMap(([, v]) => v);
+		},
+		rowsForSlice(slice) {
+			return [...partitions.get(partitionKey(slice)) ?? []];
+		},
+		reset() {
+			partitions.clear();
+			closed = false;
+		}
+	};
+}
 const TABLES_BY_SEARCH_TYPE = {
 	web: [
 		"pages",
-		"keywords",
+		"queries",
 		"countries",
-		"devices",
-		"page_keywords"
+		"page_queries",
+		"dates"
 	],
 	discover: [
 		"pages",
 		"countries",
-		"devices"
+		"dates"
 	],
 	news: [
 		"pages",
 		"countries",
-		"devices"
+		"dates"
 	],
 	googleNews: [
 		"pages",
 		"countries",
-		"devices"
+		"dates"
 	],
 	image: [
 		"pages",
 		"countries",
-		"devices"
+		"dates"
 	],
 	video: [
 		"pages",
 		"countries",
-		"devices"
+		"dates"
 	]
 };
 function parseEnabledSearchTypes(raw) {
@@ -181,10 +439,10 @@ function validateEnabledSearchTypes(value) {
 }
 const TABLE_TIERS = {
 	pages: "critical",
-	keywords: "critical",
+	queries: "critical",
 	countries: "standard",
-	devices: "standard",
-	page_keywords: "extended"
+	dates: "standard",
+	page_queries: "extended"
 };
 function getTableTier(table) {
 	return TABLE_TIERS[table] || "extended";
@@ -215,4 +473,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
 const MIN_COUNTRY_IMPRESSIONS = 10;
 const MAX_SITEMAP_URLS_PER_SITE = 5e4;
 const MAX_TRACKED_URLS_PER_SITE = 2e5;
-export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, dedupeOverlappingTiers, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, parseEnabledSearchTypes, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };

package/dist/ingest.d.mts

@@ -19,8 +19,8 @@ interface IngestOptions {
   /**
    * Canonical form of a query string, stored alongside `query` as
    * `query_canonical`. Site-specific (e.g. synonym groups, stemming); if
-   * omitted, `query_canonical` is null. Applied to `keywords` +
-   * `page_keywords` tables only.
+   * omitted, `query_canonical` is null. Applied to `queries` +
+   * `page_queries` tables only.
    */
   normalizeQuery?: (query: string) => string | null | undefined;
 }
@@ -44,6 +44,26 @@ declare function transformGscRow(table: TableName, apiRow: GscApiRow, options?:
   date: string;
   row: Row;
 } | null;
+/**
+ * Assemble one `dates` row for a single `date` from the two GSC queries that
+ * back the table:
+ *
+ * - `totalsRow` — the GSC `['date']` query result: the TRUE site totals
+ *   (clicks/impressions/position), including anonymized impressions.
+ * - `deviceRows` — the GSC `['date','device']` query results for that date:
+ *   one row per device, pivoted into the 9 `*_{device}` columns.
+ * - `queryGrainedImpressions` — total impressions summed from the
+ *   `['query','date']` (or `['page','query','date']`) query for the same date,
+ *   used to derive `anonymized_impressions_pct`.
+ *
+ * `anonymized_impressions_pct = 1 - query_grained_impressions /
+ * page_grained_impressions`, where the page/date totals come from `totalsRow`.
+ * Mirrors the legacy `dailyTotalsRollup` formula. Clamped to `[0, 1]`.
+ */
+declare function assembleDatesRow(date: string, totalsRow: GscApiRow, deviceRows: readonly GscApiRow[], queryGrainedImpressions: number): {
+  date: string;
+  row: Row;
+};
 interface RowAccumulator {
   /**
    * Push a batch of GSC API rows into the accumulator. Returns `false` if
@@ -93,4 +113,4 @@ interface RowAccumulatorOptions extends IngestOptions {
   trackDateBoundary?: boolean;
 }
 declare function createRowAccumulator(options?: RowAccumulatorOptions): RowAccumulator;
-export { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, TABLE_DIMS, createRowAccumulator, toPath, toSumPosition, transformGscRow };
+export { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, TABLE_DIMS, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow };

package/dist/ingest.mjs

@@ -1,9 +1,9 @@
 const TABLE_DIMS = {
 	pages: ["page", "date"],
-	keywords: ["query", "date"],
+	queries: ["query", "date"],
 	countries: ["country", "date"],
-	devices: ["device", "date"],
-	page_keywords: [
+	dates: ["date"],
+	page_queries: [
 		"page",
 		"query",
 		"date"
@@ -40,7 +40,7 @@ function transformGscRow(table, apiRow, options = {}) {
 			}
 		};
 	}
-	if (table === "keywords") {
+	if (table === "queries") {
 		const query = String(keys[0] ?? "");
 		const date = String(keys[1] ?? "");
 		return {
@@ -68,19 +68,6 @@ function transformGscRow(table, apiRow, options = {}) {
 			}
 		};
 	}
-	if (table === "devices") {
-		const date = String(keys[1] ?? "");
-		return {
-			date,
-			row: {
-				device: String(keys[0] ?? ""),
-				date,
-				clicks,
-				impressions,
-				sum_position
-			}
-		};
-	}
 	if (table === "hourly_pages") {
 		const hour = String(keys[0] ?? "");
 		const date = hour.slice(0, 10);
@@ -109,6 +96,7 @@ function transformGscRow(table, apiRow, options = {}) {
 			}
 		};
 	}
+	if (table === "dates") throw new Error("`dates` rows must be built via assembleDatesRow, not transformGscRow");
 	const query = String(keys[1] ?? "");
 	const date = String(keys[2] ?? "");
 	const query_canonical = options.normalizeQuery?.(query) ?? null;
@@ -125,6 +113,43 @@ function transformGscRow(table, apiRow, options = {}) {
 		}
 	};
 }
+const DEVICE_SUFFIX = {
+	DESKTOP: "desktop",
+	MOBILE: "mobile",
+	TABLET: "tablet"
+};
+function assembleDatesRow(date, totalsRow, deviceRows, queryGrainedImpressions) {
+	const clicks = totalsRow.clicks || 0;
+	const impressions = totalsRow.impressions || 0;
+	const row = {
+		date,
+		clicks,
+		impressions,
+		sum_position: toSumPosition(totalsRow.position || 0, impressions),
+		anonymized_impressions_pct: impressions > 0 ? Math.min(1, Math.max(0, 1 - queryGrainedImpressions / impressions)) : 0,
+		clicks_desktop: 0,
+		clicks_mobile: 0,
+		clicks_tablet: 0,
+		impressions_desktop: 0,
+		impressions_mobile: 0,
+		impressions_tablet: 0,
+		sum_position_desktop: 0,
+		sum_position_mobile: 0,
+		sum_position_tablet: 0
+	};
+	for (const dr of deviceRows) {
+		const suffix = DEVICE_SUFFIX[String(dr.keys?.[1] ?? dr.keys?.[0] ?? "").toUpperCase()];
+		if (!suffix) continue;
+		const dImpr = dr.impressions || 0;
+		row[`clicks_${suffix}`] = dr.clicks || 0;
+		row[`impressions_${suffix}`] = dImpr;
+		row[`sum_position_${suffix}`] = toSumPosition(dr.position || 0, dImpr);
+	}
+	return {
+		date,
+		row
+	};
+}
 const DEFAULT_MAX_ROWS = 5e5;
 function createRowAccumulator(options = {}) {
 	const maxRows = options.maxRows ?? DEFAULT_MAX_ROWS;
@@ -200,4 +225,4 @@ function createRowAccumulator(options = {}) {
 		}
 	};
 }
-export { TABLE_DIMS, createRowAccumulator, toPath, toSumPosition, transformGscRow };
+export { TABLE_DIMS, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow };

package/dist/rollups.mjs

@@ -269,10 +269,10 @@ const dailyTotalsRollup = {
         ORDER BY date
       `
 		});
-		const keywordRows = await runWindowed({
+		const queryRows = await runWindowed({
 			engine,
 			ctx,
-			table: "keywords",
+			table: "queries",
 			...searchType !== void 0 ? { searchType } : {},
 			sqlFor: (w) => `
         SELECT
@@ -297,14 +297,14 @@ const dailyTotalsRollup = {
 			cur.sum_position += Number(r.sum_position);
 			pagesByDate.set(date, cur);
 		}
-		const keywordImpressionsByDate = /* @__PURE__ */ new Map();
-		for (const r of keywordRows) {
+		const queryImpressionsByDate = /* @__PURE__ */ new Map();
+		for (const r of queryRows) {
 			const date = String(r.date);
-			keywordImpressionsByDate.set(date, (keywordImpressionsByDate.get(date) ?? BigInt(0)) + BigInt(r.impressions));
+			queryImpressionsByDate.set(date, (queryImpressionsByDate.get(date) ?? BigInt(0)) + BigInt(r.impressions));
 		}
 		return Array.from(pagesByDate.values()).sort((a, b) => a.date < b.date ? -1 : 1).map((r) => {
 			const totalImpressions = BigInt(r.impressions);
-			const queryImpressions = keywordImpressionsByDate.get(String(r.date)) ?? BigInt(0);
+			const queryImpressions = queryImpressionsByDate.get(String(r.date)) ?? BigInt(0);
 			const anonymized = totalImpressions === BigInt(0) ? 0 : 1 - Number(queryImpressions) / Number(totalImpressions);
 			return {
 				date: r.date,
@@ -439,15 +439,15 @@ const topKeywords28dRollup = {
 		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
 		const partitions = partitionsInRange(await engine.listPartitions({
 			ctx,
-			table: "keywords",
+			table: "queries",
 			...searchType !== void 0 ? { searchType } : {}
 		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
-			table: "keywords",
+			table: "queries",
 			fileSets: { FILES: {
-				table: "keywords",
+				table: "queries",
 				partitions
 			} },
 			...searchType !== void 0 ? { searchType } : {},
@@ -502,15 +502,15 @@ const topKeywords28dParquetRollup = {
 		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
 		const partitions = partitionsInRange(await engine.listPartitions({
 			ctx,
-			table: "keywords",
+			table: "queries",
 			...searchType !== void 0 ? { searchType } : {}
 		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
 		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
-			table: "keywords",
+			table: "queries",
 			fileSets: { FILES: {
-				table: "keywords",
+				table: "queries",
 				partitions
 			} },
 			...searchType !== void 0 ? { searchType } : {},

package/dist/schema.d.mts

@@ -1,2 +1,2 @@
-import { _ as hourly_pages, a as allTables, b as pages, c as dimensionToColumn, d as schemaFor, f as DrizzleSchema, g as drizzleSchema, h as devices, i as TableSchema, l as inferTable, m as countries, n as ColumnType, o as currentSchemaVersion, p as TABLE_METADATA, r as SCHEMAS, s as dedupeByNaturalKey, t as ColumnDef, u as naturalKeyColumns, v as keywords, x as search_appearance, y as page_keywords } from "./_chunks/schema.mjs";
-export { type ColumnDef, type ColumnType, type DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema, allTables, countries, currentSchemaVersion, dedupeByNaturalKey, devices, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, keywords, naturalKeyColumns, page_keywords, pages, schemaFor, search_appearance };
+import { _ as hourly_pages, a as allTables, b as queries, c as dimensionToColumn, d as schemaFor, f as DrizzleSchema, g as drizzleSchema, h as dates, i as TableSchema, l as inferTable, m as countries, n as ColumnType, o as currentSchemaVersion, p as TABLE_METADATA, r as SCHEMAS, s as dedupeByNaturalKey, t as ColumnDef, u as naturalKeyColumns, v as page_queries, x as search_appearance, y as pages } from "./_chunks/schema.mjs";
+export { type ColumnDef, type ColumnType, type DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance };

package/dist/schema.mjs

@@ -1,2 +1,2 @@
-import { _ as search_appearance, a as dimensionToColumn, c as schemaFor, d as devices, f as drizzleSchema, g as pages, h as page_keywords, i as dedupeByNaturalKey, l as TABLE_METADATA, m as keywords, n as allTables, o as inferTable, p as hourly_pages, r as currentSchemaVersion, s as naturalKeyColumns, t as SCHEMAS, u as countries } from "./_chunks/schema.mjs";
-export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dedupeByNaturalKey, devices, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, keywords, naturalKeyColumns, page_keywords, pages, schemaFor, search_appearance };
+import { _ as search_appearance, a as dimensionToColumn, c as schemaFor, d as dates, f as drizzleSchema, g as queries, h as pages, i as dedupeByNaturalKey, l as TABLE_METADATA, m as page_queries, n as allTables, o as inferTable, p as hourly_pages, r as currentSchemaVersion, s as naturalKeyColumns, t as SCHEMAS, u as countries } from "./_chunks/schema.mjs";
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance };

package/dist/sink-node.d.mts

@@ -0,0 +1,31 @@
+import { n as LocalIcebergSinkOptions, r as Sink } from "./_chunks/sink.mjs";
+/** S3-compatible credentials for the warehouse (POC: MinIO). */
+interface LocalIcebergS3Config {
+  /** S3 endpoint host (POC MinIO: `localhost:9100`). */
+  endpoint: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  region?: string;
+}
+/** Full `LocalIcebergSink` options — extends the frozen contract options. */
+interface LocalIcebergSinkFullOptions extends LocalIcebergSinkOptions {
+  /** S3 credentials for the warehouse. Defaults to the POC MinIO creds. */
+  s3?: LocalIcebergS3Config;
+  /** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
+  python?: string;
+  /** Override the writer-script path. Defaults to `scripts/iceberg-writer.py`. */
+  writerScript?: string;
+}
+interface LocalIcebergSink extends Sink {
+  /** The catalog namespace the 5 tables live under. */
+  readonly namespace: string;
+}
+/**
+ * Create a `LocalIcebergSink` pointed at a local Iceberg REST catalog.
+ *
+ * Requires the POC docker stack (`poc/iceberg/docker-compose.iceberg.yml`)
+ * running and a Python env with `pyiceberg` + `pyarrow` available. Tests that
+ * use this sink must skip when the stack is unreachable.
+ */
+declare function createLocalIcebergSink(options: LocalIcebergSinkFullOptions): LocalIcebergSink;
+export { type LocalIcebergS3Config, type LocalIcebergSink, type LocalIcebergSinkFullOptions, createLocalIcebergSink };

package/dist/sink-node.mjs

@@ -0,0 +1,76 @@
+import { i as ICEBERG_SCHEMAS } from "./_chunks/iceberg-schema.mjs";
+import { execFile } from "node:child_process";
+import { dirname, join } from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+const POC_S3 = {
+	endpoint: "localhost:9100",
+	accessKeyId: "poc",
+	secretAccessKey: "pocpocpoc",
+	region: "us-east-1"
+};
+function resolveWriterScript(override) {
+	if (override) return override;
+	return join(dirname(fileURLToPath(import.meta.url)), "..", "..", "scripts", "iceberg-writer.py");
+}
+function runWriter(python, script, job) {
+	return new Promise((resolve, reject) => {
+		execFile(python, [script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
+			let parsed;
+			if (stdout.trim()) try {
+				parsed = JSON.parse(stdout);
+			} catch {}
+			if (parsed?.error) {
+				reject(/* @__PURE__ */ new Error(`LocalIcebergSink writer failed: ${parsed.error}`));
+				return;
+			}
+			if (err) {
+				reject(/* @__PURE__ */ new Error(`LocalIcebergSink writer process failed (${err.message})${stderr ? `: ${stderr}` : ""}`));
+				return;
+			}
+			if (!parsed) {
+				reject(/* @__PURE__ */ new Error(`LocalIcebergSink writer produced no parseable output: ${stdout || stderr}`));
+				return;
+			}
+			resolve(parsed);
+		}).stdin?.end(JSON.stringify(job));
+	});
+}
+function createLocalIcebergSink(options) {
+	const s3 = options.s3 ?? POC_S3;
+	const python = options.python ?? process.env.GSCDUMP_ICEBERG_PYTHON ?? "python3";
+	const script = resolveWriterScript(options.writerScript);
+	function buildJob(op, slice, rows) {
+		return {
+			op,
+			catalogUri: options.catalogUri,
+			namespace: options.namespace,
+			warehouse: options.warehouse,
+			s3,
+			table: slice.table,
+			spec: ICEBERG_SCHEMAS[slice.table],
+			siteId: slice.ctx.siteId ?? "",
+			searchType: slice.searchType,
+			date: slice.date,
+			rows
+		};
+	}
+	const touched = /* @__PURE__ */ new Set();
+	return {
+		namespace: options.namespace,
+		capabilities: { appendOnly: true },
+		async emit(slice, rows) {
+			if (rows.length === 0) return { rowCount: 0 };
+			const res = await runWriter(python, script, buildJob("emit", slice, rows));
+			touched.add(slice.table);
+			return { rowCount: res.rowCount ?? 0 };
+		},
+		async close() {
+			return {
+				flushed: [...touched],
+				failed: []
+			};
+		}
+	};
+}
+export { createLocalIcebergSink };