@gscdump/engine 0.21.3 → 0.22.4

package/dist/_chunks/schema.mjs

@@ -51,6 +51,27 @@ const search_appearance = pgTable("search_appearance", {
 	date: dateCol(),
 	...metricCols()
 });
+const search_appearance_pages = pgTable("search_appearance_pages", {
+	searchAppearance: varchar("searchAppearance").notNull(),
+	url: varchar("url").notNull(),
+	date: dateCol(),
+	...metricCols()
+});
+const search_appearance_queries = pgTable("search_appearance_queries", {
+	searchAppearance: varchar("searchAppearance").notNull(),
+	query: varchar("query").notNull(),
+	query_canonical: varchar("query_canonical"),
+	date: dateCol(),
+	...metricCols()
+});
+const search_appearance_page_queries = pgTable("search_appearance_page_queries", {
+	searchAppearance: varchar("searchAppearance").notNull(),
+	url: varchar("url").notNull(),
+	query: varchar("query").notNull(),
+	query_canonical: varchar("query_canonical"),
+	date: dateCol(),
+	...metricCols()
+});
 const hourly_pages = pgTable("hourly_pages", {
 	url: varchar("url").notNull(),
 	hour: varchar("hour").notNull(),
@@ -64,6 +85,9 @@ const drizzleSchema = {
 	page_queries,
 	dates,
 	search_appearance,
+	search_appearance_pages,
+	search_appearance_queries,
+	search_appearance_page_queries,
 	hourly_pages
 };
 const TABLE_METADATA = {
@@ -105,6 +129,47 @@ const TABLE_METADATA = {
 		clusterKey: ["searchAppearance", "date"],
 		version: 1
 	},
+	search_appearance_pages: {
+		sortKey: [
+			"date",
+			"searchAppearance",
+			"url"
+		],
+		clusterKey: [
+			"searchAppearance",
+			"url",
+			"date"
+		],
+		version: 1
+	},
+	search_appearance_queries: {
+		sortKey: [
+			"date",
+			"searchAppearance",
+			"query"
+		],
+		clusterKey: [
+			"searchAppearance",
+			"query",
+			"date"
+		],
+		version: 1
+	},
+	search_appearance_page_queries: {
+		sortKey: [
+			"date",
+			"searchAppearance",
+			"url",
+			"query"
+		],
+		clusterKey: [
+			"searchAppearance",
+			"url",
+			"query",
+			"date"
+		],
+		version: 1
+	},
 	hourly_pages: {
 		sortKey: [
 			"date",
@@ -149,6 +214,9 @@ const METRIC_TABLES = [
 	"page_queries",
 	"dates",
 	"search_appearance",
+	"search_appearance_pages",
+	"search_appearance_queries",
+	"search_appearance_page_queries",
 	"hourly_pages"
 ];
 const SCHEMAS = Object.fromEntries(METRIC_TABLES.map((t) => [t, tableSchemaFrom(t)]));
@@ -165,12 +233,12 @@ function inferTable(dimensions) {
 	const dims = new Set(dimensions);
 	const hasPage = dims.has("page");
 	const hasQuery = dims.has("query");
+	if (dims.has("searchAppearance")) return hasPage && hasQuery ? "search_appearance_page_queries" : hasPage ? "search_appearance_pages" : hasQuery ? "search_appearance_queries" : "search_appearance";
 	if (hasPage && hasQuery) return "page_queries";
 	if (hasQuery) return "queries";
 	if (hasPage) return "pages";
 	if (dims.has("country")) return "countries";
 	if (dims.has("device")) return "dates";
-	if (dims.has("searchAppearance")) return "search_appearance";
 	return "dates";
 }
 function naturalKeyColumns(table) {
@@ -192,4 +260,4 @@ function dimensionToColumn(dim, _table) {
 	if (dim === "queryCanonical") return "query_canonical";
 	return dim;
 }
-export { search_appearance as _, dimensionToColumn as a, schemaFor as c, dates as d, drizzleSchema as f, queries as g, pages as h, dedupeByNaturalKey as i, TABLE_METADATA as l, page_queries as m, allTables as n, inferTable as o, hourly_pages as p, currentSchemaVersion as r, naturalKeyColumns as s, SCHEMAS as t, countries as u };
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/_chunks/sink.d.mts

@@ -1,9 +1,9 @@
-import { C as Row$1, N as TenantCtx$1, T as SearchType } from "./storage.mjs";
+import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { icebergAppend, restCatalogConnect, s3SignedResolver } from "icebird";
 import { TableName } from "@gscdump/contracts";
-/** The 5 fact tables that exist as global Iceberg tables. */
-type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates'>;
-/** The 5 Iceberg table names, in canonical order. */
+/** The 6 fact tables that exist as global Iceberg tables. */
+type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;
+/** The 6 Iceberg table names, in canonical order. */
 declare const ICEBERG_TABLES: readonly IcebergTableName[];
 /**
  * Iceberg-native column type. Superset-mapped from the engine `ColumnType`;
@@ -216,6 +216,43 @@ declare function ensureIcebergNamespace(conn: IcebergConnection): Promise<void>;
 declare function createIcebergTables(conn: IcebergConnection, tables?: readonly IcebergTableName[]): Promise<IcebergTableOpResult[]>;
 /** List the table names currently in the catalog namespace. */
 declare function listIcebergTables(conn: IcebergConnection): Promise<string[]>;
+/** A data file in the current snapshot's manifest, scoped to one partition. */
+interface IcebergListedDataFile {
+  /** Raw Iceberg `data_file.file_path` (e.g. `s3://gscdump-analytics/.../x.parquet`). */
+  filePath: string;
+  /** Object key relative to the warehouse bucket (the part after `s3://<bucket>/`). */
+  objectKey: string;
+  bytes: number;
+  rowCount: number;
+}
+interface ListIcebergDataFilesOptions {
+  table: IcebergTableName;
+  /** Partition identity column. */
+  siteId: string;
+  /** Partition identity column. */
+  searchType: string;
+  /**
+   * Inclusive date range. Every month touched by `[start, end]` is scanned;
+   * `month(date)` is the third partition transform.
+   */
+  range: {
+    start: string;
+    end: string;
+  };
+}
+/**
+ * List the parquet data files in the current snapshot of `table`, filtered
+ * to a single partition slice `(siteId, searchType, month(date) ∈ range)`.
+ *
+ * Cost: 1 REST `loadTable` + N manifest fetches (typically 1–10 small Avro
+ * files). Iceberg returns the manifest list embedded in `metadata`, so a
+ * cached `metadata` would let callers skip the REST call entirely.
+ *
+ * Skips deleted entries (status=2) and non-data file types (delete files).
+ * Returns object keys + bytes + rowCount so the caller can build presigned
+ * URLs without re-walking the catalog.
+ */
+declare function listIcebergDataFiles(conn: IcebergConnection, opts: ListIcebergDataFilesOptions): Promise<IcebergListedDataFile[]>;
 /**
  * Drop tables from the catalog namespace, purging their data objects.
  * Defaults to every table currently in the namespace — used to clear the
@@ -245,13 +282,15 @@ interface SinkWriteResult {
   bytes?: number;
 }
 /**
- * Static description of a sink. All sinks are append-only under the v5
- * stability-cutoff model; `appendOnly` is therefore always `true` and kept
- * only as an explicit, self-documenting marker.
+ * Static description of a sink. Production ingestion is append-only under the
+ * v5 stability-cutoff model. Test and revision-path adapters may expose an
+ * overwrite capability explicitly.
  */
 interface SinkCapabilities {
-  /** Always `true` — re-emitting a slice accumulates duplicate rows. */
-  appendOnly: true;
+  /** When true, re-emitting a slice accumulates duplicate rows. */
+  appendOnly: boolean;
+  /** Whether the sink exposes partition overwrite semantics. */
+  canOverwrite?: boolean;
 }
 /**
  * Outcome of `Sink.close()` — which tables' buffered rows reached durable
@@ -326,4 +365,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { ICEBERG_PARTITION_SPEC as A, icebergAppendRetrying as C, listIcebergTables as D, isCommitRateLimited as E, IcebergPartitionField as F, IcebergPartitionTransform as I, IcebergTableName as L, ICEBERG_TABLES as M, IcebergColumn as N, ICEBERG_FIELD_ID_BASE as O, IcebergColumnType as P, IcebergTableSpec as R, ensureIcebergNamespace as S, icebergSchemaFor as T, IcebergSchemaField as _, SinkCloseResult as a, createIcebergTables as b, SinkWriteResult as c, IcebergConnection as d, IcebergPartitionSpec as f, IcebergSchema as g, IcebergS3Config as h, SinkCapabilities as i, ICEBERG_SCHEMAS as j, ICEBERG_PARTITION_COLUMNS as k, CommitRetryOptions as l, IcebergPrimitiveType as m, LocalIcebergSinkOptions as n, SinkOptions as o, IcebergPartitionSpecField as p, Sink as r, SinkSlice as s, IcebergAppendSinkOptions as t, IcebergCatalogConfig as u, IcebergTableOpResult as v, icebergPartitionSpecFor as w, dropIcebergTables as x, connectIcebergCatalog as y, icebergTableSpec as z };
+export { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, listIcebergDataFiles, listIcebergTables };

package/dist/_chunks/snapshot.d.mts

@@ -11,4 +11,4 @@ interface SnapshotIndex {
   hot: boolean;
   hotDays: number;
 }
-export { SnapshotIndex as t };
+export { SnapshotIndex };

package/dist/_chunks/storage.d.mts

@@ -550,4 +550,4 @@ declare function dayPartition(date: string): string;
  */
 declare function hourPartition(date: string): string;
 declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
-export { SyncStateKind as A, hourPartition as B, Row$1 as C, SyncState as D, StorageEngine as E, WatermarkFilter as F, RAW_DAILY_COMPACT_THRESHOLD as G, inferSearchType as H, WatermarkScope as I, enumeratePartitions as J, countRawDailies as K, WriteCtx as L, TableName$1 as M, TenantCtx$1 as N, SyncStateDetail as O, Watermark as P, WriteResult as R, QueryResult as S, SearchType$1 as T, objectKey as U, inferLegacyTier as V, CompactionThresholds as W, splitOverlappingTiers as Y, PurgeUrlsResult as _, EngineOptions as a, QueryExecuteResult as b, Grain$1 as c, ManifestEntry as d, ManifestPurgeResult as f, PurgeResult as g, PurgeFilter as h, DataSource as i, SyncStateScope as j, SyncStateFilter as k, ListLiveFilter as l, ParquetCodec as m, CompactionTier as n, FileSetRef as o, ManifestStore as p, dedupeOverlappingTiers as q, DEFAULT_SEARCH_TYPE as r, GcCtx as s, CodecCtx as t, LockScope as u, QueryCtx as v, RunSQLOptions as w, QueryExecutor as x, QueryExecuteOptions as y, dayPartition as z };
+export { CodecCtx, CompactionThresholds, CompactionTier, DEFAULT_SEARCH_TYPE, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, RAW_DAILY_COMPACT_THRESHOLD, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, countRawDailies, dayPartition, dedupeOverlappingTiers, enumeratePartitions, hourPartition, inferLegacyTier, inferSearchType, objectKey, splitOverlappingTiers };

package/dist/_chunks/storage.mjs

@@ -39,4 +39,4 @@ function objectKey(ctx, table, partition, version, searchType) {
 function tenantPrefix(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/` : `u_${ctx.userId}/`;
 }
-export { inferSearchType as a, objectKey as c, tenantPrefix as d, weekPartition as f, inferLegacyTier as i, quarterOfMonth as l, dayPartition as n, mondayOfWeek as o, hourPartition as r, monthPartition as s, DEFAULT_SEARCH_TYPE as t, quarterPartition as u };
+export { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, tenantPrefix, weekPartition };

package/dist/_chunks/types.d.mts

@@ -51,4 +51,4 @@ interface ExtraQuery {
   sql: string;
   params: unknown[];
 }
-export { ResolvedSQLOptimized as a, ResolvedSQL as i, ExtraQuery as n, ResolverAdapter as o, ResolvedComparisonSQL as r, ResolverOptions as s, ComparisonFilter as t };
+export { ComparisonFilter, ExtraQuery, ResolvedComparisonSQL, ResolvedSQL, ResolvedSQLOptimized, ResolverAdapter, ResolverOptions };

package/dist/adapters/duckdb-node.d.mts

@@ -1,4 +1,4 @@
-import { n as DuckDBHandle } from "../_chunks/duckdb.mjs";
+import { DuckDBHandle } from "../_chunks/duckdb.mjs";
 interface NodeDuckDBOptions {
   verbose?: boolean;
 }

package/dist/adapters/filesystem.d.mts

@@ -1,4 +1,4 @@
-import { i as DataSource, p as ManifestStore } from "../_chunks/storage.mjs";
+import { DataSource, ManifestStore } from "../_chunks/storage.mjs";
 interface FilesystemDataSourceOptions {
   rootDir: string;
 }

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { a as inferSearchType, i as inferLegacyTier } from "../_chunks/storage.mjs";
+import { inferLegacyTier, inferSearchType } from "../_chunks/storage.mjs";
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";

package/dist/adapters/hyparquet.d.mts

@@ -1,5 +1,5 @@
-import { C as Row, M as TableName, i as DataSource, m as ParquetCodec, t as CodecCtx } from "../_chunks/storage.mjs";
-import { t as ColumnDef } from "../_chunks/schema.mjs";
+import { CodecCtx, DataSource, ParquetCodec, Row, TableName } from "../_chunks/storage.mjs";
+import { ColumnDef } from "../_chunks/schema.mjs";
 import { ParquetQueryFilter } from "hyparquet";
 declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
 interface EncodeFlexOptions {

package/dist/adapters/hyparquet.mjs

@@ -1,4 +1,4 @@
-import { i as dedupeByNaturalKey, l as TABLE_METADATA, t as SCHEMAS } from "../_chunks/schema.mjs";
+import { SCHEMAS, TABLE_METADATA, dedupeByNaturalKey } from "../_chunks/schema.mjs";
 import { parquetReadObjects } from "hyparquet";
 import { parquetWriteBuffer } from "hyparquet-writer";
 const ROW_GROUP_SIZE = 25e3;

package/dist/adapters/node.d.mts

@@ -1,6 +1,6 @@
-import { E as StorageEngine, i as DataSource } from "../_chunks/storage.mjs";
+import { DataSource, StorageEngine } from "../_chunks/storage.mjs";
 import { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
-import { t as SnapshotIndex } from "../_chunks/snapshot.mjs";
+import { SnapshotIndex } from "../_chunks/snapshot.mjs";
 import { Row, TableName } from "@gscdump/contracts";
 import { SearchType } from "gscdump/query";
 interface NodeHarnessOptions {

package/dist/adapters/node.mjs

@@ -1,4 +1,4 @@
-import { a as createDuckDBCodec, n as createStorageEngine, o as createDuckDBExecutor } from "../_chunks/engine.mjs";
+import { createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "../_chunks/engine.mjs";
 import { createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { createFilesystemDataSource, createFilesystemManifestStore } from "./filesystem.mjs";
 import path from "node:path";

package/dist/adapters/r2-manifest.d.mts

@@ -1,4 +1,4 @@
-import { M as TableName, p as ManifestStore } from "../_chunks/storage.mjs";
+import { ManifestStore, TableName } from "../_chunks/storage.mjs";
 interface R2ObjectMetadata {
   etag: string;
 }

package/dist/adapters/r2-manifest.mjs

@@ -1,4 +1,4 @@
-import { a as inferSearchType, i as inferLegacyTier } from "../_chunks/storage.mjs";
+import { inferLegacyTier, inferSearchType } from "../_chunks/storage.mjs";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 function defaultSnapshotId() {
 	return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;

package/dist/adapters/r2.d.mts

@@ -1,4 +1,4 @@
-import { i as DataSource } from "../_chunks/storage.mjs";
+import { DataSource } from "../_chunks/storage.mjs";
 interface R2GetOptions {
   range?: {
     offset: number;

package/dist/analysis-types.d.mts

@@ -1,2 +1,2 @@
-import { i as num, n as AnalysisResult, r as AnalysisTool, t as AnalysisParams } from "./_chunks/analysis-types.mjs";
+import { AnalysisParams, AnalysisResult, AnalysisTool, num } from "./_chunks/analysis-types.mjs";
 export { AnalysisParams, AnalysisResult, AnalysisTool, num };

package/dist/analyzer/index.d.mts

@@ -1,5 +1,5 @@
-import { n as AnalysisResult, t as AnalysisParams } from "../_chunks/analysis-types.mjs";
-import { _ as SqlExtraQuery, a as DefineAnalyzerOptions, b as requireAdapter, c as Reducer, d as Analyzer, f as BuildContext, g as RowQueriesPlan, h as RequiredCapability, i as createAnalyzerRegistry, l as SqlPlanSpec, m as ReduceContext, n as AnalyzerRegistryInit, o as DefinedAnalyzer, p as Plan, r as AnalyzerVariants, s as ReduceCtx, t as AnalyzerRegistry, u as defineAnalyzer, v as SqlPlan, x as AnalysisQuerySource, y as TypedRowQuery } from "../_chunks/registry.mjs";
+import { AnalysisParams, AnalysisResult } from "../_chunks/analysis-types.mjs";
+import { AnalysisQuerySource, Analyzer, AnalyzerRegistry, AnalyzerRegistryInit, AnalyzerVariants, BuildContext, DefineAnalyzerOptions, DefinedAnalyzer, Plan, ReduceContext, ReduceCtx, Reducer, RequiredCapability, RowQueriesPlan, SqlExtraQuery, SqlPlan, SqlPlanSpec, TypedRowQuery, createAnalyzerRegistry, defineAnalyzer, requireAdapter } from "../_chunks/registry.mjs";
 declare class AnalyzerCapabilityError extends Error {
   readonly tool: string;
   readonly missing: readonly RequiredCapability[];

package/dist/analyzer/index.mjs

@@ -1,4 +1,4 @@
-import { n as runAnalyzerFromSource, t as AnalyzerCapabilityError } from "../_chunks/dispatch.mjs";
+import { AnalyzerCapabilityError, runAnalyzerFromSource } from "../_chunks/dispatch.mjs";
 const DEFAULT_SQL_REQUIRES = ["executeSql", "fileSets"];
 function defineAnalyzer(opts) {
 	const { id, reduce, reduceSql, reduceRows, buildSql, buildRows, sqlRequires = DEFAULT_SQL_REQUIRES, rowsRequires = [] } = opts;

package/dist/contracts.d.mts

@@ -1,2 +1,2 @@
-import { A as SyncStateKind, C as Row, D as SyncState, E as StorageEngine, F as WatermarkFilter, I as WatermarkScope, L as WriteCtx, M as TableName, N as TenantCtx, O as SyncStateDetail, P as Watermark, R as WriteResult, S as QueryResult, T as SearchType, a as EngineOptions, b as QueryExecuteResult, d as ManifestEntry, i as DataSource, j as SyncStateScope, k as SyncStateFilter, l as ListLiveFilter, m as ParquetCodec, n as CompactionTier, o as FileSetRef, p as ManifestStore, s as GcCtx, t as CodecCtx, u as LockScope, v as QueryCtx, w as RunSQLOptions, x as QueryExecutor, y as QueryExecuteOptions } from "./_chunks/storage.mjs";
+import { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, ListLiveFilter, LockScope, ManifestEntry, ManifestStore, ParquetCodec, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult } from "./_chunks/storage.mjs";
 export type { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, ListLiveFilter, LockScope, ManifestEntry, ManifestStore, ParquetCodec, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult };

package/dist/entities.d.mts

@@ -1,4 +1,4 @@
-import { i as DataSource } from "./_chunks/storage.mjs";
+import { DataSource } from "./_chunks/storage.mjs";
 import { ScheduleState } from "./schedule.mjs";
 import { TenantCtx } from "@gscdump/contracts";
 /**

package/dist/index.d.mts

@@ -1,10 +1,10 @@
-import { A as SyncStateKind, B as hourPartition, C as Row, D as SyncState, E as StorageEngine, F as WatermarkFilter, G as RAW_DAILY_COMPACT_THRESHOLD, H as inferSearchType, I as WatermarkScope, J as enumeratePartitions, K as countRawDailies, L as WriteCtx, M as TableName, N as TenantCtx, O as SyncStateDetail, P as Watermark, R as WriteResult, S as QueryResult, T as SearchType, U as objectKey, V as inferLegacyTier, W as CompactionThresholds, Y as splitOverlappingTiers, _ as PurgeUrlsResult, a as EngineOptions, b as QueryExecuteResult, c as Grain, d as ManifestEntry, f as ManifestPurgeResult, g as PurgeResult, h as PurgeFilter, i as DataSource, j as SyncStateScope, k as SyncStateFilter, l as ListLiveFilter, m as ParquetCodec, n as CompactionTier, o as FileSetRef, p as ManifestStore, q as dedupeOverlappingTiers, r as DEFAULT_SEARCH_TYPE, s as GcCtx, t as CodecCtx, u as LockScope, v as QueryCtx, w as RunSQLOptions, x as QueryExecutor, y as QueryExecuteOptions, z as dayPartition } from "./_chunks/storage.mjs";
-import { a as createDuckDBExecutor, i as createDuckDBCodec, n as DuckDBHandle, r as canonicalEmptyParquetSchema, t as DuckDBFactory } from "./_chunks/duckdb.mjs";
-import { _ as hourly_pages, a as allTables, b as queries, c as dimensionToColumn, f as DrizzleSchema, g as drizzleSchema, h as dates, i as TableSchema, l as inferTable, m as countries, n as ColumnType, o as currentSchemaVersion, p as TABLE_METADATA, r as SCHEMAS, t as ColumnDef, v as page_queries, y as pages } from "./_chunks/schema.mjs";
+import { CodecCtx, CompactionThresholds, CompactionTier, DEFAULT_SEARCH_TYPE, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, RAW_DAILY_COMPACT_THRESHOLD, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, countRawDailies, dayPartition, dedupeOverlappingTiers, enumeratePartitions, hourPartition, inferLegacyTier, inferSearchType, objectKey, splitOverlappingTiers } from "./_chunks/storage.mjs";
+import { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
+import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
 import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
-import { A as ICEBERG_PARTITION_SPEC, C as icebergAppendRetrying, D as listIcebergTables, E as isCommitRateLimited, F as IcebergPartitionField, I as IcebergPartitionTransform, L as IcebergTableName, M as ICEBERG_TABLES, N as IcebergColumn, O as ICEBERG_FIELD_ID_BASE, P as IcebergColumnType, R as IcebergTableSpec, S as ensureIcebergNamespace, T as icebergSchemaFor, _ as IcebergSchemaField, a as SinkCloseResult, b as createIcebergTables, c as SinkWriteResult, d as IcebergConnection, f as IcebergPartitionSpec, g as IcebergSchema, h as IcebergS3Config, i as SinkCapabilities, j as ICEBERG_SCHEMAS, k as ICEBERG_PARTITION_COLUMNS, l as CommitRetryOptions, m as IcebergPrimitiveType, n as LocalIcebergSinkOptions, o as SinkOptions, p as IcebergPartitionSpecField, r as Sink, s as SinkSlice, t as IcebergAppendSinkOptions, u as IcebergCatalogConfig, v as IcebergTableOpResult, w as icebergPartitionSpecFor, x as dropIcebergTables, y as connectIcebergCatalog, z as icebergTableSpec } from "./_chunks/sink.mjs";
+import { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, listIcebergDataFiles, listIcebergTables } from "./_chunks/sink.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
-import { a as substituteNamedFiles, i as resolveParquetSQL, n as ResolvedQuery, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
+import { FILES_PLACEHOLDER, ResolvedQuery, resolveParquetSQL, substituteNamedFiles } from "./_chunks/planner.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
 import { Grain as Grain$1, Row as Row$1, TableName as TableName$1 } from "@gscdump/contracts";
@@ -179,4 +179,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CommitRetryOptions, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LocalIcebergSinkOptions, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CommitRetryOptions, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListIcebergDataFilesOptions, type ListLiveFilter, type LocalIcebergSinkOptions, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergDataFiles, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };

package/dist/index.mjs

@@ -1,15 +1,15 @@
-import { n as coerceRows, t as coerceRow } from "./_chunks/coerce.mjs";
-import { a as dimensionToColumn, d as dates, f as drizzleSchema, g as queries, h as pages, l as TABLE_METADATA, m as page_queries, n as allTables, o as inferTable, p as hourly_pages, r as currentSchemaVersion, t as SCHEMAS, u as countries } from "./_chunks/schema.mjs";
-import { a as inferSearchType, c as objectKey, i as inferLegacyTier, n as dayPartition, r as hourPartition, t as DEFAULT_SEARCH_TYPE } from "./_chunks/storage.mjs";
-import { a as RAW_DAILY_COMPACT_THRESHOLD, c as dedupeOverlappingTiers, i as substituteNamedFiles, l as enumeratePartitions, r as resolveParquetSQL, s as countRawDailies, t as FILES_PLACEHOLDER, u as splitOverlappingTiers } from "./_chunks/parquet-plan.mjs";
+import { coerceRow, coerceRows } from "./_chunks/coerce.mjs";
+import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
+import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./_chunks/storage.mjs";
+import { FILES_PLACEHOLDER, RAW_DAILY_COMPACT_THRESHOLD, countRawDailies, dedupeOverlappingTiers, enumeratePartitions, resolveParquetSQL, splitOverlappingTiers, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
-import { a as createDuckDBCodec, i as canonicalEmptyParquetSchema, n as createStorageEngine, o as createDuckDBExecutor, r as gcOrphansImpl, t as MAX_DAY_BYTES } from "./_chunks/engine.mjs";
-import { a as ICEBERG_TABLES, i as ICEBERG_SCHEMAS, n as ICEBERG_PARTITION_COLUMNS, o as icebergTableSpec, r as ICEBERG_PARTITION_SPEC, t as ICEBERG_FIELD_ID_BASE } from "./_chunks/iceberg-schema.mjs";
+import { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine, gcOrphansImpl } from "./_chunks/engine.mjs";
+import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, icebergTableSpec } from "./_chunks/iceberg-schema.mjs";
 import { assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import "./planner.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, s3SignedResolver } from "icebird";
+import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "icebird";
 const ICEBERG_TYPE_MAP = {
 	STRING: "string",
 	INT: "int",
@@ -109,6 +109,62 @@ async function createIcebergTables(conn, tables = ICEBERG_TABLES) {
 async function listIcebergTables(conn) {
 	return restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name).sort(), () => []);
 }
+function monthsInRange(range) {
+	const [sy, sm] = range.start.split("-").map(Number);
+	const [ey, em] = range.end.split("-").map(Number);
+	const out = [];
+	let y = sy;
+	let m = sm;
+	while (y < ey || y === ey && m <= em) {
+		out.push(`${y}-${String(m).padStart(2, "0")}`);
+		m++;
+		if (m > 12) {
+			m = 1;
+			y++;
+		}
+	}
+	return out;
+}
+function monthsSinceEpoch(ym) {
+	const [y, m] = ym.split("-").map(Number);
+	return (y - 1970) * 12 + (m - 1);
+}
+function stripBucket(filePath) {
+	if (!filePath.startsWith("s3://")) return filePath;
+	const rest = filePath.slice(5);
+	const slash = rest.indexOf("/");
+	return slash >= 0 ? rest.slice(slash + 1) : rest;
+}
+async function listIcebergDataFiles(conn, opts) {
+	const { metadata } = await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table: opts.table
+	});
+	if (metadata["current-snapshot-id"] == null) return [];
+	const wantedMonths = new Set(monthsInRange(opts.range).map(monthsSinceEpoch));
+	const manifests = await icebergManifests({
+		metadata,
+		resolver: conn.resolver
+	});
+	const out = [];
+	for (const m of manifests) for (const entry of m.entries) {
+		if (entry.status === 2) continue;
+		const df = entry.data_file;
+		if (df.content !== 0) continue;
+		const part = df.partition;
+		if (part.site_id !== opts.siteId) continue;
+		if (part.search_type !== opts.searchType) continue;
+		const month = part.date_month;
+		if (typeof month !== "number" || !wantedMonths.has(month)) continue;
+		out.push({
+			filePath: df.file_path,
+			objectKey: stripBucket(df.file_path),
+			bytes: Number(df.file_size_in_bytes),
+			rowCount: Number(df.record_count)
+		});
+	}
+	return out;
+}
 async function dropIcebergTables(conn, tables) {
 	const targets = tables ?? await restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name), () => []);
 	const results = [];
@@ -473,4 +529,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
 const MIN_COUNTRY_IMPRESSIONS = 10;
 const MAX_SITEMAP_URLS_PER_SITE = 5e4;
 const MAX_TRACKED_URLS_PER_SITE = 2e5;
-export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createRowAccumulator, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergDataFiles, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };

package/dist/ingest.d.mts

@@ -1,4 +1,4 @@
-import { C as Row, M as TableName } from "./_chunks/storage.mjs";
+import { Row, TableName } from "./_chunks/storage.mjs";
 /**
  * Canonical GSC API dimension order per table. Consumers hitting the raw
  * `searchanalytics.query` endpoint must request dimensions in this order so
@@ -23,6 +23,10 @@ interface IngestOptions {
    * `page_queries` tables only.
    */
   normalizeQuery?: (query: string) => string | null | undefined;
+  /** Date for one-day `searchAppearance` total queries, whose keys omit date. */
+  date?: string;
+  /** Search appearance filter used for contextual second-step rows. */
+  searchAppearance?: string;
 }
 /**
  * Strip a GSC URL to its pathname. Core analytics stores pages by path so

package/dist/ingest.mjs

@@ -9,6 +9,13 @@ const TABLE_DIMS = {
 		"date"
 	],
 	search_appearance: ["searchAppearance", "date"],
+	search_appearance_pages: ["page", "date"],
+	search_appearance_queries: ["query", "date"],
+	search_appearance_page_queries: [
+		"page",
+		"query",
+		"date"
+	],
 	hourly_pages: ["hour", "page"]
 };
 function toPath(gscUrl) {
@@ -84,7 +91,7 @@ function transformGscRow(table, apiRow, options = {}) {
 		};
 	}
 	if (table === "search_appearance") {
-		const date = String(keys[1] ?? "");
+		const date = String(keys[1] ?? options.date ?? "");
 		return {
 			date,
 			row: {
@@ -96,6 +103,55 @@ function transformGscRow(table, apiRow, options = {}) {
 			}
 		};
 	}
+	if (table === "search_appearance_pages") {
+		const date = String(keys[1] ?? "");
+		return {
+			date,
+			row: {
+				searchAppearance: String(options.searchAppearance ?? ""),
+				url: toPath(String(keys[0] ?? "")),
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
+	if (table === "search_appearance_queries") {
+		const query = String(keys[0] ?? "");
+		const date = String(keys[1] ?? "");
+		const query_canonical = options.normalizeQuery?.(query) ?? null;
+		return {
+			date,
+			row: {
+				searchAppearance: String(options.searchAppearance ?? ""),
+				query,
+				query_canonical,
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
+	if (table === "search_appearance_page_queries") {
+		const query = String(keys[1] ?? "");
+		const date = String(keys[2] ?? "");
+		const query_canonical = options.normalizeQuery?.(query) ?? null;
+		return {
+			date,
+			row: {
+				searchAppearance: String(options.searchAppearance ?? ""),
+				url: toPath(String(keys[0] ?? "")),
+				query,
+				query_canonical,
+				date,
+				clicks,
+				impressions,
+				sum_position
+			}
+		};
+	}
 	if (table === "dates") throw new Error("`dates` rows must be built via assembleDatesRow, not transformGscRow");
 	const query = String(keys[1] ?? "");
 	const date = String(keys[2] ?? "");

package/dist/period/index.d.mts

@@ -1,2 +1,2 @@
-import { a as ResolveWindowOptions, c as comparisonOf, d as padTimeseries, f as periodOf, h as windowToPeriod, i as PadTimeseriesOptions, l as defaultEndDate, m as windowToComparisonPeriod, n as ComparisonMode, o as ResolvedWindow, p as resolveWindow, r as ComparisonPeriod, s as WindowPreset, t as AnalysisPeriod, u as defaultStartDate } from "../_chunks/index.mjs";
+import { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow, windowToComparisonPeriod, windowToPeriod } from "../_chunks/index.mjs";
 export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, defaultStartDate, padTimeseries, periodOf, resolveWindow, windowToComparisonPeriod, windowToPeriod };

package/dist/planner.d.mts

@@ -1,3 +1,3 @@
-import { J as enumeratePartitions } from "./_chunks/storage.mjs";
-import { a as substituteNamedFiles, i as resolveParquetSQL, n as ResolvedQuery, r as compileLogicalQueryPlan, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
+import { enumeratePartitions } from "./_chunks/storage.mjs";
+import { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, resolveParquetSQL, substituteNamedFiles } from "./_chunks/planner.mjs";
 export { FILES_PLACEHOLDER, type ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };

package/dist/planner.mjs

@@ -1,2 +1,2 @@
-import { i as substituteNamedFiles, l as enumeratePartitions, n as compileLogicalQueryPlan, r as resolveParquetSQL, t as FILES_PLACEHOLDER } from "./_chunks/parquet-plan.mjs";
+import { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
 export { FILES_PLACEHOLDER, compileLogicalQueryPlan, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };