@gscdump/engine 0.27.2 → 0.28.1

package/dist/_chunks/libs/ioredis.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/_chunks/libs/lru-cache.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/_chunks/libs/unstorage.d.mts

@@ -0,0 +1,120 @@
+type StorageValue = null | string | number | boolean | object;
+type WatchEvent = "update" | "remove";
+type WatchCallback = (event: WatchEvent, key: string) => any;
+type MaybePromise<T> = T | Promise<T>;
+type MaybeDefined<T> = T extends any ? T : any;
+type Unwatch = () => MaybePromise<void>;
+interface StorageMeta {
+  atime?: Date;
+  mtime?: Date;
+  ttl?: number;
+  [key: string]: StorageValue | Date | undefined;
+}
+type TransactionOptions = Record<string, any>;
+type GetKeysOptions = TransactionOptions & {
+  maxDepth?: number;
+};
+interface DriverFlags {
+  maxDepth?: boolean;
+  ttl?: boolean;
+}
+interface Driver<OptionsT = any, InstanceT = any> {
+  name?: string;
+  flags?: DriverFlags;
+  options?: OptionsT;
+  getInstance?: () => InstanceT;
+  hasItem: (key: string, opts: TransactionOptions) => MaybePromise<boolean>;
+  getItem: (key: string, opts?: TransactionOptions) => MaybePromise<StorageValue>;
+  /** @experimental */
+  getItems?: (items: {
+    key: string;
+    options?: TransactionOptions;
+  }[], commonOptions?: TransactionOptions) => MaybePromise<{
+    key: string;
+    value: StorageValue;
+  }[]>;
+  /** @experimental */
+  getItemRaw?: (key: string, opts: TransactionOptions) => MaybePromise<unknown>;
+  setItem?: (key: string, value: string, opts: TransactionOptions) => MaybePromise<void>;
+  /** @experimental */
+  setItems?: (items: {
+    key: string;
+    value: string;
+    options?: TransactionOptions;
+  }[], commonOptions?: TransactionOptions) => MaybePromise<void>;
+  /** @experimental */
+  setItemRaw?: (key: string, value: any, opts: TransactionOptions) => MaybePromise<void>;
+  removeItem?: (key: string, opts: TransactionOptions) => MaybePromise<void>;
+  getMeta?: (key: string, opts: TransactionOptions) => MaybePromise<StorageMeta | null>;
+  getKeys: (base: string, opts: GetKeysOptions) => MaybePromise<string[]>;
+  clear?: (base: string, opts: TransactionOptions) => MaybePromise<void>;
+  dispose?: () => MaybePromise<void>;
+  watch?: (callback: WatchCallback) => MaybePromise<Unwatch>;
+}
+type StorageDefinition = {
+  items: unknown;
+  [key: string]: unknown;
+};
+type StorageItemMap<T> = T extends StorageDefinition ? T["items"] : T;
+type StorageItemType<T, K> = K extends keyof StorageItemMap<T> ? StorageItemMap<T>[K] : T extends StorageDefinition ? StorageValue : T;
+interface Storage$1<T extends StorageValue = StorageValue> {
+  hasItem<U extends Extract<T, StorageDefinition>, K extends keyof StorageItemMap<U>>(key: K, opts?: TransactionOptions): Promise<boolean>;
+  hasItem(key: string, opts?: TransactionOptions): Promise<boolean>;
+  getItem<U extends Extract<T, StorageDefinition>, K extends string & keyof StorageItemMap<U>>(key: K, ops?: TransactionOptions): Promise<StorageItemType<T, K> | null>;
+  getItem<R = StorageItemType<T, string>>(key: string, opts?: TransactionOptions): Promise<R | null>;
+  /** @experimental */
+  getItems: <U extends T>(items: (string | {
+    key: string;
+    options?: TransactionOptions;
+  })[], commonOptions?: TransactionOptions) => Promise<{
+    key: string;
+    value: U;
+  }[]>;
+  /** @experimental See https://github.com/unjs/unstorage/issues/142 */
+  getItemRaw: <T = any>(key: string, opts?: TransactionOptions) => Promise<MaybeDefined<T> | null>;
+  setItem<U extends Extract<T, StorageDefinition>, K extends keyof StorageItemMap<U>>(key: K, value: StorageItemType<T, K>, opts?: TransactionOptions): Promise<void>;
+  setItem<U extends T>(key: string, value: U, opts?: TransactionOptions): Promise<void>;
+  /** @experimental */
+  setItems: <U extends T>(items: {
+    key: string;
+    value: U;
+    options?: TransactionOptions;
+  }[], commonOptions?: TransactionOptions) => Promise<void>;
+  /** @experimental See https://github.com/unjs/unstorage/issues/142 */
+  setItemRaw: <T = any>(key: string, value: MaybeDefined<T>, opts?: TransactionOptions) => Promise<void>;
+  removeItem<U extends Extract<T, StorageDefinition>, K extends keyof StorageItemMap<U>>(key: K, opts?: (TransactionOptions & {
+    removeMeta?: boolean;
+  }) | boolean): Promise<void>;
+  removeItem(key: string, opts?: (TransactionOptions & {
+    removeMeta?: boolean;
+  }) | boolean): Promise<void>;
+  getMeta: (key: string, opts?: (TransactionOptions & {
+    nativeOnly?: boolean;
+  }) | boolean) => MaybePromise<StorageMeta>;
+  setMeta: (key: string, value: StorageMeta, opts?: TransactionOptions) => Promise<void>;
+  removeMeta: (key: string, opts?: TransactionOptions) => Promise<void>;
+  getKeys: (base?: string, opts?: GetKeysOptions) => Promise<string[]>;
+  clear: (base?: string, opts?: TransactionOptions) => Promise<void>;
+  dispose: () => Promise<void>;
+  watch: (callback: WatchCallback) => Promise<Unwatch>;
+  unwatch: () => Promise<void>;
+  mount: (base: string, driver: Driver) => Storage$1;
+  unmount: (base: string, dispose?: boolean) => Promise<void>;
+  getMount: (key?: string) => {
+    base: string;
+    driver: Driver;
+  };
+  getMounts: (base?: string, options?: {
+    parents?: boolean;
+  }) => {
+    base: string;
+    driver: Driver;
+  }[];
+  keys: Storage$1["getKeys"];
+  get: Storage$1<T>["getItem"];
+  set: Storage$1<T>["setItem"];
+  has: Storage$1<T>["hasItem"];
+  del: Storage$1<T>["removeItem"];
+  remove: Storage$1<T>["removeItem"];
+}
+export { Storage$1 as Storage };

package/dist/_chunks/sink.d.mts

@@ -1,8 +1,20 @@
-import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
+import { QueryProfiler, Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { EngineError } from "./errors.mjs";
+import { Storage } from "./libs/unstorage.mjs";
+import { cachingResolver, icebergAppend, restCatalogConnect } from "./libs/icebird.mjs";
 import { Result } from "gscdump/result";
-import { icebergAppend, restCatalogConnect, s3SignedResolver } from "icebird";
 import { TableName } from "@gscdump/contracts";
+/** Injected catalog cache: an unstorage `Storage` plus an optional defer hook. */
+interface CatalogCache {
+  /** unstorage storage instance — the driver is the caller's choice. */
+  storage: Storage;
+  /**
+   * Optional hook to run a cache WRITE off the response critical path, e.g.
+   * Cloudflare's `ctx.waitUntil`. When omitted the writer awaits the put
+   * inline so it is never cut off when the response returns.
+   */
+  defer?: (write: Promise<unknown>) => void;
+}
 /**
  * S3-compatible credentials for the Iceberg warehouse object store (R2 in prod,
  * MinIO in the POC). The single definition shared by every catalog/writer/sink
@@ -150,8 +162,8 @@ interface IcebergCatalogConfig {
 interface IcebergConnection {
   /** icebird REST catalog context, passed as `{ catalog }` to icebird write fns. */
   catalog: Awaited<ReturnType<typeof restCatalogConnect>>;
-  /** icebird S3 resolver, passed as `{ resolver }` to icebird write fns. */
-  resolver: ReturnType<typeof s3SignedResolver>;
+  /** icebird S3 resolver (caching-wrapped), passed as `{ resolver }` to icebird fns. */
+  resolver: ReturnType<typeof cachingResolver>;
   /** The namespace the fact tables live under. */
   namespace: string;
 }
@@ -168,12 +180,30 @@ declare function icebergSchemaFor(table: IcebergTableName): IcebergSchema;
  * {@link icebergSchemaFor}.
  */
 declare function icebergPartitionSpecFor(table: IcebergTableName): IcebergPartitionSpec;
+/** Options for {@link connectIcebergCatalog}. */
+interface ConnectIcebergOptions {
+  /**
+   * Optional cross-isolate cache (any unstorage driver). When supplied, the
+   * `/v1/config` REST probe is served from cache on a warm catalog, removing
+   * one serial network hop from cold-isolate connects. The bearer token is
+   * NEVER cached — only the warehouse-static routing config (`url`, `prefix`,
+   * `defaults`, `overrides`) is; `requestInit` is rebuilt from `config`.
+   */
+  cache?: CatalogCache;
+  /** Injectable clock for the cache TTL. Defaults to `Date.now`. */
+  clock?: () => number;
+}
 /**
  * Connect to the R2 Data Catalog: a REST catalog context + a signed S3
  * resolver. Runs in Node and in `workerd` — SigV4 is Web Crypto, I/O is
  * `fetch`, no node builtins.
+ *
+ * With a `cache`, the `/v1/config` probe is skipped on a warm catalog and the
+ * context is rebuilt from the cached routing config plus the freshly-derived
+ * bearer `requestInit`. icebird reads only `url`/`prefix`/`requestInit` from
+ * the context downstream, so this is a faithful, secret-free reconstruction.
  */
-declare function connectIcebergCatalog(config: IcebergCatalogConfig): Promise<IcebergConnection>;
+declare function connectIcebergCatalog(config: IcebergCatalogConfig, opts?: ConnectIcebergOptions): Promise<IcebergConnection>;
 /** Tunable retry policy for {@link icebergAppendRetrying}. */
 interface CommitRetryOptions {
   /** Total attempts, including the first. Default 6. */
@@ -266,14 +296,35 @@ interface ListIcebergDataFilesOptions {
     start: string;
     end: string;
   };
+  /**
+   * Optional cross-isolate cache (any unstorage driver). When supplied, the
+   * snapshot pointer is cached short (so a warm catalog skips `loadTable`) and
+   * the resolved file list is cached long, content-addressed by snapshot id
+   * (so it skips the manifest walk). Omit it to read straight from the catalog.
+   */
+  cache?: CatalogCache;
+  /** Injectable clock for the cache TTLs. Defaults to `Date.now`. */
+  clock?: () => number;
+  /**
+   * Optional read-path profiler. Emits `iceberg.snapshot` (snapshot-pointer
+   * load), `iceberg.cache` (resolved-files lookup + hit/miss), and
+   * `iceberg.walk` (manifest fetch + entry scan, with manifest/file counts) —
+   * the catalog cold-start breakdown a hosted reader wants in `Server-Timing`.
+   */
+  profiler?: QueryProfiler;
 }
 /**
- * List the parquet data files in the current snapshot of `table`, filtered
- * to a single partition slice `(siteId, searchType, month(date) ∈ range)`.
+ * List the parquet data files in the current snapshot of `table`, filtered to a
+ * single partition slice `(siteId, searchType, month(date) ∈ range)`.
  *
- * Cost: 1 REST `loadTable` + N manifest fetches (typically 1–10 small Avro
- * files). Iceberg returns the manifest list embedded in `metadata`, so a
- * cached `metadata` would let callers skip the REST call entirely.
+ * The shared `gsc.<table>` tables are multi-tenant, so a naive walk is O(all
+ * tenants). This prunes the manifest LIST by partition summaries before
+ * fetching any manifest's entries (see {@link buildPartitionFilter}), making
+ * the fetch count independent of tenant count, and — when an unstorage `cache`
+ * is supplied — skips the `loadTable` round-trip on a warm snapshot pointer and
+ * the manifest walk entirely on a resolved-files hit. The final entry-level
+ * partition filter is the authoritative correctness check; pruning only avoids
+ * reading manifests that cannot match.
  *
  * Skips deleted entries (status=2) and non-data file types (delete files).
  * Returns object keys + bytes + rowCount so the caller can build presigned
@@ -396,4 +447,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/_chunks/storage.d.mts

@@ -72,9 +72,30 @@ interface WriteCtx extends TenantCtx {
    */
   grain?: Grain;
 }
+/**
+ * A closed profiling span: a named slice of read-path work with its
+ * wall-clock cost and optional dimensional meta (file counts, row counts).
+ * Emitted by an injected {@link QueryProfiler}; see `./profile.ts`.
+ */
+interface QuerySpan {
+  readonly name: string;
+  readonly ms: number;
+  readonly meta?: Readonly<Record<string, string | number | boolean>>;
+}
+/**
+ * Injected read-path profiler. `start(name, meta)` opens a span and returns an
+ * `end` thunk to call when that work finishes (merging completion-only meta).
+ * Absent by default — every emit site optional-chains it, so an unprofiled
+ * query pays nothing. Build one with `createQueryProfiler` / `collectSpans`.
+ */
+interface QueryProfiler {
+  readonly start: (name: string, meta?: Record<string, string | number | boolean>) => (extra?: Record<string, string | number | boolean>) => void;
+}
 interface QueryCtx extends TenantCtx {
   table?: TableName;
   signal?: AbortSignal;
+  /** Optional read-path profiler; forwarded into `runSQL` and the executor. */
+  profiler?: QueryProfiler;
   /**
    * Restrict the query to a single GSC search-type partition (`web`,
    * `discover`, etc.). Undefined preserves the cross-type union for
@@ -390,6 +411,11 @@ interface QueryExecuteOptions {
     bytes?: number;
     reason: string;
   }) => void;
+  /**
+   * Optional profiler. An instrumented executor emits `files.register` and
+   * `query.run` spans through it; an absent profiler is a no-op skip.
+   */
+  profiler?: QueryProfiler;
 }
 interface QueryExecuteResult {
   rows: Row[];
@@ -449,6 +475,12 @@ interface RunSQLOptions {
    * Undefined keeps the legacy cross-type union.
    */
   searchType?: SearchType;
+  /**
+   * Optional read-path profiler. `runSQL` emits `manifest.list` +
+   * `executor.execute` spans and forwards it into the executor for the
+   * finer `files.register` / `query.run` breakdown.
+   */
+  profiler?: QueryProfiler;
 }
 interface StorageEngine {
   writeDay: (ctx: WriteCtx, rows: Row[]) => Promise<void>;
@@ -541,4 +573,4 @@ interface EngineOptions {
   executor: QueryExecutor;
   now?: () => number;
 }
-export { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, dedupeOverlappingTiers, enumeratePartitions, splitOverlappingTiers };
+export { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, dedupeOverlappingTiers, enumeratePartitions, splitOverlappingTiers };

package/dist/iceberg/index.d.mts

@@ -1,4 +1,5 @@
-import { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+import { icebergCreateTable, icebergManifests, restCatalogLoadTable } from "../_chunks/libs/icebird.mjs";
 type IcebergAppendSink = Sink;
 /**
  * Create an `IcebergAppendSink` over the R2 Data Catalog.
@@ -9,4 +10,4 @@ type IcebergAppendSink = Sink;
  * with no rows never touches the network.
  */
 declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
-export { type CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };

package/dist/iceberg/index.mjs

@@ -1,7 +1,72 @@
 import { engineErrors } from "../errors.mjs";
 import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "../_chunks/libs/icebird.mjs";
 import { err, ok } from "gscdump/result";
-import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "icebird";
+async function cacheGet(cache, key, now) {
+	const boxed = await cache.storage.getItem(key).catch(() => null);
+	if (!boxed || typeof boxed.exp !== "number" || boxed.exp <= now) return void 0;
+	return boxed.v;
+}
+function cachePut(cache, key, value, ttlMs, now) {
+	const boxed = {
+		v: value,
+		exp: now + ttlMs
+	};
+	const write = cache.storage.setItem(key, boxed, { ttl: Math.ceil(ttlMs / 1e3) }).catch(() => {});
+	if (cache.defer) {
+		cache.defer(write);
+		return Promise.resolve();
+	}
+	return write;
+}
+const SITE_ID_FIELD_INDEX = ICEBERG_PARTITION_SPEC.findIndex((f) => f.sourceColumn === "site_id" && f.transform === "identity");
+const SEARCH_TYPE_FIELD_INDEX = ICEBERG_PARTITION_SPEC.findIndex((f) => f.sourceColumn === "search_type" && f.transform === "identity");
+const DATE_MONTH_FIELD_INDEX = ICEBERG_PARTITION_SPEC.findIndex((f) => f.transform === "month");
+function toUint8(bytes) {
+	if (bytes == null) return null;
+	return bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
+}
+function decodeString(bytes) {
+	const u = toUint8(bytes);
+	return u == null ? null : new TextDecoder().decode(u);
+}
+function decodeInt(bytes) {
+	const u = toUint8(bytes);
+	if (u == null) return null;
+	return new DataView(u.buffer, u.byteOffset, u.byteLength).getInt32(0, true);
+}
+function buildPartitionFilter(siteId, searchType, wantedMonths) {
+	return (partitions) => {
+		const parts = partitions;
+		if (!parts || parts.length === 0) return true;
+		const siteSummary = parts[SITE_ID_FIELD_INDEX];
+		if (siteSummary && (siteSummary.lower_bound != null || siteSummary.upper_bound != null)) {
+			const lo = decodeString(siteSummary.lower_bound);
+			const hi = decodeString(siteSummary.upper_bound);
+			if (lo != null && hi != null && (siteId < lo || siteId > hi)) return false;
+		}
+		const searchTypeSummary = parts[SEARCH_TYPE_FIELD_INDEX];
+		if (searchTypeSummary && (searchTypeSummary.lower_bound != null || searchTypeSummary.upper_bound != null)) {
+			const lo = decodeString(searchTypeSummary.lower_bound);
+			const hi = decodeString(searchTypeSummary.upper_bound);
+			if (lo != null && hi != null && (searchType < lo || searchType > hi)) return false;
+		}
+		const monthSummary = parts[DATE_MONTH_FIELD_INDEX];
+		if (monthSummary && (monthSummary.lower_bound != null || monthSummary.upper_bound != null)) {
+			const lo = decodeInt(monthSummary.lower_bound);
+			const hi = decodeInt(monthSummary.upper_bound);
+			if (lo != null && hi != null) {
+				let anyInRange = false;
+				for (const wm of wantedMonths) if (wm >= lo && wm <= hi) {
+					anyInRange = true;
+					break;
+				}
+				if (!anyInRange) return false;
+			}
+		}
+		return true;
+	};
+}
 const ICEBERG_TYPE_MAP = {
 	STRING: "string",
 	INT: "int",
@@ -38,20 +103,51 @@ function icebergPartitionSpecFor(table) {
 		}))
 	};
 }
-async function connectIcebergCatalog(config) {
-	return {
-		catalog: await restCatalogConnect({
+const CATALOG_CONFIG_TTL_MS = 3600 * 1e3;
+function catalogConfigKey(config) {
+	return `gsc-catalog-cfg\0${config.catalogUri}\0${config.warehouse}`;
+}
+async function connectIcebergCatalog(config, opts = {}) {
+	const now = (opts.clock ?? Date.now)();
+	const requestInit = { headers: { Authorization: `Bearer ${config.catalogToken}` } };
+	let catalog;
+	if (opts.cache) {
+		const cached = await cacheGet(opts.cache, catalogConfigKey(config), now);
+		if (cached) catalog = Object.freeze({
+			type: "rest",
+			url: cached.url,
+			prefix: cached.prefix,
+			defaults: cached.defaults,
+			overrides: cached.overrides,
+			requestInit
+		});
+	}
+	if (!catalog) {
+		catalog = await restCatalogConnect({
 			url: config.catalogUri,
 			warehouse: config.warehouse,
-			requestInit: { headers: { Authorization: `Bearer ${config.catalogToken}` } }
-		}),
-		resolver: s3SignedResolver({
-			accessKeyId: config.s3.accessKeyId,
-			secretAccessKey: config.s3.secretAccessKey,
-			region: config.s3.region ?? "auto",
-			endpoint: config.s3.endpoint,
-			pathStyle: true
-		}),
+			requestInit
+		});
+		if (opts.cache) {
+			const toCache = {
+				url: catalog.url,
+				prefix: catalog.prefix,
+				defaults: catalog.defaults,
+				overrides: catalog.overrides
+			};
+			await cachePut(opts.cache, catalogConfigKey(config), toCache, CATALOG_CONFIG_TTL_MS, now);
+		}
+	}
+	const resolver = cachingResolver(s3SignedResolver({
+		accessKeyId: config.s3.accessKeyId,
+		secretAccessKey: config.s3.secretAccessKey,
+		region: config.s3.region ?? "auto",
+		endpoint: config.s3.endpoint,
+		pathStyle: true
+	}));
+	return {
+		catalog,
+		resolver,
 		namespace: config.namespace
 	};
 }
@@ -100,6 +196,14 @@ async function createIcebergTables(conn, tables = ICEBERG_TABLES) {
 async function listIcebergTables(conn) {
 	return (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name).sort();
 }
+const SNAPSHOT_REF_TTL_MS = 3e4;
+const RESOLVED_FILES_TTL_MS = 1440 * 60 * 1e3;
+function snapshotRefKey(namespace, table) {
+	return `gsc-snapref\0${namespace}\0${table}`;
+}
+function resolvedFilesKey(namespace, table, snapshotId, siteId, searchType, wantedMonths) {
+	return `gsc-files\0${namespace}\0${table}\0${snapshotId}\0${siteId}\0${searchType}\0${[...wantedMonths].sort((a, b) => a - b).join(",")}`;
+}
 function monthsInRange(range) {
 	const [sy, sm] = range.start.split("-").map(Number);
 	const [ey, em] = range.end.split("-").map(Number);
@@ -126,16 +230,56 @@ function stripBucket(filePath) {
 	const slash = rest.indexOf("/");
 	return slash >= 0 ? rest.slice(slash + 1) : rest;
 }
-async function listIcebergDataFiles(conn, opts) {
+async function loadSnapshotId(conn, opts, now) {
+	if (opts.cache) {
+		const cached = await cacheGet(opts.cache, snapshotRefKey(conn.namespace, opts.table), now);
+		if (cached !== void 0) return {
+			snapshotId: cached,
+			metadata: null
+		};
+	}
 	const { metadata } = await restCatalogLoadTable(conn.catalog, {
 		namespace: conn.namespace,
 		table: opts.table
 	});
-	if (metadata["current-snapshot-id"] == null) return [];
+	const raw = metadata["current-snapshot-id"];
+	const snapshotId = raw == null ? null : String(raw);
+	if (opts.cache) await cachePut(opts.cache, snapshotRefKey(conn.namespace, opts.table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
+	return {
+		snapshotId,
+		metadata
+	};
+}
+async function listIcebergDataFiles(conn, opts) {
+	const profiler = opts.profiler;
+	const now = (opts.clock ?? Date.now)();
 	const wantedMonths = new Set(monthsInRange(opts.range).map(monthsSinceEpoch));
+	const endSnapshot = profiler?.start("iceberg.snapshot");
+	let { snapshotId, metadata } = await loadSnapshotId(conn, opts, now);
+	endSnapshot?.({ cached: metadata == null && snapshotId != null });
+	if (snapshotId == null) return [];
+	const filesKey = resolvedFilesKey(conn.namespace, opts.table, snapshotId, opts.siteId, opts.searchType, wantedMonths);
+	if (opts.cache) {
+		const endCache = profiler?.start("iceberg.cache");
+		const cached = await cacheGet(opts.cache, filesKey, now);
+		endCache?.({ hit: cached !== void 0 });
+		if (cached !== void 0) return cached;
+	}
+	if (!metadata) {
+		const reloaded = await loadSnapshotId(conn, {
+			...opts,
+			cache: void 0
+		}, now);
+		snapshotId = reloaded.snapshotId;
+		metadata = reloaded.metadata;
+		if (snapshotId == null || !metadata) return [];
+	}
+	const endWalk = profiler?.start("iceberg.walk");
+	const partitionFilter = buildPartitionFilter(opts.siteId, opts.searchType, wantedMonths);
 	const manifests = await icebergManifests({
 		metadata,
-		resolver: conn.resolver
+		resolver: conn.resolver,
+		partitionFilter
 	});
 	const out = [];
 	for (const m of manifests) for (const entry of m.entries) {
@@ -154,6 +298,14 @@ async function listIcebergDataFiles(conn, opts) {
 			rowCount: Number(df.record_count)
 		});
 	}
+	endWalk?.({
+		manifests: manifests.length,
+		files: out.length
+	});
+	if (opts.cache) {
+		const freshKey = resolvedFilesKey(conn.namespace, opts.table, snapshotId, opts.siteId, opts.searchType, wantedMonths);
+		await cachePut(opts.cache, freshKey, out, RESOLVED_FILES_TTL_MS, now);
+	}
 	return out;
 }
 async function dropIcebergTables(conn, tables) {
@@ -266,4 +418,4 @@ function createIcebergAppendSink(options) {
 		}
 	};
 }
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions } from "./_chunks/storage.mjs";
+import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions } from "./_chunks/storage.mjs";
 import { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
 import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
 import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, formatEngineError, isEngineError } from "./_chunks/errors.mjs";
@@ -138,6 +138,25 @@ declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): Sear
  * decide how to handle it.
  */
 declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
+/**
+ * Build a {@link QueryProfiler} that records each closed span to `sink`.
+ *
+ * `start(name, meta)` stamps the open time and returns an `end` thunk; calling
+ * `end(extra)` records `{ name, ms, meta }` with `extra` merged over the
+ * open-time `meta` (so completion-only facts — row counts, buffered-file
+ * counts — land on the same span). `now` is injectable for deterministic
+ * tests; it defaults to `Date.now`.
+ */
+declare function createQueryProfiler(sink: (span: QuerySpan) => void, now?: () => number): QueryProfiler;
+/**
+ * A profiler that accumulates closed spans into an array — for tests, the CLI
+ * `query` command, or any ad-hoc "where did the time go" probe. The returned
+ * `spans` array is filled as spans close, in completion order.
+ */
+declare function collectSpans(now?: () => number): {
+  profiler: QueryProfiler;
+  spans: QuerySpan[];
+};
 /** A row as stored by the fake — data columns plus the injected identity columns. */
 type StoredRow = Row & {
   site_id: string;
@@ -190,4 +209,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };

package/dist/index.mjs

@@ -127,6 +127,32 @@ function createIngestAccumulator(opts) {
 		}
 	};
 }
+function createQueryProfiler(sink, now = () => Date.now()) {
+	return { start(name, meta) {
+		const t0 = now();
+		return (extra) => {
+			const merged = meta || extra ? {
+				...meta,
+				...extra
+			} : void 0;
+			sink(merged ? {
+				name,
+				ms: now() - t0,
+				meta: merged
+			} : {
+				name,
+				ms: now() - t0
+			});
+		};
+	} };
+}
+function collectSpans(now) {
+	const spans = [];
+	return {
+		profiler: createQueryProfiler((s) => spans.push(s), now),
+		spans
+	};
+}
 const KEY_SEP = "\0";
 function partitionKey(slice) {
 	return [
@@ -282,4 +308,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
 const MIN_COUNTRY_IMPRESSIONS = 10;
 const MAX_SITEMAP_URLS_PER_SITE = 5e4;
 const MAX_TRACKED_URLS_PER_SITE = 2e5;
-export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };