@gscdump/engine 1.4.0 → 1.4.2

package/dist/schema.mjs

@@ -1,2 +1,79 @@
-import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
+import { TABLE_METADATA, countries, dates, drizzleSchema, hourly_pages, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./drizzle-schema.mjs";
+import { getTableConfig } from "drizzle-orm/pg-core/utils";
+function pgSqlTypeToColumnType(sqlType) {
+	const t = sqlType.toLowerCase();
+	if (t.startsWith("varchar") || t === "text" || t.startsWith("char")) return "VARCHAR";
+	if (t === "date" || t.startsWith("timestamp")) return "DATE";
+	if (t.startsWith("double") || t === "real" || t.startsWith("numeric") || t.startsWith("decimal")) return "DOUBLE";
+	if (t === "bigint" || t === "int8") return "BIGINT";
+	if (t === "integer" || t === "int" || t === "int4" || t === "smallint" || t === "int2") return "INTEGER";
+	throw new Error(`unmapped pg type '${sqlType}' — extend pgSqlTypeToColumnType in @gscdump/engine/schema`);
+}
+function tableSchemaFrom(tableName) {
+	const columns = getTableConfig(drizzleSchema[tableName]).columns.map((col) => ({
+		name: col.name,
+		type: pgSqlTypeToColumnType(col.getSQLType()),
+		nullable: !col.notNull
+	}));
+	const meta = TABLE_METADATA[tableName];
+	return {
+		name: tableName,
+		columns,
+		sortKey: meta.sortKey,
+		version: meta.version
+	};
+}
+const METRIC_TABLES = [
+	"pages",
+	"queries",
+	"countries",
+	"page_queries",
+	"dates",
+	"search_appearance",
+	"search_appearance_pages",
+	"search_appearance_queries",
+	"search_appearance_page_queries",
+	"hourly_pages"
+];
+const SCHEMAS = Object.fromEntries(METRIC_TABLES.map((t) => [t, tableSchemaFrom(t)]));
+function currentSchemaVersion(table) {
+	return SCHEMAS[table].version;
+}
+function dateColumnsFor(table) {
+	return SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
+}
+function allTables() {
+	return METRIC_TABLES;
+}
+function inferTable(dimensions) {
+	const dims = new Set(dimensions);
+	const hasPage = dims.has("page");
+	const hasQuery = dims.has("query");
+	if (dims.has("searchAppearance")) return hasPage && hasQuery ? "search_appearance_page_queries" : hasPage ? "search_appearance_pages" : hasQuery ? "search_appearance_queries" : "search_appearance";
+	if (hasPage && hasQuery) return "page_queries";
+	if (hasQuery) return "queries";
+	if (hasPage) return "pages";
+	if (dims.has("country")) return "countries";
+	if (dims.has("device")) return "dates";
+	return "dates";
+}
+function naturalKeyColumns(table) {
+	return TABLE_METADATA[table].sortKey;
+}
+function dedupeByNaturalKey(table, rows) {
+	const key = TABLE_METADATA[table].sortKey;
+	if (key.length === 0) return rows.slice();
+	const seen = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const r = row;
+		const k = key.map((col) => `${r[col] ?? ""}`).join("\0");
+		seen.set(k, row);
+	}
+	return [...seen.values()];
+}
+function dimensionToColumn(dim, _table) {
+	if (dim === "page") return "url";
+	if (dim === "queryCanonical") return "query_canonical";
+	return dim;
+}
 export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/sink-node.d.mts

@@ -1,160 +1,2 @@
-import { Row } from "./_chunks/storage.mjs";
-import { IcebergS3Config, Sink, SinkCloseResult, SliceOverwriteWriter } from "./_chunks/sink.mjs";
-/** Connection details for the Iceberg REST catalog the writer targets. */
-interface OverwriteWriterCatalogConfig {
-  /**
-   * Iceberg REST catalog endpoint, e.g. `http://localhost:8181` (POC) or the
-   *  R2 Data Catalog endpoint (prod).
-   */
-  catalogUri: string;
-  /** Catalog namespace the 5 fact tables live under, e.g. `gsc`. */
-  namespace: string;
-  /** Warehouse identifier the catalog resolves table locations under. */
-  warehouse: string;
-  /** S3-compatible object store backing the catalog. */
-  s3: IcebergS3Config;
-  /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
-  catalogToken?: string;
-}
-/**
- * The JSON job wire-format shared with `scripts/iceberg-writer.py`.
- * `op` is always `overwrite` here.
- */
-interface OverwriteJob {
-  op: 'overwrite';
-  catalogUri: string;
-  namespace: string;
-  warehouse: string;
-  s3: IcebergS3Config;
-  table: string;
-  spec: unknown;
-  siteId: string;
-  searchType: string;
-  date: string;
-  /** Data columns only — the script injects `site_id` / `search_type`. */
-  rows: readonly Row[];
-}
-/**
- * Per-site delete job — removes EVERY row for one `siteId` across a whole
- * table (all dates / search types). Recovery op for clearing a corrupt site
- * (e.g. a cross-run append double) from a shared per-team shard before a clean
- * re-backfill, without touching sibling sites. Carries no rows. `siteId` is a
- * `number` for an INT-encoded shard, a `string` for STRING encoding.
- */
-interface DeleteJob {
-  op: 'delete';
-  catalogUri: string;
-  namespace: string;
-  warehouse: string;
-  s3: IcebergS3Config;
-  /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
-  catalogToken?: string;
-  table: string;
-  siteId: string | number;
-}
-/** Any job the shared PyIceberg backend can execute. */
-type PyIcebergJob = OverwriteJob | DeleteJob;
-/** Result wire-format from the PyIceberg backend. */
-interface OverwriteJobResult {
-  /** Rows written (emit/overwrite) or removed (delete). */
-  rowCount?: number;
-  error?: string;
-}
-/**
- * A transport that executes one PyIceberg job and returns its result.
- * `subprocessBackend` and `httpBackend` are the two shipped implementations;
- * tests inject a fake. The param is the `OverwriteJob | DeleteJob` union, so a
- * backend value is still assignable where the narrower `OverwriteBackend` is
- * expected (a wider-param function accepts the narrower call).
- */
-type OverwriteBackend = (job: PyIcebergJob) => Promise<OverwriteJobResult>;
-interface IcebergOverwriteWriterOptions {
-  catalog: OverwriteWriterCatalogConfig;
-  /** The transport that runs the PyIceberg job. */
-  backend: OverwriteBackend;
-}
-/**
- * The trailing-window Iceberg overwrite writer. Conforms to the frozen
- * `SliceOverwriteWriter` contract; `overwriteSlice` builds one PyIceberg
- * overwrite job per call and runs it through the configured backend.
- */
-interface IcebergOverwriteWriter extends SliceOverwriteWriter {
-  /** Release any backend resources. Idempotent. */
-  close: () => Promise<SinkCloseResult>;
-}
-/**
- * Build a trailing-window Iceberg overwrite writer over the given backend.
- * Pure — all I/O is the backend's; the writer only shapes the job.
- */
-declare function createIcebergOverwriteWriter(opts: IcebergOverwriteWriterOptions): IcebergOverwriteWriter;
-/** Options for the PyIceberg subprocess backend. */
-interface SubprocessBackendOptions {
-  /** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
-  python?: string;
-  /**
-   * Path to the PyIceberg writer script. Defaults to
-   *  `<engine>/scripts/iceberg-writer.py`.
-   */
-  writerScript?: string;
-}
-/**
- * Backend that runs the PyIceberg overwrite by spawning the shared writer
- * script (`scripts/iceberg-writer.py`). Node-only — used by local integration
- * tests and a Node-side scheduled job box. The script reads the job JSON on
- * stdin and writes `{rowCount}` / `{error}` on stdout.
- */
-declare function subprocessBackend(opts?: SubprocessBackendOptions): OverwriteBackend;
-/** Options for the PyIceberg HTTP backend (the Cloudflare Container in prod). */
-interface HttpBackendOptions {
-  /**
-   * URL of the PyIceberg overwrite service — a POST endpoint accepting the
-   *  `OverwriteJob` JSON and returning `OverwriteJobResult`.
-   */
-  endpoint: string;
-  /** Optional bearer token for the service. */
-  token?: string;
-  /** Injectable fetch — defaults to the global. */
-  fetch?: typeof fetch;
-}
-/**
- * Backend that POSTs the overwrite job to a PyIceberg HTTP service — the
- * Cloudflare Container running `iceberg-writer.py` behind an HTTP shim. This
- * is the prod transport: the request-path Worker holds no Python and never
- * does an Iceberg metadata commit itself.
- */
-declare function httpBackend(opts: HttpBackendOptions): OverwriteBackend;
-/**
- * Adapt an `IcebergOverwriteWriter` into a full `Sink` whose `emit` is also a
- * partition-overwrite — convenient for tests so a single Iceberg writer backs
- * both append and revision paths against the local stack. Prod keeps `emit`
- * on the Pipeline; only `overwriteSlice` delegates here.
- */
-declare function overwriteWriterAsSink(writer: IcebergOverwriteWriter): Sink & SliceOverwriteWriter;
-/** Per-table outcome of a {@link deleteSiteFromShard} run. */
-interface DeleteSiteResult {
-  table: string;
-  /** Rows removed, when the backend reported a count. */
-  rowCount?: number;
-  /** Present when this table's delete failed; the sweep continues regardless. */
-  error?: string;
-}
-/**
- * Remove every row for one `siteId` from a per-team Iceberg shard, across the
- * given tables. Recovery op: clear a corrupt site (e.g. a cross-run append
- * double) before a clean re-backfill, leaving sibling sites in the shared shard
- * untouched. `site_id` is an identity partition column, so PyIceberg prunes
- * whole data files (no row-level delete files).
- *
- * Runs SEQUENTIALLY (one commit per table at a time) so the per-table catalog
- * commit-rate ceiling is never tripped. A table failure is captured in its
- * result row, not thrown — the sweep always attempts every table so a partial
- * failure is visible and re-runnable (delete is idempotent: re-deleting an
- * already-empty site is a no-op).
- */
-declare function deleteSiteFromShard(args: {
-  catalog: OverwriteWriterCatalogConfig;
-  backend: OverwriteBackend;
-  siteId: string | number;
-  tables: readonly string[];
-}): Promise<DeleteSiteResult[]>;
+import { DeleteJob, DeleteSiteResult, IcebergOverwriteWriter, IcebergOverwriteWriterOptions, OverwriteBackend, OverwriteJob, OverwriteJobResult, OverwriteWriterCatalogConfig, PyIcebergJob, createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend } from "./iceberg/overwrite-writer.mjs";
 export { type DeleteJob, type DeleteSiteResult, type IcebergOverwriteWriter, type IcebergOverwriteWriterOptions, type OverwriteBackend, type OverwriteJob, type OverwriteJobResult, type OverwriteWriterCatalogConfig, type PyIcebergJob, createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/dist/sink-node.mjs

@@ -1,134 +1,2 @@
-import { ICEBERG_SCHEMAS, assertIcebergTable } from "./_chunks/schema2.mjs";
-import process from "node:process";
-const PYICEBERG_PYTHON_ENV = "GSCDUMP_ICEBERG_PYTHON";
-const DEFAULT_PYICEBERG_PYTHON = "python3";
-function resolvePyIcebergPython(override) {
-	return override ?? process.env[PYICEBERG_PYTHON_ENV] ?? DEFAULT_PYICEBERG_PYTHON;
-}
-async function runPyIcebergWriter(options) {
-	const { execFile } = await import("node:child_process");
-	return new Promise((resolve, reject) => {
-		execFile(options.python, [options.script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
-			let parsed;
-			let parseError;
-			if (stdout.trim()) try {
-				parsed = JSON.parse(stdout);
-			} catch (error) {
-				parseError = error;
-			}
-			if (parsed && !(err && options.rejectOnProcessError)) {
-				resolve(parsed);
-				return;
-			}
-			if (err) {
-				if (options.processErrorAsParseFailure) {
-					reject(/* @__PURE__ */ new Error(`${options.label} produced no parseable output (${err.message})${stderr ? `: ${stderr}` : ""}`));
-					return;
-				}
-				reject(/* @__PURE__ */ new Error(`${options.label} process failed (${err.message})${stderr ? `: ${stderr}` : ""}`));
-				return;
-			}
-			reject(new Error(`${options.label} produced no parseable output: ${stdout || stderr}`, { cause: parseError }));
-		}).stdin?.end(JSON.stringify(options.job));
-	});
-}
-function createIcebergOverwriteWriter(opts) {
-	const { catalog, backend } = opts;
-	async function overwriteSlice(slice, rows) {
-		const siteId = slice.ctx.siteId;
-		if (!siteId) throw new Error("overwriteSlice: slice.ctx.siteId is required for the Iceberg partition key");
-		const table = assertIcebergTable(slice.table);
-		const job = {
-			op: "overwrite",
-			catalogUri: catalog.catalogUri,
-			namespace: catalog.namespace,
-			warehouse: catalog.warehouse,
-			s3: catalog.s3,
-			table,
-			spec: ICEBERG_SCHEMAS[table],
-			siteId,
-			searchType: slice.searchType,
-			date: slice.date,
-			rows
-		};
-		const res = await backend(job);
-		if (res.error) throw new Error(`overwriteSlice: PyIceberg backend failed: ${res.error}`);
-		return { rowCount: res.rowCount ?? rows.length };
-	}
-	return {
-		overwriteSlice,
-		async close() {
-			return {
-				flushed: [],
-				failed: []
-			};
-		}
-	};
-}
-function subprocessBackend(opts = {}) {
-	return async (job) => {
-		const { dirname, join } = await import("node:path");
-		const { fileURLToPath } = await import("node:url");
-		return runPyIcebergWriter({
-			python: resolvePyIcebergPython(opts.python),
-			script: opts.writerScript ?? join(dirname(fileURLToPath(import.meta.url)), "..", "scripts", "iceberg-writer.py"),
-			job,
-			label: "iceberg overwrite subprocess",
-			processErrorAsParseFailure: true
-		});
-	};
-}
-function httpBackend(opts) {
-	const doFetch = opts.fetch ?? fetch;
-	return async (job) => {
-		const res = await doFetch(opts.endpoint, {
-			method: "POST",
-			headers: {
-				"content-type": "application/json",
-				...opts.token ? { authorization: `Bearer ${opts.token}` } : {}
-			},
-			body: JSON.stringify(job)
-		});
-		if (!res.ok) {
-			const body = await res.text().catch(() => "");
-			return { error: `HTTP ${res.status}${body ? `: ${body}` : ""}` };
-		}
-		return await res.json();
-	};
-}
-function overwriteWriterAsSink(writer) {
-	return {
-		capabilities: {
-			canOverwrite: true,
-			appendOnly: false
-		},
-		emit: (slice, rows) => writer.overwriteSlice(slice, rows),
-		overwriteSlice: (slice, rows) => writer.overwriteSlice(slice, rows),
-		close: () => writer.close()
-	};
-}
-async function deleteSiteFromShard(args) {
-	const { catalog, backend, siteId, tables } = args;
-	const results = [];
-	for (const table of tables) {
-		const res = await backend({
-			op: "delete",
-			catalogUri: catalog.catalogUri,
-			namespace: catalog.namespace,
-			warehouse: catalog.warehouse,
-			s3: catalog.s3,
-			...catalog.catalogToken ? { catalogToken: catalog.catalogToken } : {},
-			table,
-			siteId
-		}).catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
-		results.push(res.error ? {
-			table,
-			error: res.error
-		} : {
-			table,
-			rowCount: res.rowCount
-		});
-	}
-	return results;
-}
+import { createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend } from "./iceberg/overwrite-writer.mjs";
 export { createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/dist/sink.d.mts

@@ -0,0 +1,125 @@
+import { Row, SearchType, TenantCtx } from "./storage.mjs";
+import { EngineError } from "./errors.mjs";
+import { IcebergTableName, PartitionKeyEncoding } from "./iceberg/schema.mjs";
+import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig } from "./iceberg/catalog.mjs";
+/**
+ * Identifies one fact slice — the atomic unit a sink emits.
+ * `(table, site, searchType, date)`. `userId` rides along on `ctx` for
+ * tenant-scoped sinks (local/in-memory); the prod Iceberg table is global
+ * and keys only on `siteId`.
+ */
+interface SinkSlice {
+  ctx: TenantCtx;
+  table: IcebergTableName;
+  /** GSC search-type partition. */
+  searchType: SearchType;
+  /** Calendar day (PT), `YYYY-MM-DD`. The slice's `month(date)` partition. */
+  date: string;
+}
+/**
+ * Outcome of a sink write. `rowCount` is the number of rows accepted;
+ * `bytes` is best-effort — `IcebergAppendSink` does not report it (undefined there).
+ */
+interface SinkWriteResult {
+  rowCount: number;
+  bytes?: number;
+}
+/**
+ * Static description of a sink. Production ingestion is append-only under the
+ * v5 stability-cutoff model. Test and revision-path adapters may expose an
+ * overwrite capability explicitly.
+ */
+interface SinkCapabilities {
+  /** When true, re-emitting a slice accumulates duplicate rows. */
+  appendOnly: boolean;
+  /** Whether the sink exposes partition overwrite semantics. */
+  canOverwrite?: boolean;
+}
+/** Partition-overwrite writer for revision paths. */
+interface SliceOverwriteWriter {
+  overwriteSlice: (slice: SinkSlice, rows: readonly Row[]) => Promise<SinkWriteResult>;
+}
+/**
+ * Outcome of `Sink.close()` — which tables' buffered rows reached durable
+ * storage and which failed.
+ *
+ * `IcebergAppendSink.emit` only BUFFERS; the durable Iceberg commit happens
+ * in `close()`, one `icebergAppend()` per table. The ingest ledger
+ * (`sinkAsIngestEngine`) records a `(site, table, searchType, date)` slice
+ * ONLY after the table holding it appears in `flushed` — a table in `failed`
+ * leaves its slices un-recorded so the next sync re-emits them. This is what
+ * keeps the D1 ledger from ever running ahead of Iceberg.
+ */
+interface SinkCloseResult {
+  /** Tables whose buffered rows committed durably. */
+  flushed: IcebergTableName[];
+  /**
+   * Tables whose flush failed — their slices must NOT be ledger-recorded. Each
+   * carries a typed `sink-table-flush-failed` `EngineError`; the human-readable
+   * cause is on `error.message`.
+   */
+  failed: {
+    table: IcebergTableName;
+    error: EngineError;
+  }[];
+}
+interface Sink {
+  readonly capabilities: SinkCapabilities;
+  /**
+   * Emit the fact rows for one slice. Append semantics — for `IcebergAppendSink`
+   * this commits one Iceberg snapshot. Re-emitting the same slice produces
+   * DUPLICATE rows; exactly-once is enforced upstream by the D1 ingested-days
+   * ledger, which only calls `emit` for a slice once.
+   *
+   * `rows` carry the table's data columns; the sink injects the partition
+   * identity columns (`site_id`, `search_type`) from `slice` — callers MUST
+   * NOT pre-populate them.
+   */
+  emit: (slice: SinkSlice, rows: readonly Row[]) => Promise<SinkWriteResult>;
+  /**
+   * Flush any buffered rows and release resources, returning which tables
+   * reached durable storage. Idempotent — a second `close()` after a flush
+   * reports empty `flushed`/`failed` (nothing left buffered).
+   *
+   * `IcebergAppendSink` buffers in `emit` and commits one `icebergAppend()`
+   * per table HERE; `flushed`/`failed` reflect those per-table commits.
+   * In-memory / local sinks write durably in `emit`, so they report every
+   * table they received rows for as `flushed`.
+   */
+  close: () => Promise<SinkCloseResult>;
+}
+/** Construction options shared by all sink implementations. */
+interface SinkOptions {
+  now?: () => number;
+}
+/**
+ * `IcebergAppendSink`-specific options — the R2 Data Catalog coordinates the
+ * sink appends to via `icebird`. The catalog config shape (`IcebergCatalogConfig`)
+ * is type-only here so this contract file stays implementation-free.
+ */
+interface IcebergAppendSinkOptions extends SinkOptions {
+  /** R2 Data Catalog connection config (catalog URI, warehouse, namespace, token, S3 creds). */
+  catalog: IcebergCatalogConfig;
+  /**
+   * Options forwarded to the lazy catalog connection. Supply a shared
+   * `CatalogCache` here to serve the REST `/v1/config` probe from durable cache
+   * across short-lived Worker isolates.
+   */
+  connect?: ConnectIcebergOptions;
+  /**
+   * Retry policy for the per-table `icebergAppend()` commit, applied on R2
+   * Data Catalog 429 ("too many commits") rate-limits. Optional — production
+   * uses the defaults; tests inject a synchronous `sleep`.
+   */
+  commitRetry?: CommitRetryOptions;
+  /**
+   * Partition-key encoding (default `'int'` for new catalogs). `'int'` writes
+   * BOTH `site_id` and `search_type` as INT — the caller MUST pass the numeric
+   * `site_id` (a numeric string is fine; it's `Number()`-coerced) in
+   * `slice.ctx.siteId`. Pass `'string'` explicitly for legacy catalogs. A small
+   * INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
+   * {@link import('./iceberg/schema').PartitionKeyEncoding}.
+   */
+  encoding?: PartitionKeyEncoding;
+}
+export { IcebergAppendSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter };

package/dist/sinks/in-memory-sink.d.mts

@@ -0,0 +1,25 @@
+import { Row } from "../storage.mjs";
+import { IcebergTableName } from "../iceberg/schema.mjs";
+import { Sink, SinkSlice } from "../sink.mjs";
+/** A row as stored by the fake — data columns plus the injected identity columns. */
+type StoredRow = Row & {
+  site_id: string;
+  search_type: string;
+};
+interface InMemorySink extends Sink {
+  /** All rows accepted, in partition order. */
+  readonly rows: readonly StoredRow[];
+  /** Rows for one table. */
+  rowsFor: (table: IcebergTableName) => StoredRow[];
+  /** Rows for one exact slice partition. */
+  rowsForSlice: (slice: SinkSlice) => StoredRow[];
+  /** `true` once `close()` has run. */
+  readonly closed: boolean;
+  /** Drop everything — handy between test cases. */
+  reset: () => void;
+}
+/**
+ * Create an in-memory append-only `Sink`.
+ */
+declare function createInMemorySink(): InMemorySink;
+export { InMemorySink, StoredRow, createInMemorySink };

package/dist/sinks/in-memory-sink.mjs

@@ -0,0 +1,61 @@
+const KEY_SEP = "\0";
+function partitionKey(slice) {
+	return [
+		slice.table,
+		slice.ctx.siteId ?? "",
+		slice.searchType,
+		slice.date
+	].join(KEY_SEP);
+}
+function tableOfKey(key) {
+	return key.slice(0, key.indexOf(KEY_SEP));
+}
+function withIdentity(slice, rows) {
+	return rows.map((r) => ({
+		...r,
+		site_id: slice.ctx.siteId ?? "",
+		search_type: slice.searchType
+	}));
+}
+function createInMemorySink() {
+	const partitions = /* @__PURE__ */ new Map();
+	let closed = false;
+	function allRows() {
+		return [...partitions.values()].flat();
+	}
+	return {
+		capabilities: { appendOnly: true },
+		async emit(slice, rows) {
+			const key = partitionKey(slice);
+			const stored = withIdentity(slice, rows);
+			const existing = partitions.get(key);
+			if (existing) existing.push(...stored);
+			else partitions.set(key, stored);
+			return { rowCount: stored.length };
+		},
+		async close() {
+			closed = true;
+			return {
+				flushed: [...new Set([...partitions.keys()].map((k) => tableOfKey(k)))],
+				failed: []
+			};
+		},
+		get rows() {
+			return allRows();
+		},
+		get closed() {
+			return closed;
+		},
+		rowsFor(table) {
+			return [...partitions.entries()].filter(([k]) => tableOfKey(k) === table).flatMap(([, v]) => v);
+		},
+		rowsForSlice(slice) {
+			return [...partitions.get(partitionKey(slice)) ?? []];
+		},
+		reset() {
+			partitions.clear();
+			closed = false;
+		}
+	};
+}
+export { createInMemorySink };

package/dist/sinks/index.d.mts

@@ -0,0 +1,2 @@
+import { InMemorySink, StoredRow, createInMemorySink } from "./in-memory-sink.mjs";
+export { type InMemorySink, type StoredRow, createInMemorySink };

package/dist/sinks/index.mjs

@@ -0,0 +1,2 @@
+import { createInMemorySink } from "./in-memory-sink.mjs";
+export { createInMemorySink };

package/dist/snapshot.d.mts

@@ -0,0 +1,14 @@
+/**
+ * Describes a hot/cold snapshot set. Produced by the snapshot builder,
+ * consumed by `attachSnapshotIndex`. Filenames are derived from `cold`
+ * via `cold-${yearMonth}.duckdb`; hot is always `hot.duckdb` when
+ * `hot: true`.
+ */
+interface SnapshotIndex {
+  version: 1;
+  builtAt: string;
+  cold: string[];
+  hot: boolean;
+  hotDays: number;
+}
+export { SnapshotIndex };

package/dist/source/attached-table.d.mts

@@ -0,0 +1,45 @@
+import { Row } from "../storage.mjs";
+import { ResolverAdapter } from "../resolver/types.mjs";
+import { AnalysisQuerySource } from "./source-types.mjs";
+import { EngineError } from "../errors.mjs";
+import "../contracts.mjs";
+interface AttachedTableRunner {
+  /**
+   * Run a query with positional (`?`) bound parameters. Return objects keyed
+   * by column name. BIGINT → number coercion is applied by the source factory
+   * (see `coerceRows`); runners only need to handle DATE → ISO string (or
+   * let the analyzer reducer normalize via `num(v)`/`str(v)`).
+   */
+  query: (sql: string, params?: unknown[], signal?: AbortSignal) => Promise<Row[]>;
+}
+interface AttachedTableSourceOptions {
+  /** Schema name the exported DuckDB file was attached under — e.g. `gsc`. */
+  schema: string;
+  /**
+   * Abort in-flight queries when the caller no longer cares about the
+   * result. Every `runner.query` call receives the same signal.
+   */
+  signal?: AbortSignal;
+  /**
+   * List of table names actually attached to this connection. When provided,
+   * `executeSql` short-circuits with a specific "table not attached" error
+   * if the SQL plan references a table that isn't in this list — letting
+   * callers (e.g. the analytics layer) route to cloud fallback without
+   * paying the SQL execution cost. Omit to disable the check.
+   */
+  attachedTables?: readonly string[];
+  /**
+   * Dialect adapter surfaced on the source for analyzers that compose SQL
+   * from a `BuilderState` at plan-build time (e.g. `data-query`,
+   * `data-detail`). Attached-table sources execute pg-flavored DuckDB SQL,
+   * so callers should pass `pgResolverAdapter` here.
+   */
+  adapter?: ResolverAdapter<any>;
+}
+declare class AttachedTableMissingError extends Error {
+  readonly missing: readonly string[];
+  readonly engineError: EngineError;
+  constructor(missing: readonly string[]);
+}
+declare function createAttachedTableSource(runner: AttachedTableRunner, options: AttachedTableSourceOptions): AnalysisQuerySource;
+export { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, createAttachedTableSource };