@gscdump/engine 1.0.2 → 1.0.4

package/README.md

@@ -23,15 +23,17 @@ Optional peers (install only what your runtime needs):
 
 | Subpath | Purpose |
 |---|---|
-| `@gscdump/engine` | Barrel: `createStorageEngine`, codec/executor factories, all storage contracts. |
+| `@gscdump/engine` | Storage runtime: `createStorageEngine`, codec/executor factories, storage contracts. |
 | `@gscdump/engine/contracts` | `StorageEngine`, `Row`, `TableName`, `WriteCtx`, `ManifestEntry`, ... |
 | `@gscdump/engine/schema` | `SCHEMAS`, `allTables`, `inferTable`, column metadata. |
 | `@gscdump/engine/planner` | `resolveToSQL`, `enumeratePartitions`, partition planning. |
 | `@gscdump/engine/ingest` | GSC row → storage row helpers (`createRowAccumulator`, `transformGscRow`). |
+| `@gscdump/engine/ingest-accumulator` | Stateful ingest flushing behind injected storage and recovery hooks. |
 | `@gscdump/engine/sql` | SQL literal binding helpers (`bindLiterals`, `formatLiteral`). |
 | `@gscdump/engine/sql-fragments` | Reusable SQL fragments shared across analyzers. |
 | `@gscdump/engine/rollups` | Pre-aggregated rollup contracts + helpers. |
-| `@gscdump/engine/entities` | Entity helpers (sites, tenants, scope keys). |
+| `@gscdump/engine/entities` | Parquet-backed entity stores and their record contracts. |
+| `@gscdump/engine/entity-keys` | Pure entity object-key and URL-hash helpers, with no Parquet runtime. |
 | `@gscdump/engine/resolver` | Dialect-neutral SQL composition: `ResolverAdapter`, `pgResolverAdapter`, `resolveToSQL`. |
 | `@gscdump/engine/scope` | Multi-tenant scope predicates. |
 | `@gscdump/engine/arrow` | Apache Arrow utilities for engine result conversion. |

package/dist/_chunks/coerce.mjs

@@ -1,4 +1,4 @@
-import { coerceBigIntToNumber } from "@gscdump/lakehouse";
+import { coerceBigIntToNumber } from "@gscdump/lakehouse/bigint";
 function coerceRow(row) {
 	let mutated = null;
 	for (const k in row) {

package/dist/_chunks/engine.mjs

@@ -4,7 +4,7 @@ import { SCHEMAS, TABLE_METADATA, currentSchemaVersion, dateColumnsFor, dedupeBy
 import { compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, splitOverlappingTiers, substituteNamedFiles } from "./parquet-plan.mjs";
 import { dateReplaceClause as dateReplaceClause$1 } from "../sql-fragments.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
-import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
 const DEFAULT_BUFFER_READ_CONCURRENCY = 16;

package/dist/_chunks/entities.mjs

@@ -1,5 +1,6 @@
+import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "../entity-keys.mjs";
 import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
-import { encodeJsonBigintSafe } from "@gscdump/lakehouse";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 function isMissingKeyError(e) {
 	if (typeof e !== "object" || e === null) return false;
 	if (e.code === "ENOENT") return true;
@@ -41,15 +42,6 @@ const QUERY_DIM_COLUMNS = [
 		nullable: false
 	}
 ];
-function queryDimPrefix(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/query_dim` : `u_${ctx.userId}/entities/query_dim`;
-}
-function queryDimParquetKey(ctx) {
-	return `${queryDimPrefix(ctx)}/index.parquet`;
-}
-function queryDimMetaKey(ctx) {
-	return `${queryDimPrefix(ctx)}/index.json`;
-}
 function buildQueryDimRecords(queries, deps) {
 	const seen = /* @__PURE__ */ new Set();
 	const out = [];
@@ -125,45 +117,7 @@ async function mapEntityIo(items, fn) {
 	await Promise.all(Array.from({ length: Math.min(ENTITY_IO_CONCURRENCY, items.length) }, worker));
 	return results;
 }
-function inspectionIndexKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
-}
-function emptyTypesKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/empty-types.json` : `u_${ctx.userId}/entities/empty-types.json`;
-}
-function inspectionParquetKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.parquet` : `u_${ctx.userId}/entities/inspections/index.parquet`;
-}
-function inspectionEventsPrefix(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/events` : `u_${ctx.userId}/entities/inspections/events`;
-}
-function inspectionEventKey(ctx, yearMonth, batchId) {
-	return `${inspectionEventsPrefix(ctx)}/${yearMonth}/${batchId}.parquet`;
-}
-function inspectionBaseKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/base.parquet` : `u_${ctx.userId}/entities/inspections/base.parquet`;
-}
 const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
-function inspectionHistoryPrefix(ctx, yearMonth) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/history/${yearMonth}` : `u_${ctx.userId}/entities/inspections/history/${yearMonth}`;
-}
-function inspectionHistoryShardKey(ctx, yearMonth, batchId) {
-	return `${inspectionHistoryPrefix(ctx, yearMonth)}/${batchId}.json`;
-}
-function hashUrl(url) {
-	let hi = 2166136261;
-	let lo = 3421674724;
-	for (let i = 0; i < url.length; i++) {
-		const c = url.charCodeAt(i);
-		lo ^= c;
-		const loMul = Math.imul(lo, 435) >>> 0;
-		const carry = Math.floor(lo * 435 / 4294967296);
-		const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
-		lo = loMul;
-		hi = hiMul;
-	}
-	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
-}
 const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
 const INSPECTION_PARQUET_COLUMNS = [
 	{
@@ -459,24 +413,6 @@ function createInspectionStore(opts) {
 		}
 	};
 }
-function sitemapIndexKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/index.json` : `u_${ctx.userId}/entities/sitemaps/index.json`;
-}
-function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json` : `u_${ctx.userId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
-}
-function sitemapUrlsPrefix(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/urls` : `u_${ctx.userId}/entities/sitemaps/urls`;
-}
-function sitemapUrlsIndexPrefix(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/by-feed`;
-}
-function sitemapUrlsIndexKey(ctx, feedpathHash) {
-	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
-}
-function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
-	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
-}
 const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
 const URLS_INDEX_COLUMNS = [
 	{
@@ -584,27 +520,6 @@ function urlRecordToRow(r) {
 function isoDate(ms) {
 	return new Date(ms).toISOString().slice(0, 10);
 }
-function hashUrlList(urls) {
-	return hashSortedUrlList(urls.map((u) => u.loc).sort());
-}
-function hashSortedUrlList(locs) {
-	let hi = 2166136261;
-	let lo = 3421674724;
-	for (let locIndex = 0; locIndex < locs.length; locIndex++) {
-		const loc = locs[locIndex];
-		const length = loc.length + (locIndex < locs.length - 1 ? 1 : 0);
-		for (let i = 0; i < length; i++) {
-			const c = i < loc.length ? loc.charCodeAt(i) : 10;
-			lo ^= c;
-			const loMul = Math.imul(lo, 435) >>> 0;
-			const carry = Math.floor(lo * 435 / 4294967296);
-			const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
-			lo = loMul;
-			hi = hiMul;
-		}
-	}
-	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
-}
 function createSitemapStore(opts) {
 	const ds = opts.dataSource;
 	const hash = opts.hash ?? hashUrl;
@@ -968,9 +883,6 @@ function createSitemapStore(opts) {
 		}
 	};
 }
-function indexingMetadataIndexKey(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/indexing/index.json` : `u_${ctx.userId}/entities/indexing/index.json`;
-}
 function createIndexingMetadataStore(opts) {
 	const ds = opts.dataSource;
 	const hash = opts.hash ?? hashUrl;
@@ -1052,4 +964,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore };

package/dist/_chunks/index.d.mts

@@ -1,104 +1,53 @@
-import { Row, SearchType as SearchType$1, StorageEngine, TenantCtx } from "./storage.mjs";
-import { AnalysisParams, AnalysisResult } from "./analysis-types.mjs";
-import { ResolverAdapter } from "./types.mjs";
-import { AnalysisQuerySource, AnalysisSourceKind, AnalyzerRegistry, QueryRow, SourceCapabilities } from "./registry.mjs";
-import { EngineError } from "./errors.mjs";
-import "./contracts.mjs";
-import { PlannerCapabilities } from "gscdump/query/plan";
-import { BuilderState } from "gscdump/query";
-interface AttachedTableRunner {
-  /**
-   * Run a query with positional (`?`) bound parameters. Return objects keyed
-   * by column name. BIGINT → number coercion is applied by the source factory
-   * (see `coerceRows`); runners only need to handle DATE → ISO string (or
-   * let the analyzer reducer normalize via `num(v)`/`str(v)`).
-   */
-  query: (sql: string, params?: unknown[], signal?: AbortSignal) => Promise<Row[]>;
+import { AnalysisParams } from "./analysis-types.mjs";
+type WindowPreset = 'last-7d' | 'last-28d' | 'last-30d' | 'last-90d' | 'last-180d' | 'last-365d' | 'mtd' | 'ytd' | 'custom';
+type ComparisonMode = 'none' | 'prev-period' | 'yoy';
+interface ResolveWindowOptions {
+  preset: WindowPreset;
+  comparison?: ComparisonMode;
+  anchor?: string;
+  start?: string;
+  end?: string;
 }
-interface AttachedTableSourceOptions {
-  /** Schema name the exported DuckDB file was attached under — e.g. `gsc`. */
-  schema: string;
-  /**
-   * Abort in-flight queries when the caller no longer cares about the
-   * result. Every `runner.query` call receives the same signal.
-   */
-  signal?: AbortSignal;
-  /**
-   * List of table names actually attached to this connection. When provided,
-   * `executeSql` short-circuits with a specific "table not attached" error
-   * if the SQL plan references a table that isn't in this list — letting
-   * callers (e.g. the analytics layer) route to cloud fallback without
-   * paying the SQL execution cost. Omit to disable the check.
-   */
-  attachedTables?: readonly string[];
-  /**
-   * Dialect adapter surfaced on the source for analyzers that compose SQL
-   * from a `BuilderState` at plan-build time (e.g. `data-query`,
-   * `data-detail`). Attached-table sources execute pg-flavored DuckDB SQL,
-   * so callers should pass `pgResolverAdapter` here.
-   */
-  adapter?: ResolverAdapter<any>;
+interface ResolvedWindow {
+  start: string;
+  end: string;
+  days: number;
+  comparison?: {
+    start: string;
+    end: string;
+  };
 }
-declare class AttachedTableMissingError extends Error {
-  readonly missing: readonly string[];
-  readonly engineError: EngineError;
-  constructor(missing: readonly string[]);
+interface AnalysisPeriod {
+  startDate: string;
+  endDate: string;
 }
-declare function createAttachedTableSource(runner: AttachedTableRunner, options: AttachedTableSourceOptions): AnalysisQuerySource;
-interface CreateSqlQuerySourceOptions<TKey extends string> {
-  /** Debug-only identifier surfaced on the source for error messages. */
-  name: string;
-  /** Telemetry tag stamped onto analyzer result meta. */
-  kind?: AnalysisSourceKind;
-  /** Dialect-specific adapter; compiles `BuilderState` → `{ sql, params }`. */
-  adapter: ResolverAdapter<TKey>;
-  /** Drives the underlying DB. Called for both typed queries and raw SQL. */
-  execute: (sql: string, params: unknown[]) => Promise<QueryRow[]>;
-  /** Tenant id for multi-tenant dialects; forwarded to `resolveToSQL`. */
-  siteId?: string | number;
-  /**
-   * Search-type scope for multi-tenant dialects; forwarded to `resolveToSQL`.
-   * `number` = int-encoded code (`SEARCH_TYPE_INT`) for INT `search_type`
-   * catalogs (bound bare so the int partition prunes); `string` otherwise.
-   */
-  searchType?: string | number;
-  /** Additional capability flags merged on top of `adapter.capabilities`. */
-  extraCapabilities?: Partial<SourceCapabilities>;
+interface ComparisonPeriod {
+  current: AnalysisPeriod;
+  previous: AnalysisPeriod;
 }
-declare function createSqlQuerySource<TKey extends string>(options: CreateSqlQuerySourceOptions<TKey>): AnalysisQuerySource;
-/**
- * Capabilities the engine query path honors. Matches what the DuckDB compiler
- * passes to `buildLogicalPlan`: regex pushes down; comparison joins and
- * multi-dataset queries belong to the analyzer dispatcher, not the engine's
- * builder-state query path.
- */
-declare const ENGINE_QUERY_CAPABILITIES: PlannerCapabilities;
-interface EngineQuerySourceOptions {
-  engine: StorageEngine;
-  ctx: TenantCtx;
+declare function defaultEndDate(): string;
+declare function periodOf(params: AnalysisParams): AnalysisPeriod;
+declare function comparisonOf(params: AnalysisParams): ComparisonPeriod;
+declare function resolveWindow(opts: ResolveWindowOptions): ResolvedWindow;
+interface PadTimeseriesOptions<T> {
+  /** ISO date (YYYY-MM-DD), inclusive lower bound. */
+  startDate: string;
+  /** ISO date (YYYY-MM-DD), inclusive upper bound. */
+  endDate: string;
   /**
-   * Restrict every manifest lookup the source performs to a single search-type
-   * slice. Threads into `engine.query` and `engine.runSQL` so the wrapped
-   * source returns rows from one cohort instead of unioning web + non-web
-   * parquet. Undefined preserves legacy cross-type behaviour for web-only
-   * tenants and admin paths.
+   * Row to insert for missing dates. Defaults to `{ clicks: 0, impressions: 0, ctr: 0, position: 0 }`.
+   * The `date` field is set automatically.
    */
-  searchType?: SearchType$1;
+  fill?: Omit<T, 'date'>;
+  /** Row-field that carries the ISO date. Defaults to `date`. */
+  dateKey?: string;
 }
+type DateRowShape = Record<string, unknown> & {
+  date?: unknown;
+};
 /**
- * Wraps a storage engine as an `AnalysisQuerySource` with SQL execution.
- * `queryRows` runs typed builder-state queries; `executeSql` delegates to
- * `engine.runSQL` and requires `opts.fileSets` (with a `FILES` entry so the
- * target table can be resolved for partition lookup).
- */
-declare function createEngineQuerySource(options: EngineQuerySourceOptions): AnalysisQuerySource;
-/**
- * Convenience: wrap a storage engine + tenant ctx in a source and dispatch.
- * Equivalent to wrapping `createEngineQuerySource`, with omitted searchType
- * defaulted to web at this public helper boundary.
+ * Pad rows so every calendar day in `[startDate, endDate]` appears at least
+ * once. Existing dates keep all their rows (grouped timeseries safe).
  */
-declare function runAnalyzerWithEngine(deps: {
-  engine: StorageEngine;
-}, ctx: TenantCtx, params: AnalysisParams, registry: AnalyzerRegistry): Promise<AnalysisResult>;
-declare function queryRows<TRow = QueryRow>(source: AnalysisQuerySource, state: BuilderState): Promise<TRow[]>;
-export { AttachedTableMissingError, AttachedTableRunner, AttachedTableSourceOptions, CreateSqlQuerySourceOptions, ENGINE_QUERY_CAPABILITIES, EngineQuerySourceOptions, createAttachedTableSource, createEngineQuerySource, createSqlQuerySource, queryRows, runAnalyzerWithEngine };
+declare function padTimeseries<T extends DateRowShape = DateRowShape>(rows: readonly T[], options: PadTimeseriesOptions<T>): T[];
+export { AnalysisPeriod, ComparisonMode, ComparisonPeriod, PadTimeseriesOptions, ResolveWindowOptions, ResolvedWindow, WindowPreset, comparisonOf, defaultEndDate, padTimeseries, periodOf, resolveWindow };

package/dist/_chunks/resolver.mjs

@@ -2,10 +2,10 @@ import { SCHEMAS, drizzleSchema } from "./schema.mjs";
 import { enumeratePartitions } from "./parquet-plan.mjs";
 import { escapeLike } from "../sql-fragments.mjs";
 import "../planner.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING } from "./schema2.mjs";
 import { UnresolvableDatasetError, buildLogicalComparisonPlan, buildLogicalPlan, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
 import { PgDialect, pgTable, varchar } from "drizzle-orm/pg-core";
 import { normalizeUrl } from "gscdump/normalize";
+import { DEFAULT_PARTITION_KEY_ENCODING } from "@gscdump/lakehouse/schema";
 import { sql } from "drizzle-orm";
 import { and as and$1, gte as gte$1, impressions } from "gscdump/query";
 const DIMENSION_SURFACES = {
@@ -392,110 +392,6 @@ function createResolverAdapter(config) {
 		compile: config.compile
 	};
 }
-const pgDialect = new PgDialect();
-function withTenantCols(tableName, baseTable) {
-	const t = pgTable(tableName, {
-		site_id: varchar("site_id").notNull(),
-		search_type: varchar("search_type").notNull()
-	});
-	return {
-		...baseTable,
-		site_id: t.site_id,
-		search_type: t.search_type
-	};
-}
-const icebergSchema = {
-	pages: withTenantCols("pages", drizzleSchema.pages),
-	queries: withTenantCols("queries", drizzleSchema.queries),
-	countries: withTenantCols("countries", drizzleSchema.countries),
-	page_queries: withTenantCols("page_queries", drizzleSchema.page_queries),
-	dates: withTenantCols("dates", drizzleSchema.dates),
-	search_appearance: withTenantCols("search_appearance", drizzleSchema.search_appearance),
-	search_appearance_pages: withTenantCols("search_appearance_pages", drizzleSchema.search_appearance_pages),
-	search_appearance_queries: withTenantCols("search_appearance_queries", drizzleSchema.search_appearance_queries),
-	search_appearance_page_queries: withTenantCols("search_appearance_page_queries", drizzleSchema.search_appearance_page_queries),
-	hourly_pages: withTenantCols("hourly_pages", drizzleSchema.hourly_pages)
-};
-function compilePg(query) {
-	const compiled = pgDialect.sqlToQuery(query);
-	return {
-		sql: compiled.sql,
-		params: compiled.params
-	};
-}
-const PG_BASE_CONFIG = {
-	schema: drizzleSchema,
-	datasetToTableKey: {
-		pages: "pages",
-		queries: "queries",
-		page_queries: "page_queries",
-		countries: "countries",
-		dates: "dates",
-		search_appearance: "search_appearance",
-		search_appearance_pages: "search_appearance_pages",
-		search_appearance_queries: "search_appearance_queries",
-		search_appearance_page_queries: "search_appearance_page_queries",
-		hourly_pages: "hourly_pages"
-	},
-	metricCast: "DOUBLE",
-	regexPredicate: (expr, pattern, negate) => negate ? sql`NOT regexp_matches(${expr}, ${pattern})` : sql`regexp_matches(${expr}, ${pattern})`,
-	urlToPathExpr: (col) => `CASE WHEN ${col} LIKE 'http%' THEN COALESCE(NULLIF(regexp_replace(${col}, '^https?://[^/]+', ''), ''), '/') ELSE ${col} END`,
-	includeSiteId: false,
-	compile: compilePg,
-	capabilities: {
-		regex: true,
-		comparisonJoin: true,
-		windowTotals: true
-	}
-};
-const pgResolverAdapter = createResolverAdapter({
-	...PG_BASE_CONFIG,
-	tableLabel: "pg-resolver-adapter"
-});
-function createParquetResolverAdapter(options = {}) {
-	return createResolverAdapter({
-		...PG_BASE_CONFIG,
-		tableLabel: "parquet-resolver-adapter",
-		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
-		tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`),
-		queryDimTableRef: () => sql.raw("read_parquet({{QUERY_DIM}}, union_by_name = true) AS \"query_dim\"")
-	});
-}
-function createIcebergResolverAdapter(options = {}) {
-	return createResolverAdapter({
-		...PG_BASE_CONFIG,
-		schema: icebergSchema,
-		includeSiteId: true,
-		includeSearchType: true,
-		tableLabel: "iceberg-resolver-adapter",
-		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
-		tableRef: (tk) => sql.raw(`"${tk}"`),
-		queryDimTableRef: () => sql.raw("\"query_dim\"")
-	});
-}
-function createR2SqlResolverAdapter(options = {}) {
-	const adapter = createResolverAdapter({
-		...PG_BASE_CONFIG,
-		schema: icebergSchema,
-		includeSiteId: true,
-		includeSearchType: true,
-		tableLabel: "r2-sql-resolver-adapter",
-		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
-		capabilities: {
-			regex: false,
-			comparisonJoin: false,
-			windowTotals: false
-		},
-		tableRef: (tk) => sql.raw(`"${tk}"`),
-		queryDimTableRef: () => sql.raw("\"query_dim\"")
-	});
-	if ((options.partitionKeyEncoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "int") return adapter;
-	return {
-		...adapter,
-		siteIdColRef: (tk) => sql`CONCAT(${adapter.siteIdColRef(tk)}, '')`,
-		searchTypeColRef: (tk) => sql`CONCAT(${adapter.searchTypeColRef(tk)}, '')`
-	};
-}
 const ALLOWED_FILTER_DIMS = /* @__PURE__ */ new Set(["date", "queryCanonical"]);
 function planCoveredByCanonicalRollup(plan) {
 	if (plan.dataset !== "queries") return false;
@@ -973,6 +869,110 @@ function matchesTopLevelPage(row) {
 	for (let index = 0; index < path.length; index++) if (path.charCodeAt(index) === 47 && ++slashes > 1) return false;
 	return true;
 }
+const pgDialect = new PgDialect();
+function withTenantCols(tableName, baseTable) {
+	const t = pgTable(tableName, {
+		site_id: varchar("site_id").notNull(),
+		search_type: varchar("search_type").notNull()
+	});
+	return {
+		...baseTable,
+		site_id: t.site_id,
+		search_type: t.search_type
+	};
+}
+const icebergSchema = {
+	pages: withTenantCols("pages", drizzleSchema.pages),
+	queries: withTenantCols("queries", drizzleSchema.queries),
+	countries: withTenantCols("countries", drizzleSchema.countries),
+	page_queries: withTenantCols("page_queries", drizzleSchema.page_queries),
+	dates: withTenantCols("dates", drizzleSchema.dates),
+	search_appearance: withTenantCols("search_appearance", drizzleSchema.search_appearance),
+	search_appearance_pages: withTenantCols("search_appearance_pages", drizzleSchema.search_appearance_pages),
+	search_appearance_queries: withTenantCols("search_appearance_queries", drizzleSchema.search_appearance_queries),
+	search_appearance_page_queries: withTenantCols("search_appearance_page_queries", drizzleSchema.search_appearance_page_queries),
+	hourly_pages: withTenantCols("hourly_pages", drizzleSchema.hourly_pages)
+};
+function compilePg(query) {
+	const compiled = pgDialect.sqlToQuery(query);
+	return {
+		sql: compiled.sql,
+		params: compiled.params
+	};
+}
+const PG_BASE_CONFIG = {
+	schema: drizzleSchema,
+	datasetToTableKey: {
+		pages: "pages",
+		queries: "queries",
+		page_queries: "page_queries",
+		countries: "countries",
+		dates: "dates",
+		search_appearance: "search_appearance",
+		search_appearance_pages: "search_appearance_pages",
+		search_appearance_queries: "search_appearance_queries",
+		search_appearance_page_queries: "search_appearance_page_queries",
+		hourly_pages: "hourly_pages"
+	},
+	metricCast: "DOUBLE",
+	regexPredicate: (expr, pattern, negate) => negate ? sql`NOT regexp_matches(${expr}, ${pattern})` : sql`regexp_matches(${expr}, ${pattern})`,
+	urlToPathExpr: (col) => `CASE WHEN ${col} LIKE 'http%' THEN COALESCE(NULLIF(regexp_replace(${col}, '^https?://[^/]+', ''), ''), '/') ELSE ${col} END`,
+	includeSiteId: false,
+	compile: compilePg,
+	capabilities: {
+		regex: true,
+		comparisonJoin: true,
+		windowTotals: true
+	}
+};
+const pgResolverAdapter = createResolverAdapter({
+	...PG_BASE_CONFIG,
+	tableLabel: "pg-resolver-adapter"
+});
+function createParquetResolverAdapter(options = {}) {
+	return createResolverAdapter({
+		...PG_BASE_CONFIG,
+		tableLabel: "parquet-resolver-adapter",
+		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
+		tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`),
+		queryDimTableRef: () => sql.raw("read_parquet({{QUERY_DIM}}, union_by_name = true) AS \"query_dim\"")
+	});
+}
+function createIcebergResolverAdapter(options = {}) {
+	return createResolverAdapter({
+		...PG_BASE_CONFIG,
+		schema: icebergSchema,
+		includeSiteId: true,
+		includeSearchType: true,
+		tableLabel: "iceberg-resolver-adapter",
+		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
+		tableRef: (tk) => sql.raw(`"${tk}"`),
+		queryDimTableRef: () => sql.raw("\"query_dim\"")
+	});
+}
+function createR2SqlResolverAdapter(options = {}) {
+	const adapter = createResolverAdapter({
+		...PG_BASE_CONFIG,
+		schema: icebergSchema,
+		includeSiteId: true,
+		includeSearchType: true,
+		tableLabel: "r2-sql-resolver-adapter",
+		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
+		capabilities: {
+			regex: false,
+			comparisonJoin: false,
+			windowTotals: false
+		},
+		tableRef: (tk) => sql.raw(`"${tk}"`),
+		queryDimTableRef: () => sql.raw("\"query_dim\"")
+	});
+	if ((options.partitionKeyEncoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "int") return adapter;
+	return {
+		...adapter,
+		siteIdColRef: (tk) => sql`CONCAT(${adapter.siteIdColRef(tk)}, '')`,
+		searchTypeColRef: (tk) => sql`CONCAT(${adapter.searchTypeColRef(tk)}, '')`
+	};
+}
 var QuerySourceCoverageError = class extends Error {
 	fallback;
 	name = "QuerySourceCoverageError";

package/dist/_chunks/schema2.mjs

@@ -1,5 +1,6 @@
 import { SCHEMAS, TABLE_METADATA } from "./schema.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, defineIcebergDataset } from "@gscdump/lakehouse";
+import { defineIcebergDataset } from "@gscdump/lakehouse";
+import { DEFAULT_PARTITION_KEY_ENCODING, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1 } from "@gscdump/lakehouse/schema";
 function mapColumnType(t) {
 	switch (t) {
 		case "VARCHAR": return "STRING";

package/dist/_chunks/sink.d.mts

@@ -2,7 +2,8 @@ import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mj
 import { EngineError } from "./errors.mjs";
 import { icebergCreateTable } from "./libs/icebird.mjs";
 import { Result } from "gscdump/result";
-import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergColumn, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1, QueryProfiler } from "@gscdump/lakehouse";
+import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, QueryProfiler } from "@gscdump/lakehouse";
+import { IcebergColumn, IcebergPartitionField, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1 } from "@gscdump/lakehouse/schema";
 import { TableName } from "@gscdump/contracts";
 /** The 9 fact tables that exist as global Iceberg tables. */
 type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;

package/dist/_chunks/snapshot.d.mts

@@ -1,3 +1,29 @@
+import { ParquetCodec, QueryExecutor, Row } from "./storage.mjs";
+interface DuckDBHandle {
+  query: (sql: string, params?: unknown[]) => Promise<Row[]>;
+  registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;
+  copyFileToBuffer: (name: string) => Promise<Uint8Array>;
+  dropFiles: (names: string[]) => Promise<void>;
+  /**
+   * Returns a unique path suitable for `COPY TO '…'` + `copyFileToBuffer`.
+   * In Node this is an absolute path under `os.tmpdir()` so DuckDB doesn't
+   * litter the CWD; in browsers/Workers it's a plain virtual-FS name.
+   */
+  makeTempPath: (ext: string) => string;
+}
+interface DuckDBFactory {
+  getDuckDB: () => Promise<DuckDBHandle>;
+}
+interface DuckDBReadOptions {
+  /**
+   * Maximum non-URI parquet reads held in flight while registering DuckDB
+   * buffers. Defaults to 16; callers handling unusually large files can lower
+   * it to trade latency for peak JS memory.
+   */
+  bufferReadConcurrency?: number;
+}
+declare function createDuckDBCodec(factory: DuckDBFactory, options?: DuckDBReadOptions): ParquetCodec;
+declare function createDuckDBExecutor(factory: DuckDBFactory, options?: DuckDBReadOptions): QueryExecutor;
 /**
  * Describes a hot/cold snapshot set. Produced by the snapshot builder,
  * consumed by `attachSnapshotIndex`. Filenames are derived from `cold`
@@ -11,4 +37,4 @@ interface SnapshotIndex {
   hot: boolean;
   hotDays: number;
 }
-export { SnapshotIndex };
+export { DuckDBFactory, DuckDBHandle, SnapshotIndex, createDuckDBCodec, createDuckDBExecutor };

package/dist/adapters/node.d.mts

@@ -1,8 +1,12 @@
 import { DataSource, StorageEngine } from "../_chunks/storage.mjs";
-import { NodeDuckDBOptions, createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
-import { SnapshotIndex } from "../_chunks/snapshot.mjs";
+import { DuckDBHandle, SnapshotIndex } from "../_chunks/snapshot.mjs";
 import { SearchType } from "gscdump/query";
 import { Row, TableName } from "@gscdump/contracts";
+interface NodeDuckDBOptions {
+  verbose?: boolean;
+}
+declare function createNodeDuckDBHandle(opts?: NodeDuckDBOptions): DuckDBHandle;
+declare function resetNodeDuckDB(): void;
 interface NodeHarnessOptions {
   dataDir: string;
   /** Tenant user id. Defaults to `'local'` for single-user CLI installs. */