@gscdump/engine 1.0.1 → 1.0.3

package/dist/adapters/node.mjs

@@ -1,10 +1,101 @@
 import { engineErrors } from "../errors.mjs";
 import { createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "../_chunks/engine.mjs";
-import { createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
+import { arrowToRows } from "../arrow-utils.mjs";
 import { createFilesystemDataSource, createFilesystemManifestStore } from "./filesystem.mjs";
+import { createRequire } from "node:module";
 import { err, ok, unwrapResult } from "gscdump/result";
-import path from "node:path";
+import process from "node:process";
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path, { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
 import { encodeSiteId } from "gscdump/tenant";
+const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
+let singleton = null;
+let singletonOpts = null;
+function bundles() {
+	return {
+		mvp: {
+			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm"),
+			mainWorker: null
+		},
+		eh: {
+			mainModule: require_.resolve("@duckdb/duckdb-wasm/dist/duckdb-eh.wasm"),
+			mainWorker: null
+		}
+	};
+}
+async function initialize(opts) {
+	const logger = opts.verbose ? new ConsoleLogger() : new VoidLogger();
+	const db = await createDuckDB(bundles(), logger, NODE_RUNTIME);
+	await db.instantiate();
+	return {
+		db,
+		conn: db.connect()
+	};
+}
+function getSingleton(opts) {
+	if (!singleton) {
+		singleton = initialize(opts);
+		singletonOpts = opts;
+	}
+	return singleton;
+}
+function createNodeDuckDBHandle(opts = {}) {
+	if (singleton && opts.verbose !== void 0 && opts.verbose !== (singletonOpts?.verbose ?? false)) console.warn(`[gscdump] createNodeDuckDBHandle: ignoring verbose=${opts.verbose} — a shared DuckDB instance was already initialized with verbose=${singletonOpts?.verbose ?? false}. Call resetNodeDuckDB() before re-initializing to change it.`);
+	getSingleton(opts);
+	return {
+		async query(sql, params) {
+			const { conn } = await getSingleton(opts);
+			if (!params || params.length === 0) return arrowToRows(conn.query(sql));
+			const stmt = conn.prepare(sql);
+			try {
+				return arrowToRows(stmt.query(...params));
+			} finally {
+				stmt.close();
+			}
+		},
+		async registerFileBuffer(name, bytes) {
+			const { db } = await getSingleton(opts);
+			db.registerFileBuffer(name, bytes);
+		},
+		async copyFileToBuffer(name) {
+			const { db } = await getSingleton(opts);
+			return db.copyFileToBuffer(name);
+		},
+		async dropFiles(names) {
+			const { db } = await getSingleton(opts);
+			for (const name of names) {
+				try {
+					db.dropFile(name);
+				} catch (error) {
+					const message = error instanceof Error ? error.message : String(error);
+					if (!/not found|does not exist|unknown file/i.test(message)) throw error;
+				}
+				try {
+					unlinkSync(name);
+				} catch (error) {
+					if (error.code !== "ENOENT") throw error;
+				}
+			}
+		},
+		makeTempPath(ext) {
+			return join(tmpdir(), `gscdump-${Math.random().toString(36).slice(2, 10)}.${ext}`);
+		}
+	};
+}
+function resetNodeDuckDB() {
+	const pending = singleton;
+	singleton = null;
+	singletonOpts = null;
+	pending?.then(({ db, conn }) => {
+		conn.close();
+		db.reset();
+	}).catch((err) => {
+		console.warn("[gscdump] resetNodeDuckDB: failed to release DuckDB instance", err);
+	});
+}
 function createNodeHarness(opts) {
 	const dataDir = opts.dataDir;
 	const userId = opts.userId ?? "local";

package/dist/entities.d.mts

@@ -1,5 +1,6 @@
 import { DataSource } from "./_chunks/storage.mjs";
-import { ScheduleState } from "./schedule.mjs";
+import { ScheduleState } from "./_chunks/schedule.mjs";
+import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
 import { TenantCtx } from "@gscdump/contracts";
 interface QueryDimRecord {
   query: string;
@@ -18,8 +19,6 @@ interface QueryDimMeta {
   normalizerVersion: number;
   intentVersion: number;
 }
-declare function queryDimParquetKey(ctx: TenantCtx): string;
-declare function queryDimMetaKey(ctx: TenantCtx): string;
 /**
  * Injected derivation. `engine` never imports `@gscdump/analysis`; the host
  * passes `normalizeQuery` / `classifyIntentCode` (e.g. `encodeIntent ∘
@@ -109,32 +108,11 @@ interface InspectionHistoryShard {
   /** Records persisted in this batch. */
   records: InspectionRecord[];
 }
-declare function inspectionIndexKey(ctx: TenantCtx): string;
-declare function emptyTypesKey(ctx: TenantCtx): string;
-declare function inspectionParquetKey(ctx: TenantCtx): string;
-/** Directory prefix holding a tenant's immutable inspection-event parquets. */
-declare function inspectionEventsPrefix(ctx: TenantCtx): string;
-/**
- * Object key for one immutable inspection-event batch, partitioned by the
- * `YYYY-MM` of the records' `inspectedAt`. The `batchId` is caller-supplied so
- * a job retry re-writes the SAME key (idempotent whole-file overwrite).
- */
-declare function inspectionEventKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
-/** Compacted latest-per-url base produced by `compactInspections`. */
-declare function inspectionBaseKey(ctx: TenantCtx): string;
 /**
  * Directory prefix for a month's history shards. Each shard is a UUID-keyed
  * blob under this prefix; `appendHistory` writes one per call, `loadHistory`
  * lists + concatenates.
  */
-declare function inspectionHistoryPrefix(ctx: TenantCtx, yearMonth: string): string;
-declare function inspectionHistoryShardKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
-/**
- * Stable URL hash used as the index key. Short, URL-safe, deterministic.
- * Uses a 64-bit FNV-1a; collisions vanishingly unlikely at the scales we
- * care about (≤100k URLs/site).
- */
-declare function hashUrl(url: string): string;
 /**
  * Row shape for the inspections parquet sidecar. Caller-side schema for
  * `materialize` — D1 is the source of truth in the 2026-05-19 redesign, so
@@ -332,11 +310,6 @@ interface SitemapHistoryDoc {
   capturedAt: string;
   record: SitemapRecord;
 }
-declare function sitemapIndexKey(ctx: TenantCtx): string;
-declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
-declare function sitemapUrlsIndexPrefix(ctx: TenantCtx): string;
-declare function sitemapUrlsIndexKey(ctx: TenantCtx, feedpathHash: string): string;
-declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, date: string): string;
 /** Parsed URL entry from a sitemap XML. */
 interface ParsedUrl {
   loc: string;
@@ -410,11 +383,6 @@ interface DateRange {
 interface LoadUrlsOptions {
   includeRemoved?: boolean;
 }
-/**
- * Hash a URL list for change detection. Sorts then folds via FNV-1a so it's
- * deterministic, locale-free, and cheap on Workers.
- */
-declare function hashUrlList(urls: readonly ParsedUrl[]): string;
 interface SitemapStore {
   /**
    * Persist a snapshot run. Updates the index + writes one immutable
@@ -484,7 +452,6 @@ interface IndexingMetadataIndex {
   version: 1;
   records: Record<string, IndexingMetadataRecord>;
 }
-declare function indexingMetadataIndexKey(ctx: TenantCtx): string;
 interface IndexingMetadataStore {
   writeBatch: (ctx: TenantCtx, records: readonly IndexingMetadataRecord[]) => Promise<void>;
   loadIndex: (ctx: TenantCtx) => Promise<IndexingMetadataIndex>;
@@ -514,4 +481,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CompactUrlsOptions, CompactUrlsResult, CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { CompactUrlsOptions, CompactUrlsResult, CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };

package/dist/entities.mjs

@@ -1,2 +1,4 @@
-import { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
-export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import "./adapters/hyparquet.mjs";
+import { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore } from "./_chunks/entities.mjs";
+export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };

package/dist/entity-keys.d.mts

@@ -0,0 +1,27 @@
+import { TenantCtx } from "@gscdump/contracts";
+declare function inspectionIndexKey(ctx: TenantCtx): string;
+declare function emptyTypesKey(ctx: TenantCtx): string;
+declare function inspectionParquetKey(ctx: TenantCtx): string;
+declare function inspectionEventsPrefix(ctx: TenantCtx): string;
+declare function inspectionEventKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
+declare function inspectionBaseKey(ctx: TenantCtx): string;
+declare function inspectionHistoryPrefix(ctx: TenantCtx, yearMonth: string): string;
+declare function inspectionHistoryShardKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
+/** Stable, URL-safe FNV-1a hash used for entity keys. */
+declare function hashUrl(url: string): string;
+declare function sitemapIndexKey(ctx: TenantCtx): string;
+declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
+declare function sitemapUrlsPrefix(ctx: TenantCtx): string;
+declare function sitemapUrlsIndexPrefix(ctx: TenantCtx): string;
+declare function sitemapUrlsIndexKey(ctx: TenantCtx, feedpathHash: string): string;
+declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, date: string): string;
+/** Hash a URL list for deterministic change detection. */
+declare function hashUrlList(urls: readonly {
+  loc: string;
+}[]): string;
+/** Hash sorted URL strings as though joined by a newline, without allocating the join. */
+declare function hashSortedUrlList(locs: readonly string[]): string;
+declare function indexingMetadataIndexKey(ctx: TenantCtx): string;
+declare function queryDimParquetKey(ctx: TenantCtx): string;
+declare function queryDimMetaKey(ctx: TenantCtx): string;
+export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };

package/dist/entity-keys.mjs

@@ -0,0 +1,93 @@
+function tenantEntityPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities` : `u_${ctx.userId}/entities`;
+}
+function inspectionIndexKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/inspections/index.json`;
+}
+function emptyTypesKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/empty-types.json`;
+}
+function inspectionParquetKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/inspections/index.parquet`;
+}
+function inspectionEventsPrefix(ctx) {
+	return `${tenantEntityPrefix(ctx)}/inspections/events`;
+}
+function inspectionEventKey(ctx, yearMonth, batchId) {
+	return `${inspectionEventsPrefix(ctx)}/${yearMonth}/${batchId}.parquet`;
+}
+function inspectionBaseKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/inspections/base.parquet`;
+}
+function inspectionHistoryPrefix(ctx, yearMonth) {
+	return `${tenantEntityPrefix(ctx)}/inspections/history/${yearMonth}`;
+}
+function inspectionHistoryShardKey(ctx, yearMonth, batchId) {
+	return `${inspectionHistoryPrefix(ctx, yearMonth)}/${batchId}.json`;
+}
+function hashUrl(url) {
+	let hi = 2166136261;
+	let lo = 3421674724;
+	for (let i = 0; i < url.length; i++) {
+		const c = url.charCodeAt(i);
+		lo ^= c;
+		const loMul = Math.imul(lo, 435) >>> 0;
+		const carry = Math.floor(lo * 435 / 4294967296);
+		const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
+		lo = loMul;
+		hi = hiMul;
+	}
+	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
+}
+function sitemapIndexKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/sitemaps/index.json`;
+}
+function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
+	return `${tenantEntityPrefix(ctx)}/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
+}
+function sitemapUrlsPrefix(ctx) {
+	return `${tenantEntityPrefix(ctx)}/sitemaps/urls`;
+}
+function sitemapUrlsIndexPrefix(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/by-feed`;
+}
+function sitemapUrlsIndexKey(ctx, feedpathHash) {
+	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
+}
+function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
+	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
+}
+function hashUrlList(urls) {
+	return hashSortedUrlList(urls.map((url) => url.loc).sort());
+}
+function hashSortedUrlList(locs) {
+	let hi = 2166136261;
+	let lo = 3421674724;
+	for (let locIndex = 0; locIndex < locs.length; locIndex++) {
+		const loc = locs[locIndex];
+		const length = loc.length + (locIndex < locs.length - 1 ? 1 : 0);
+		for (let i = 0; i < length; i++) {
+			const c = i < loc.length ? loc.charCodeAt(i) : 10;
+			lo ^= c;
+			const loMul = Math.imul(lo, 435) >>> 0;
+			const carry = Math.floor(lo * 435 / 4294967296);
+			const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
+			lo = loMul;
+			hi = hiMul;
+		}
+	}
+	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
+}
+function indexingMetadataIndexKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/indexing/index.json`;
+}
+function queryDimPrefix(ctx) {
+	return `${tenantEntityPrefix(ctx)}/query_dim`;
+}
+function queryDimParquetKey(ctx) {
+	return `${queryDimPrefix(ctx)}/index.parquet`;
+}
+function queryDimMetaKey(ctx) {
+	return `${queryDimPrefix(ctx)}/index.json`;
+}
+export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };

package/dist/iceberg/index.mjs

@@ -1,11 +1,11 @@
 import { engineErrors } from "../errors.mjs";
 import { TABLE_METADATA } from "../_chunks/schema.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
 import { icebergCreateTable } from "../_chunks/libs/icebird.mjs";
 import { err, ok } from "gscdump/result";
 import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables as dropIcebergTables$1, resolveIcebergDataFiles } from "@gscdump/lakehouse";
 import { icebergAppendRetrying } from "@gscdump/lakehouse/unsafe-raw";
-function icebergSchemaFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING$1) {
+function icebergSchemaFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	return {
 		"type": "struct",
 		"schema-id": 0,
@@ -17,7 +17,7 @@ function icebergSchemaFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING$1) {
 		}))
 	};
 }
-function icebergPartitionSpecFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING$1) {
+function icebergPartitionSpecFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	const fields = icebergSchemasFor(encoding)[table].columns;
 	const fieldId = (name) => {
 		const col = fields.find((c) => c.name === name);
@@ -34,7 +34,7 @@ function icebergPartitionSpecFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODIN
 		}))
 	};
 }
-function icebergSortOrderFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING$1) {
+function icebergSortOrderFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	const fields = icebergSchemasFor(encoding)[table].columns;
 	const fieldId = (name) => {
 		const col = fields.find((c) => c.name === name);
@@ -51,7 +51,7 @@ function icebergSortOrderFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING$1)
 		}))
 	};
 }
-async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = DEFAULT_PARTITION_KEY_ENCODING$1) {
+async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	const results = [];
 	for (const table of tables) await icebergCreateTable({
 		catalog: conn.catalog,
@@ -76,7 +76,7 @@ async function dropIcebergTables(conn, tables) {
 	}));
 }
 async function listIcebergDataFiles(conn, opts) {
-	const matches = (opts.encoding ?? DEFAULT_PARTITION_KEY_ENCODING$1) === "string" ? [{
+	const matches = (opts.encoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "string" ? [{
 		field: "site_id",
 		value: opts.siteId,
 		encoding: "string"
@@ -178,7 +178,7 @@ function toRecords(slice, rows, encoding) {
 }
 function createIcebergAppendSink(options) {
 	let connection;
-	const encoding = options.encoding ?? DEFAULT_PARTITION_KEY_ENCODING$1;
+	const encoding = options.encoding ?? DEFAULT_PARTITION_KEY_ENCODING;
 	const buffers = /* @__PURE__ */ new Map();
 	function connect() {
 		connection ??= connectIcebergCatalog(options.catalog, options.connect);

package/dist/index.d.mts

@@ -1,19 +1,11 @@
-import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions } from "./_chunks/storage.mjs";
-import { DuckDBFactory, DuckDBHandle, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
-import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
-import { SnapshotIndex } from "./_chunks/snapshot.mjs";
-import { EngineError, EngineErrorKind, engineErrorToException, engineErrors, isEngineError } from "./_chunks/errors.mjs";
-import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
+import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryProfiler, QueryResult, QuerySpan, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult } from "./_chunks/storage.mjs";
+import { DuckDBFactory, DuckDBHandle, SnapshotIndex, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/snapshot.mjs";
+import { EngineError } from "./_chunks/errors.mjs";
+import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./_chunks/schedule.mjs";
 import { IcebergTableName, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./_chunks/sink.mjs";
-import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
-import { FILES_PLACEHOLDER, ResolvedQuery, resolveParquetSQL, substituteNamedFiles } from "./_chunks/planner.mjs";
-import { createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter } from "./_chunks/pg-adapter.mjs";
-import { rebuildDailyFromHourly } from "./rollups.mjs";
-import { ENGINE_QUERY_CAPABILITIES, createSqlQuerySource } from "./_chunks/index.mjs";
-import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
 import { Result } from "gscdump/result";
 import { SearchType as SearchType$1 } from "gscdump/query";
-import { Grain as Grain$1, Row as Row$1, TableName as TableName$1, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
+import { Row as Row$1, TableName as TableName$1, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
 interface R2ObjectMetadata {
   etag: string;
 }
@@ -98,99 +90,6 @@ declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-interface IngestAccumulatorEngine {
-  writeDay: (scope: TenantCtx & {
-    table: TableName$1;
-    date: string;
-    searchType?: SearchType;
-  }, rows: Row$1[]) => Promise<void>;
-  /**
-   * Routed when the accumulator's `ctx.grain === 'hour'`. Same scope shape as
-   * `writeDay`; `date` is the PT calendar day, rows carry `hour` + `date`.
-   * Optional so hosts that never opt into hourly need not implement it.
-   */
-  writeHour?: (scope: TenantCtx & {
-    table: TableName$1;
-    date: string;
-    searchType?: SearchType;
-  }, rows: Row$1[]) => Promise<void>;
-  setSyncState: (scope: TenantCtx & {
-    table: TableName$1;
-    date: string;
-    searchType?: SearchType;
-  }, state: 'done' | 'failed', info?: {
-    error?: string;
-  }) => Promise<void>;
-}
-interface IngestAccumulatorCtx {
-  userId: string | number;
-  siteId: string;
-  searchType?: SearchType;
-  /**
-   * Temporal granularity for this accumulator. `'day'` (default) routes
-   * flushed buckets to `engine.writeDay`. `'hour'` routes to
-   * `engine.writeHour` and requires the engine implementation to be set.
-   */
-  grain?: Grain$1;
-}
-interface IngestAccumulatorHooks {
-  /**
-   * Called once per (table, date) when the job must abandon in-memory rows
-   *  (overflow or `hasMore` continuation). Host queues a forced re-sync from
-   *  the source. Return true iff a recovery job was actually queued.
-   */
-  onRecover: (table: TableName$1, date: string) => Promise<boolean>;
-  /**
-   * Called when an engine.writeDay fails or recovery itself errors. Host
-   *  logs to its error sink (e.g. `r2_write_errors` D1 table).
-   */
-  onWriteError: (info: {
-    table: TableName$1 | null;
-    date: string | null;
-    error: unknown;
-  }) => Promise<void>;
-  /**
-   * Called after a successful writeDay for a (table, date). Host typically
-   *  busts the manifest cache here so the next read sees the new parquet.
-   */
-  onWritten?: (info: {
-    table: TableName$1;
-    date: string;
-    rowCount: number;
-  }) => void | Promise<void>;
-  /**
-   * Called once at end of `finalize`, only when at least one (table, date)
-   *  actually landed. Host queues rollup rebuild + compaction.
-   */
-  onJobComplete?: (info: {
-    flushed: number;
-    rowsWritten: number;
-  }) => Promise<void>;
-}
-interface FinalizeOptions {
-  /**
-   * The GSC `hasMore` flag for the whole job. When true, in-memory buckets
-   *  only reflect this job's slice; we re-queue forced single-day re-syncs
-   *  via `onRecover` so R2 stays authoritative.
-   */
-  hasMore: boolean;
-}
-interface FinalizeResult {
-  flushed: number;
-  recovered: number;
-  failed: number;
-  rowsWritten: number;
-}
-interface IngestAccumulator {
-  push: (table: TableName$1, rows: readonly GscApiRow[]) => boolean;
-  finalize: (opts: FinalizeOptions) => Promise<FinalizeResult>;
-}
-interface CreateIngestAccumulatorOptions extends RowAccumulatorOptions {
-  engine: IngestAccumulatorEngine;
-  ctx: IngestAccumulatorCtx;
-  hooks: IngestAccumulatorHooks;
-}
-declare function createIngestAccumulator(opts: CreateIngestAccumulatorOptions): IngestAccumulator;
 declare function dayPartition(date: string): string;
 /**
  * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
@@ -289,4 +188,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, type CreateR2ManifestStoreOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, type R2ManifestBucketLike, type R2ManifestEvent, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type SnapshotIndex, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2ManifestStore, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export { type CodecCtx, type CompactionThresholds, type CompactionTier, type CreateR2ManifestStoreOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type FileSetRef, type GcCtx, type Grain, type InMemorySink, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, type R2ManifestBucketLike, type R2ManifestEvent, ROW_LIMIT_R2, type Row, type RunSQLOptions, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type SnapshotIndex, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, coerceRow, coerceRows, collectSpans, createDuckDBCodec, createDuckDBExecutor, createInMemorySink, createQueryProfiler, createR2ManifestStore, createStorageEngine, dayPartition, fixedPolicy, getDateWeight, getTableTier, getTablesForTier, hourPartition, inferLegacyTier, inferSearchType, inspectionPolicy, objectKey, parseEnabledSearchTypes, sitemapPolicy, validateEnabledSearchTypes, validateEnabledSearchTypesResult };