@gscdump/engine 0.24.1 → 0.25.1

package/dist/_chunks/storage.d.mts

@@ -11,11 +11,6 @@ interface CompactionThresholds {
   d7?: number;
   d30?: number;
 }
-declare const RAW_DAILY_COMPACT_THRESHOLD = 7;
-declare function countRawDailies(entries: ReadonlyArray<{
-  tier?: string | null;
-  partition: string;
-}>): number;
 declare function enumeratePartitions(startDate: string, endDate: string): string[];
 /**
  * Split manifest entries into the set worth reading (`kept`) and the set whose
@@ -59,12 +54,6 @@ declare function dedupeOverlappingTiers(entries: ManifestEntry[], queryRange?: {
   start: string;
   end: string;
 }): ManifestEntry[];
-/**
- * Default `searchType` for entries written before the field landed and for
- * sync paths that don't request a specific type. GSC's own default; the
- * vast majority of stored data is web-search.
- */
-declare const DEFAULT_SEARCH_TYPE: SearchType;
 interface WriteCtx extends TenantCtx {
   table: TableName;
   date?: string;
@@ -152,18 +141,6 @@ interface ManifestEntry {
    */
   searchType?: SearchType;
 }
-/**
- * Resolve the search type for an entry, defaulting legacy entries to `web`.
- * Use this anywhere code needs to bucket entries by searchType.
- */
-declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType;
-/**
- * Infer the tier for an entry that pre-dates the `tier` field. Daily files
- * are `raw`; monthly files are `d30`. Anything else (already migrated, or
- * a partition shape we haven't seen) returns undefined and the caller must
- * decide how to handle it.
- */
-declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
 interface ListLiveFilter {
   userId: string;
   siteId?: string;
@@ -493,6 +470,29 @@ interface StorageEngine {
    */
   runSQL: (opts: RunSQLOptions) => Promise<QueryResult>;
   compactTiered: (ctx: WriteCtx, thresholds?: CompactionThresholds) => Promise<void>;
+  /**
+   * Write-time half of the manifest tier invariant: retire every live entry
+   * whose every covered day is already served by a finer-or-newer live entry.
+   *
+   * `compactTiered` retires the inputs it merges, but cannot retire a coarse
+   * partition that outlived the finer files it should have superseded (a D1→R2
+   * backfill writing coarse directly, a re-sync landing fresh dailies after a
+   * month already rolled up). Those stale overlaps make the query resolver
+   * union the same dates twice. Subsumption is evaluated per searchType, over
+   * the full live set (so a `web` monthly never cancels a `discover` weekly),
+   * then the subsumed set is retired via the manifest's `registerVersions([], …)`
+   * primitive — atomic, no inserts. Safe by construction: it only drops files
+   * whose days are already covered, so no data is lost.
+   *
+   * Reads and retires through the engine's own manifest store, so it is
+   * read-your-writes-consistent with the `compactTiered` that precedes it.
+   * Returns audit counters. Hosts running a cached manifest store must bust
+   * their cache afterwards — the engine has no knowledge of host-side caching.
+   */
+  reconcileSubsumed: (ctx: WriteCtx) => Promise<{
+    retired: number;
+    partitions: string[];
+  }>;
   gcOrphans: (ctx: GcCtx, graceMs: number) => Promise<{
     deleted: number;
   }>;
@@ -541,13 +541,4 @@ interface EngineOptions {
   executor: QueryExecutor;
   now?: () => number;
 }
-declare function dayPartition(date: string): string;
-/**
- * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
- * per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
- * idempotency across retries. Names sort lexically alongside daily ones but
- * never collide because of the `hourly/` prefix.
- */
-declare function hourPartition(date: string): string;
-declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
-export { CodecCtx, CompactionThresholds, CompactionTier, DEFAULT_SEARCH_TYPE, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, RAW_DAILY_COMPACT_THRESHOLD, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, countRawDailies, dayPartition, dedupeOverlappingTiers, enumeratePartitions, hourPartition, inferLegacyTier, inferSearchType, objectKey, splitOverlappingTiers };
+export { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, dedupeOverlappingTiers, enumeratePartitions, splitOverlappingTiers };

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/storage.mjs";
+import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";

package/dist/adapters/node.mjs

@@ -1,8 +1,8 @@
 import { createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "../_chunks/engine.mjs";
 import { createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { createFilesystemDataSource, createFilesystemManifestStore } from "./filesystem.mjs";
+import { encodeSiteId } from "gscdump";
 import path from "node:path";
-import { encodeSiteId } from "gscdump/tenant";
 function createNodeHarness(opts) {
 	const dataDir = opts.dataDir;
 	const userId = opts.userId ?? "local";

package/dist/adapters/r2-manifest.mjs

@@ -1,4 +1,4 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/storage.mjs";
+import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 const CAS_BACKOFF_BASE_MS = 5;
 const CAS_BACKOFF_CAP_MS = 250;

package/dist/compaction-public.d.mts

@@ -0,0 +1,15 @@
+import { dedupeOverlappingTiers, splitOverlappingTiers } from "./_chunks/storage.mjs";
+/**
+ * Host-policy predicate: true once a table's live raw-daily count crosses the
+ * engine's daily→weekly compaction gate. Wraps the internal threshold so hosts
+ * decide "is compaction due?" without importing the constant or the counter.
+ *
+ * Pure — pass the entries the host already fetched (typically via its own cached
+ * manifest store, so the hot-path check stays on the host's cache rather than
+ * forcing an uncached read through the engine).
+ */
+declare function isRawDailyCompactionDue(entries: ReadonlyArray<{
+  tier?: string | null;
+  partition: string;
+}>): boolean;
+export { dedupeOverlappingTiers, isRawDailyCompactionDue, splitOverlappingTiers };

package/dist/compaction-public.mjs

@@ -0,0 +1,5 @@
+import { countRawDailies, dedupeOverlappingTiers, splitOverlappingTiers } from "./_chunks/compaction.mjs";
+function isRawDailyCompactionDue(entries) {
+	return countRawDailies(entries) > 7;
+}
+export { dedupeOverlappingTiers, isRawDailyCompactionDue, splitOverlappingTiers };

package/dist/iceberg/index.d.mts

@@ -0,0 +1,12 @@
+import { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+type IcebergAppendSink = Sink;
+/**
+ * Create an `IcebergAppendSink` over the R2 Data Catalog.
+ *
+ * `emit` buffers; `close()` commits one `icebergAppend()` per table touched.
+ * The catalog connection (REST context + signed S3 resolver) is established
+ * lazily on the first flush and reused — a sink that is opened and closed
+ * with no rows never touches the network.
+ */
+declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
+export { type CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/iceberg/index.mjs

@@ -0,0 +1,269 @@
+import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "icebird";
+const ICEBERG_TYPE_MAP = {
+	STRING: "string",
+	INT: "int",
+	LONG: "long",
+	DOUBLE: "double",
+	DATE: "date"
+};
+function icebergSchemaFor(table) {
+	return {
+		"type": "struct",
+		"schema-id": 0,
+		"fields": ICEBERG_SCHEMAS[table].columns.map((col) => ({
+			id: col.fieldId,
+			name: col.name,
+			required: col.required,
+			type: ICEBERG_TYPE_MAP[col.type]
+		}))
+	};
+}
+function icebergPartitionSpecFor(table) {
+	const fields = ICEBERG_SCHEMAS[table].columns;
+	const fieldId = (name) => {
+		const col = fields.find((c) => c.name === name);
+		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"spec-id": 0,
+		"fields": ICEBERG_PARTITION_SPEC.map((p, i) => ({
+			"source-id": fieldId(p.sourceColumn),
+			"field-id": 1e3 + i,
+			"name": p.name,
+			"transform": p.transform
+		}))
+	};
+}
+async function connectIcebergCatalog(config) {
+	return {
+		catalog: await restCatalogConnect({
+			url: config.catalogUri,
+			warehouse: config.warehouse,
+			requestInit: { headers: { Authorization: `Bearer ${config.catalogToken}` } }
+		}),
+		resolver: s3SignedResolver({
+			accessKeyId: config.s3.accessKeyId,
+			secretAccessKey: config.s3.secretAccessKey,
+			region: config.s3.region ?? "auto",
+			endpoint: config.s3.endpoint,
+			pathStyle: true
+		}),
+		namespace: config.namespace
+	};
+}
+function isCommitRateLimited(err) {
+	if (err && typeof err === "object" && err.status === 429) return true;
+	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+	return msg.includes("429") || msg.includes("too many commits") || msg.includes("rate limit");
+}
+function defaultCommitSleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function icebergAppendRetrying(args, options = {}) {
+	const maxAttempts = options.maxAttempts ?? 6;
+	const baseDelayMs = options.baseDelayMs ?? 1e3;
+	const maxDelayMs = options.maxDelayMs ?? 2e4;
+	const sleep = options.sleep ?? defaultCommitSleep;
+	const random = options.random ?? Math.random;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		const err = await icebergAppend(args).then(() => void 0, (e) => e);
+		if (err === void 0) return;
+		if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
+		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
+		await sleep(Math.floor(random() * ceiling));
+	}
+}
+async function ensureIcebergNamespace(conn) {
+	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
+}
+async function createIcebergTables(conn, tables = ICEBERG_TABLES) {
+	const results = [];
+	for (const table of tables) await icebergCreateTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		schema: icebergSchemaFor(table),
+		partitionSpec: icebergPartitionSpecFor(table)
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (e) => results.push({
+		table,
+		ok: false,
+		error: String(e)
+	}));
+	return results;
+}
+async function listIcebergTables(conn) {
+	return restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name).sort(), () => []);
+}
+function monthsInRange(range) {
+	const [sy, sm] = range.start.split("-").map(Number);
+	const [ey, em] = range.end.split("-").map(Number);
+	const out = [];
+	let y = sy;
+	let m = sm;
+	while (y < ey || y === ey && m <= em) {
+		out.push(`${y}-${String(m).padStart(2, "0")}`);
+		m++;
+		if (m > 12) {
+			m = 1;
+			y++;
+		}
+	}
+	return out;
+}
+function monthsSinceEpoch(ym) {
+	const [y, m] = ym.split("-").map(Number);
+	return (y - 1970) * 12 + (m - 1);
+}
+function stripBucket(filePath) {
+	if (!filePath.startsWith("s3://")) return filePath;
+	const rest = filePath.slice(5);
+	const slash = rest.indexOf("/");
+	return slash >= 0 ? rest.slice(slash + 1) : rest;
+}
+async function listIcebergDataFiles(conn, opts) {
+	const { metadata } = await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table: opts.table
+	});
+	if (metadata["current-snapshot-id"] == null) return [];
+	const wantedMonths = new Set(monthsInRange(opts.range).map(monthsSinceEpoch));
+	const manifests = await icebergManifests({
+		metadata,
+		resolver: conn.resolver
+	});
+	const out = [];
+	for (const m of manifests) for (const entry of m.entries) {
+		if (entry.status === 2) continue;
+		const df = entry.data_file;
+		if (df.content !== 0) continue;
+		const part = df.partition;
+		if (part.site_id !== opts.siteId) continue;
+		if (part.search_type !== opts.searchType) continue;
+		const month = part.date_month;
+		if (typeof month !== "number" || !wantedMonths.has(month)) continue;
+		out.push({
+			filePath: df.file_path,
+			objectKey: stripBucket(df.file_path),
+			bytes: Number(df.file_size_in_bytes),
+			rowCount: Number(df.record_count)
+		});
+	}
+	return out;
+}
+async function dropIcebergTables(conn, tables) {
+	const targets = tables ?? await restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name), () => []);
+	const results = [];
+	for (const table of targets) await icebergDropTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		purgeRequested: true
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (e) => results.push({
+		table,
+		ok: false,
+		error: String(e)
+	}));
+	return results;
+}
+const DAY_MILLIS = 864e5;
+function toIcebergDate(value) {
+	if (typeof value === "string") {
+		const ms = Date.parse(`${value}T00:00:00Z`);
+		if (Number.isNaN(ms)) throw new TypeError(`toIcebergDate: invalid date string '${value}'`);
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	if (value instanceof Date) {
+		const ms = value.getTime();
+		if (Number.isNaN(ms)) throw new TypeError("toIcebergDate: invalid Date (NaN)");
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	return value;
+}
+function coerceJsonSafe(value) {
+	if (typeof value === "bigint") return Number(value);
+	return value;
+}
+function toRecords(slice, rows) {
+	const siteId = slice.ctx.siteId ?? "";
+	return rows.map((row) => {
+		const out = {};
+		for (const k in row) out[k] = coerceJsonSafe(row[k]);
+		out.date = toIcebergDate(out.date);
+		out.site_id = siteId;
+		out.search_type = slice.searchType;
+		return out;
+	});
+}
+function createIcebergAppendSink(options) {
+	let connection;
+	const buffers = /* @__PURE__ */ new Map();
+	function connect() {
+		connection ??= connectIcebergCatalog(options.catalog);
+		return connection;
+	}
+	return {
+		capabilities: { appendOnly: true },
+		async emit(slice, rows) {
+			if (rows.length === 0) return { rowCount: 0 };
+			const records = toRecords(slice, rows);
+			const buffer = buffers.get(slice.table);
+			if (buffer) for (let i = 0; i < records.length; i++) buffer.push(records[i]);
+			else buffers.set(slice.table, records);
+			return { rowCount: records.length };
+		},
+		async close() {
+			const flushed = [];
+			const failed = [];
+			if (buffers.size === 0) return {
+				flushed,
+				failed
+			};
+			const conn = await connect().then((c) => c, (err) => {
+				connection = void 0;
+				return { error: String(err) };
+			});
+			if ("error" in conn) {
+				for (const [table, records] of buffers) if (records.length > 0) failed.push({
+					table,
+					error: conn.error
+				});
+				buffers.clear();
+				return {
+					flushed,
+					failed
+				};
+			}
+			for (const [table, records] of buffers) {
+				if (records.length === 0) continue;
+				await icebergAppendRetrying({
+					catalog: conn.catalog,
+					namespace: conn.namespace,
+					table,
+					resolver: conn.resolver,
+					records
+				}, options.commitRetry).then(() => {
+					flushed.push(table);
+				}, (err) => {
+					failed.push({
+						table,
+						error: String(err)
+					});
+				});
+			}
+			buffers.clear();
+			return {
+				flushed,
+				failed
+			};
+		}
+	};
+}
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/index.d.mts

@@ -1,36 +1,20 @@
-import { CodecCtx, CompactionThresholds, CompactionTier, DEFAULT_SEARCH_TYPE, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, RAW_DAILY_COMPACT_THRESHOLD, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, countRawDailies, dayPartition, dedupeOverlappingTiers, enumeratePartitions, hourPartition, inferLegacyTier, inferSearchType, objectKey, splitOverlappingTiers } from "./_chunks/storage.mjs";
+import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions } from "./_chunks/storage.mjs";
 import { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
 import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
 import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
-import { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, listIcebergDataFiles, listIcebergTables } from "./_chunks/sink.mjs";
+import { IcebergTableName, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./_chunks/sink.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { FILES_PLACEHOLDER, ResolvedQuery, resolveParquetSQL, substituteNamedFiles } from "./_chunks/planner.mjs";
 import { createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "./_chunks/pg-adapter.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { ENGINE_QUERY_CAPABILITIES, createSqlQuerySource } from "./_chunks/index.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
-import { Grain as Grain$1, Row as Row$1, TableName as TableName$1 } from "@gscdump/contracts";
+import { Grain as Grain$1, Row as Row$1, TableName as TableName$1, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
+import { SearchType as SearchType$1 } from "gscdump/query";
 declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-interface GcDeps {
-  dataSource: DataSource;
-  manifestStore: ManifestStore;
-}
-interface GcOptions {
-  userId?: string;
-  siteId?: string;
-  /**
-   * Retention for hourly partitions (`hourly/{date}`) in milliseconds.
-   * Defaults to 90 days; entries with `createdAt < now - hourlyRetentionMs`
-   * are retired and their bytes deleted alongside ordinary orphan sweeping.
-   */
-  hourlyRetentionMs?: number;
-}
-declare function gcOrphansImpl(deps: GcDeps, now: number, graceMs: number, opts?: GcOptions): Promise<{
-  deleted: number;
-}>;
 interface IngestAccumulatorEngine {
   writeDay: (scope: TenantCtx & {
     table: TableName$1;
@@ -125,16 +109,33 @@ interface CreateIngestAccumulatorOptions extends RowAccumulatorOptions {
 }
 declare function createNoopIngestAccumulator(): IngestAccumulator;
 declare function createIngestAccumulator(opts: CreateIngestAccumulatorOptions): IngestAccumulator;
-type IcebergAppendSink = Sink;
+declare function dayPartition(date: string): string;
+/**
+ * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
+ * per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
+ * idempotency across retries. Names sort lexically alongside daily ones but
+ * never collide because of the `hourly/` prefix.
+ */
+declare function hourPartition(date: string): string;
+/**
+ * Default `searchType` for entries written before the field landed and for
+ * sync paths that don't request a specific type. GSC's own default; the
+ * vast majority of stored data is web-search.
+ */
+declare const DEFAULT_SEARCH_TYPE: SearchType$1;
+declare function objectKey(ctx: TenantCtx$1, table: TableName$1, partition: string, version: number, searchType?: SearchType$1): string;
+/**
+ * Resolve the search type for an entry, defaulting legacy entries to `web`.
+ * Use this anywhere code needs to bucket entries by searchType.
+ */
+declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType$1;
 /**
- * Create an `IcebergAppendSink` over the R2 Data Catalog.
- *
- * `emit` buffers; `close()` commits one `icebergAppend()` per table touched.
- * The catalog connection (REST context + signed S3 resolver) is established
- * lazily on the first flush and reused — a sink that is opened and closed
- * with no rows never touches the network.
+ * Infer the tier for an entry that pre-dates the `tier` field. Daily files
+ * are `raw`; monthly files are `d30`. Anything else (already migrated, or
+ * a partition shape we haven't seen) returns undefined and the caller must
+ * decide how to handle it.
  */
-declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
+declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
 /** A row as stored by the fake — data columns plus the injected identity columns. */
 type StoredRow = Row & {
   site_id: string;
@@ -181,4 +182,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CommitRetryOptions, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListIcebergDataFilesOptions, type ListLiveFilter, type LocalIcebergSinkOptions, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergResolverAdapter, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergDataFiles, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };