@gscdump/engine 0.24.0 → 0.25.0

package/dist/_chunks/engine.mjs

@@ -1,9 +1,9 @@
+import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
 import { SCHEMAS, currentSchemaVersion, dedupeByNaturalKey } from "./schema.mjs";
-import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./storage.mjs";
 import { compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, substituteNamedFiles } from "./parquet-plan.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
-import { normalizeUrl } from "gscdump/normalize";
+import { normalizeUrl } from "gscdump";
 async function encodeBytes(db, table, rows) {
 	const inName = db.makeTempPath("json");
 	const outName = db.makeTempPath("parquet");
@@ -601,4 +601,4 @@ function createStorageEngine(opts) {
 		readObject: (key) => dataSource.read(key)
 	};
 }
-export { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine, gcOrphansImpl };
+export { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine };

package/dist/_chunks/parquet-plan.mjs

@@ -1,8 +1,8 @@
+import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./layout.mjs";
 import { currentSchemaVersion, dimensionToColumn } from "./schema.mjs";
-import { dayPartition, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, weekPartition } from "./storage.mjs";
 import { METRIC_EXPR, escapeLike, topLevelPagePredicateSql } from "../sql-fragments.mjs";
-import { MS_PER_DAY } from "gscdump";
 import { buildLogicalPlan } from "gscdump/query/plan";
+import { MS_PER_DAY } from "gscdump";
 const DAILY_PARTITION_RE = /^daily\/(\d{4}-\d{2}-\d{2})$/;
 const WEEKLY_PARTITION_RE = /^weekly\/(\d{4}-\d{2}-\d{2})$/;
 const MONTHLY_PARTITION_RE = /^monthly\/(\d{4}-\d{2})$/;
@@ -12,10 +12,6 @@ const DEFAULT_THRESHOLDS = {
 	d7: 30,
 	d30: 90
 };
-const RAW_DAILY_COMPACT_THRESHOLD = 7;
-function countRawDailies(entries) {
-	return entries.filter((e) => e.tier === "raw" || e.tier == null && e.partition.startsWith("daily/")).length;
-}
 const PENDING_WINDOW_DAYS = 4;
 const STAGES = [
 	{
@@ -381,4 +377,4 @@ function substituteNamedFiles(sql, sets) {
 	for (const [name, keys] of Object.entries(sets)) out = out.replace(new RegExp(`\\{\\{${name}\\}\\}`, "g"), fileList(keys));
 	return out;
 }
-export { FILES_PLACEHOLDER, RAW_DAILY_COMPACT_THRESHOLD, compactTieredImpl, compileLogicalQueryPlan, countRawDailies, dedupeOverlappingTiers, enumeratePartitions, resolveParquetSQL, splitOverlappingTiers, substituteNamedFiles };
+export { FILES_PLACEHOLDER, compactTieredImpl, compileLogicalQueryPlan, dedupeOverlappingTiers, enumeratePartitions, resolveParquetSQL, substituteNamedFiles };

package/dist/_chunks/resolver.mjs

@@ -2,9 +2,9 @@ import { SCHEMAS, drizzleSchema } from "./schema.mjs";
 import { enumeratePartitions } from "./parquet-plan.mjs";
 import { escapeLike } from "../sql-fragments.mjs";
 import "../planner.mjs";
-import { PgDialect, pgTable, varchar } from "drizzle-orm/pg-core";
 import { UnresolvableDatasetError, buildLogicalComparisonPlan, buildLogicalPlan, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
-import { normalizeUrl } from "gscdump/normalize";
+import { normalizeUrl } from "gscdump";
+import { PgDialect, pgTable, varchar } from "drizzle-orm/pg-core";
 import { sql } from "drizzle-orm";
 const DIMENSION_SURFACES = {
 	page: ["api", "stored"],

package/dist/_chunks/{iceberg-schema.mjs → schema2.mjs}

@@ -68,5 +68,12 @@ function icebergTableSpec(table) {
 	};
 }
 const ICEBERG_SCHEMAS = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t)]));
-new Set(ICEBERG_TABLES);
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, icebergTableSpec };
+const ICEBERG_TABLE_SET = new Set(ICEBERG_TABLES);
+function isIcebergTable(table) {
+	return ICEBERG_TABLE_SET.has(table);
+}
+function assertIcebergTable(table) {
+	if (!isIcebergTable(table)) throw new Error(`Unknown Iceberg table '${table}'. Expected one of: ${ICEBERG_TABLES.join(", ")}`);
+	return table;
+}
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, icebergTableSpec, isIcebergTable };

package/dist/_chunks/sink.d.mts

@@ -94,6 +94,16 @@ declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
 declare function icebergTableSpec(table: IcebergTableName): IcebergTableSpec;
 /** All Iceberg table specs, keyed by table name. */
 declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+/** True when `table` is one of the canonical {@link ICEBERG_TABLES}. */
+declare function isIcebergTable(table: string): table is IcebergTableName;
+/**
+ * Narrow an arbitrary table name to a canonical {@link IcebergTableName},
+ * throwing a clear error otherwise. Guards write paths that index
+ * `ICEBERG_SCHEMAS` (a `Record<IcebergTableName, …>`) — a non-canonical name
+ * silently yields `undefined` there, propagating a corrupt/empty spec into the
+ * Iceberg job instead of failing loudly.
+ */
+declare function assertIcebergTable(table: string): IcebergTableName;
 /** icebird's lowercase Iceberg primitive types (subset we use). */
 type IcebergPrimitiveType = 'string' | 'int' | 'long' | 'double' | 'date';
 /** A field in an icebird table `Schema`. */
@@ -370,4 +380,4 @@ interface LocalIcebergSinkOptions extends SinkOptions {
   /** S3-compatible warehouse location (POC: MinIO). */
   warehouse: string;
 }
-export { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, listIcebergDataFiles, listIcebergTables };
+export { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/_chunks/source.mjs

@@ -1,4 +1,4 @@
-import "./storage.mjs";
+import "./layout.mjs";
 import { assertDimensionsSupported, getFilterDimensions, pgResolverAdapter, resolveToSQL } from "./resolver.mjs";
 import { runAnalyzerFromSource } from "./dispatch.mjs";
 function coerceRow(row) {

package/dist/_chunks/storage.d.mts

@@ -11,60 +11,7 @@ interface CompactionThresholds {
   d7?: number;
   d30?: number;
 }
-declare const RAW_DAILY_COMPACT_THRESHOLD = 7;
-declare function countRawDailies(entries: ReadonlyArray<{
-  tier?: string | null;
-  partition: string;
-}>): number;
 declare function enumeratePartitions(startDate: string, endDate: string): string[];
-/**
- * Split manifest entries into the set worth reading (`kept`) and the set whose
- * every covered day is already served by a finer-or-newer live entry
- * (`subsumed`).
- *
- * Tiered compaction (daily→weekly→monthly→quarterly) is meant to retire its
- * inputs, but coarse files can outlive their finer counterparts: a D1→R2
- * backfill writes daily files that compact to monthly while a later re-sync
- * writes fresh daily/weekly for the same dates, and same-partition re-writes
- * leave a stale prior version live. All stay live, the resolver unions every
- * live tier whose partition intersects the range, and `union_by_name` sums the
- * overlap — impressions/clicks double-count.
- *
- * Entries are walked finest-tier-first, newest-first within a tier, so a
- * coarse or stale file is dropped only when every day it covers is already
- * claimed. Subsumption is evaluated per searchType — a `web` monthly never
- * cancels a `discover` weekly, they cover disjoint data. Partial
- * month-boundary overlap (a weekly straddling two months alongside a kept
- * monthly) still double-counts those boundary days — eliminating that needs
- * per-file date predicates in the SQL, tracked separately. Unrecognised
- * partition shapes (`hourly/`, sidecar keys) are always kept.
- *
- * `queryRange` clamps every entry's day-span to the window the caller will
- * actually read. This is required when `entries` came from a partition-
- * filtered `listLive` (`runSQL` enumerates only the partitions intersecting
- * the query): a `monthly/2026-04` whose Apr 27-30 falls past the query end
- * must not be judged "unsubsumed" just because `weekly/2026-04-27` wasn't
- * enumerated — those out-of-window days are SQL-filtered to nothing anyway.
- * Omit `queryRange` when `entries` is the full manifest (e.g. analysis-sources).
- */
-declare function splitOverlappingTiers(entries: ManifestEntry[], queryRange?: {
-  start: string;
-  end: string;
-}): {
-  kept: ManifestEntry[];
-  subsumed: ManifestEntry[];
-};
-/** Entries worth reading — see {@link splitOverlappingTiers}. */
-declare function dedupeOverlappingTiers(entries: ManifestEntry[], queryRange?: {
-  start: string;
-  end: string;
-}): ManifestEntry[];
-/**
- * Default `searchType` for entries written before the field landed and for
- * sync paths that don't request a specific type. GSC's own default; the
- * vast majority of stored data is web-search.
- */
-declare const DEFAULT_SEARCH_TYPE: SearchType;
 interface WriteCtx extends TenantCtx {
   table: TableName;
   date?: string;
@@ -152,18 +99,6 @@ interface ManifestEntry {
    */
   searchType?: SearchType;
 }
-/**
- * Resolve the search type for an entry, defaulting legacy entries to `web`.
- * Use this anywhere code needs to bucket entries by searchType.
- */
-declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType;
-/**
- * Infer the tier for an entry that pre-dates the `tier` field. Daily files
- * are `raw`; monthly files are `d30`. Anything else (already migrated, or
- * a partition shape we haven't seen) returns undefined and the caller must
- * decide how to handle it.
- */
-declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
 interface ListLiveFilter {
   userId: string;
   siteId?: string;
@@ -541,13 +476,4 @@ interface EngineOptions {
   executor: QueryExecutor;
   now?: () => number;
 }
-declare function dayPartition(date: string): string;
-/**
- * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
- * per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
- * idempotency across retries. Names sort lexically alongside daily ones but
- * never collide because of the `hourly/` prefix.
- */
-declare function hourPartition(date: string): string;
-declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
-export { CodecCtx, CompactionThresholds, CompactionTier, DEFAULT_SEARCH_TYPE, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, RAW_DAILY_COMPACT_THRESHOLD, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, countRawDailies, dayPartition, dedupeOverlappingTiers, enumeratePartitions, hourPartition, inferLegacyTier, inferSearchType, objectKey, splitOverlappingTiers };
+export { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, type Grain$1 as Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, type Row$1 as Row, RunSQLOptions, type SearchType$1 as SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, type TableName$1 as TableName, type TenantCtx$1 as TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions };

package/dist/adapters/filesystem.mjs

@@ -1,4 +1,4 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/storage.mjs";
+import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
 import { dirname, join, resolve } from "node:path";
 import { Buffer } from "node:buffer";
 import { randomBytes } from "node:crypto";

package/dist/adapters/node.mjs

@@ -1,8 +1,8 @@
 import { createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "../_chunks/engine.mjs";
 import { createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { createFilesystemDataSource, createFilesystemManifestStore } from "./filesystem.mjs";
+import { encodeSiteId } from "gscdump";
 import path from "node:path";
-import { encodeSiteId } from "gscdump/tenant";
 function createNodeHarness(opts) {
 	const dataDir = opts.dataDir;
 	const userId = opts.userId ?? "local";

package/dist/adapters/r2-manifest.mjs

@@ -1,4 +1,4 @@
-import { inferLegacyTier, inferSearchType } from "../_chunks/storage.mjs";
+import { inferLegacyTier, inferSearchType } from "../_chunks/layout.mjs";
 const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
 const CAS_BACKOFF_BASE_MS = 5;
 const CAS_BACKOFF_CAP_MS = 250;

package/dist/iceberg/index.d.mts

@@ -0,0 +1,12 @@
+import { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, Sink, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables } from "../_chunks/sink.mjs";
+type IcebergAppendSink = Sink;
+/**
+ * Create an `IcebergAppendSink` over the R2 Data Catalog.
+ *
+ * `emit` buffers; `close()` commits one `icebergAppend()` per table touched.
+ * The catalog connection (REST context + signed S3 resolver) is established
+ * lazily on the first flush and reused — a sink that is opened and closed
+ * with no rows never touches the network.
+ */
+declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
+export { type CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type ListIcebergDataFilesOptions, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/iceberg/index.mjs

@@ -0,0 +1,269 @@
+import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "icebird";
+const ICEBERG_TYPE_MAP = {
+	STRING: "string",
+	INT: "int",
+	LONG: "long",
+	DOUBLE: "double",
+	DATE: "date"
+};
+function icebergSchemaFor(table) {
+	return {
+		"type": "struct",
+		"schema-id": 0,
+		"fields": ICEBERG_SCHEMAS[table].columns.map((col) => ({
+			id: col.fieldId,
+			name: col.name,
+			required: col.required,
+			type: ICEBERG_TYPE_MAP[col.type]
+		}))
+	};
+}
+function icebergPartitionSpecFor(table) {
+	const fields = ICEBERG_SCHEMAS[table].columns;
+	const fieldId = (name) => {
+		const col = fields.find((c) => c.name === name);
+		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"spec-id": 0,
+		"fields": ICEBERG_PARTITION_SPEC.map((p, i) => ({
+			"source-id": fieldId(p.sourceColumn),
+			"field-id": 1e3 + i,
+			"name": p.name,
+			"transform": p.transform
+		}))
+	};
+}
+async function connectIcebergCatalog(config) {
+	return {
+		catalog: await restCatalogConnect({
+			url: config.catalogUri,
+			warehouse: config.warehouse,
+			requestInit: { headers: { Authorization: `Bearer ${config.catalogToken}` } }
+		}),
+		resolver: s3SignedResolver({
+			accessKeyId: config.s3.accessKeyId,
+			secretAccessKey: config.s3.secretAccessKey,
+			region: config.s3.region ?? "auto",
+			endpoint: config.s3.endpoint,
+			pathStyle: true
+		}),
+		namespace: config.namespace
+	};
+}
+function isCommitRateLimited(err) {
+	if (err && typeof err === "object" && err.status === 429) return true;
+	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
+	return msg.includes("429") || msg.includes("too many commits") || msg.includes("rate limit");
+}
+function defaultCommitSleep(ms) {
+	return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function icebergAppendRetrying(args, options = {}) {
+	const maxAttempts = options.maxAttempts ?? 6;
+	const baseDelayMs = options.baseDelayMs ?? 1e3;
+	const maxDelayMs = options.maxDelayMs ?? 2e4;
+	const sleep = options.sleep ?? defaultCommitSleep;
+	const random = options.random ?? Math.random;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		const err = await icebergAppend(args).then(() => void 0, (e) => e);
+		if (err === void 0) return;
+		if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
+		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
+		await sleep(Math.floor(random() * ceiling));
+	}
+}
+async function ensureIcebergNamespace(conn) {
+	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
+}
+async function createIcebergTables(conn, tables = ICEBERG_TABLES) {
+	const results = [];
+	for (const table of tables) await icebergCreateTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		schema: icebergSchemaFor(table),
+		partitionSpec: icebergPartitionSpecFor(table)
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (e) => results.push({
+		table,
+		ok: false,
+		error: String(e)
+	}));
+	return results;
+}
+async function listIcebergTables(conn) {
+	return restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name).sort(), () => []);
+}
+function monthsInRange(range) {
+	const [sy, sm] = range.start.split("-").map(Number);
+	const [ey, em] = range.end.split("-").map(Number);
+	const out = [];
+	let y = sy;
+	let m = sm;
+	while (y < ey || y === ey && m <= em) {
+		out.push(`${y}-${String(m).padStart(2, "0")}`);
+		m++;
+		if (m > 12) {
+			m = 1;
+			y++;
+		}
+	}
+	return out;
+}
+function monthsSinceEpoch(ym) {
+	const [y, m] = ym.split("-").map(Number);
+	return (y - 1970) * 12 + (m - 1);
+}
+function stripBucket(filePath) {
+	if (!filePath.startsWith("s3://")) return filePath;
+	const rest = filePath.slice(5);
+	const slash = rest.indexOf("/");
+	return slash >= 0 ? rest.slice(slash + 1) : rest;
+}
+async function listIcebergDataFiles(conn, opts) {
+	const { metadata } = await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table: opts.table
+	});
+	if (metadata["current-snapshot-id"] == null) return [];
+	const wantedMonths = new Set(monthsInRange(opts.range).map(monthsSinceEpoch));
+	const manifests = await icebergManifests({
+		metadata,
+		resolver: conn.resolver
+	});
+	const out = [];
+	for (const m of manifests) for (const entry of m.entries) {
+		if (entry.status === 2) continue;
+		const df = entry.data_file;
+		if (df.content !== 0) continue;
+		const part = df.partition;
+		if (part.site_id !== opts.siteId) continue;
+		if (part.search_type !== opts.searchType) continue;
+		const month = part.date_month;
+		if (typeof month !== "number" || !wantedMonths.has(month)) continue;
+		out.push({
+			filePath: df.file_path,
+			objectKey: stripBucket(df.file_path),
+			bytes: Number(df.file_size_in_bytes),
+			rowCount: Number(df.record_count)
+		});
+	}
+	return out;
+}
+async function dropIcebergTables(conn, tables) {
+	const targets = tables ?? await restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name), () => []);
+	const results = [];
+	for (const table of targets) await icebergDropTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		purgeRequested: true
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (e) => results.push({
+		table,
+		ok: false,
+		error: String(e)
+	}));
+	return results;
+}
+const DAY_MILLIS = 864e5;
+function toIcebergDate(value) {
+	if (typeof value === "string") {
+		const ms = Date.parse(`${value}T00:00:00Z`);
+		if (Number.isNaN(ms)) throw new TypeError(`toIcebergDate: invalid date string '${value}'`);
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	if (value instanceof Date) {
+		const ms = value.getTime();
+		if (Number.isNaN(ms)) throw new TypeError("toIcebergDate: invalid Date (NaN)");
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	return value;
+}
+function coerceJsonSafe(value) {
+	if (typeof value === "bigint") return Number(value);
+	return value;
+}
+function toRecords(slice, rows) {
+	const siteId = slice.ctx.siteId ?? "";
+	return rows.map((row) => {
+		const out = {};
+		for (const k in row) out[k] = coerceJsonSafe(row[k]);
+		out.date = toIcebergDate(out.date);
+		out.site_id = siteId;
+		out.search_type = slice.searchType;
+		return out;
+	});
+}
+function createIcebergAppendSink(options) {
+	let connection;
+	const buffers = /* @__PURE__ */ new Map();
+	function connect() {
+		connection ??= connectIcebergCatalog(options.catalog);
+		return connection;
+	}
+	return {
+		capabilities: { appendOnly: true },
+		async emit(slice, rows) {
+			if (rows.length === 0) return { rowCount: 0 };
+			const records = toRecords(slice, rows);
+			const buffer = buffers.get(slice.table);
+			if (buffer) for (let i = 0; i < records.length; i++) buffer.push(records[i]);
+			else buffers.set(slice.table, records);
+			return { rowCount: records.length };
+		},
+		async close() {
+			const flushed = [];
+			const failed = [];
+			if (buffers.size === 0) return {
+				flushed,
+				failed
+			};
+			const conn = await connect().then((c) => c, (err) => {
+				connection = void 0;
+				return { error: String(err) };
+			});
+			if ("error" in conn) {
+				for (const [table, records] of buffers) if (records.length > 0) failed.push({
+					table,
+					error: conn.error
+				});
+				buffers.clear();
+				return {
+					flushed,
+					failed
+				};
+			}
+			for (const [table, records] of buffers) {
+				if (records.length === 0) continue;
+				await icebergAppendRetrying({
+					catalog: conn.catalog,
+					namespace: conn.namespace,
+					table,
+					resolver: conn.resolver,
+					records
+				}, options.commitRetry).then(() => {
+					flushed.push(table);
+				}, (err) => {
+					failed.push({
+						table,
+						error: String(err)
+					});
+				});
+			}
+			buffers.clear();
+			return {
+				flushed,
+				failed
+			};
+		}
+	};
+}
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/index.d.mts

@@ -1,36 +1,20 @@
-import { CodecCtx, CompactionThresholds, CompactionTier, DEFAULT_SEARCH_TYPE, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, RAW_DAILY_COMPACT_THRESHOLD, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, countRawDailies, dayPartition, dedupeOverlappingTiers, enumeratePartitions, hourPartition, inferLegacyTier, inferSearchType, objectKey, splitOverlappingTiers } from "./_chunks/storage.mjs";
+import { CodecCtx, CompactionThresholds, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, Grain, ListLiveFilter, LockScope, ManifestEntry, ManifestPurgeResult, ManifestStore, ParquetCodec, PurgeFilter, PurgeResult, PurgeUrlsResult, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult, enumeratePartitions } from "./_chunks/storage.mjs";
 import { DuckDBFactory, DuckDBHandle, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor } from "./_chunks/duckdb.mjs";
 import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
 import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
-import { CommitRetryOptions, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, listIcebergDataFiles, listIcebergTables } from "./_chunks/sink.mjs";
+import { IcebergTableName, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./_chunks/sink.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { FILES_PLACEHOLDER, ResolvedQuery, resolveParquetSQL, substituteNamedFiles } from "./_chunks/planner.mjs";
 import { createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "./_chunks/pg-adapter.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { ENGINE_QUERY_CAPABILITIES, createSqlQuerySource } from "./_chunks/index.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
-import { Grain as Grain$1, Row as Row$1, TableName as TableName$1 } from "@gscdump/contracts";
+import { Grain as Grain$1, Row as Row$1, TableName as TableName$1, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
+import { SearchType as SearchType$1 } from "gscdump/query";
 declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-interface GcDeps {
-  dataSource: DataSource;
-  manifestStore: ManifestStore;
-}
-interface GcOptions {
-  userId?: string;
-  siteId?: string;
-  /**
-   * Retention for hourly partitions (`hourly/{date}`) in milliseconds.
-   * Defaults to 90 days; entries with `createdAt < now - hourlyRetentionMs`
-   * are retired and their bytes deleted alongside ordinary orphan sweeping.
-   */
-  hourlyRetentionMs?: number;
-}
-declare function gcOrphansImpl(deps: GcDeps, now: number, graceMs: number, opts?: GcOptions): Promise<{
-  deleted: number;
-}>;
 interface IngestAccumulatorEngine {
   writeDay: (scope: TenantCtx & {
     table: TableName$1;
@@ -125,16 +109,33 @@ interface CreateIngestAccumulatorOptions extends RowAccumulatorOptions {
 }
 declare function createNoopIngestAccumulator(): IngestAccumulator;
 declare function createIngestAccumulator(opts: CreateIngestAccumulatorOptions): IngestAccumulator;
-type IcebergAppendSink = Sink;
+declare function dayPartition(date: string): string;
+/**
+ * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
+ * per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
+ * idempotency across retries. Names sort lexically alongside daily ones but
+ * never collide because of the `hourly/` prefix.
+ */
+declare function hourPartition(date: string): string;
+/**
+ * Default `searchType` for entries written before the field landed and for
+ * sync paths that don't request a specific type. GSC's own default; the
+ * vast majority of stored data is web-search.
+ */
+declare const DEFAULT_SEARCH_TYPE: SearchType$1;
+declare function objectKey(ctx: TenantCtx$1, table: TableName$1, partition: string, version: number, searchType?: SearchType$1): string;
+/**
+ * Resolve the search type for an entry, defaulting legacy entries to `web`.
+ * Use this anywhere code needs to bucket entries by searchType.
+ */
+declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType$1;
 /**
- * Create an `IcebergAppendSink` over the R2 Data Catalog.
- *
- * `emit` buffers; `close()` commits one `icebergAppend()` per table touched.
- * The catalog connection (REST context + signed S3 resolver) is established
- * lazily on the first flush and reused — a sink that is opened and closed
- * with no rows never touches the network.
+ * Infer the tier for an entry that pre-dates the `tier` field. Daily files
+ * are `raw`; monthly files are `d30`. Anything else (already migrated, or
+ * a partition shape we haven't seen) returns undefined and the caller must
+ * decide how to handle it.
  */
-declare function createIcebergAppendSink(options: IcebergAppendSinkOptions): IcebergAppendSink;
+declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
 /** A row as stored by the fake — data columns plus the injected identity columns. */
 type StoredRow = Row & {
   site_id: string;
@@ -181,4 +182,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CommitRetryOptions, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, type IcebergAppendSink, type IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergTableName, type IcebergTableOpResult, type IcebergTableSpec, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListIcebergDataFilesOptions, type ListLiveFilter, type LocalIcebergSinkOptions, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergResolverAdapter, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergDataFiles, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };

package/dist/index.mjs

@@ -1,189 +1,14 @@
 import { ENGINE_QUERY_CAPABILITIES, coerceRow, coerceRows, createSqlQuerySource } from "./_chunks/source.mjs";
+import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./_chunks/layout.mjs";
 import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, page_queries, pages, queries } from "./_chunks/schema.mjs";
-import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./_chunks/storage.mjs";
-import { FILES_PLACEHOLDER, RAW_DAILY_COMPACT_THRESHOLD, countRawDailies, dedupeOverlappingTiers, enumeratePartitions, resolveParquetSQL, splitOverlappingTiers, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
+import { FILES_PLACEHOLDER, enumeratePartitions, resolveParquetSQL, substituteNamedFiles } from "./_chunks/parquet-plan.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
-import { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine, gcOrphansImpl } from "./_chunks/engine.mjs";
-import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, icebergTableSpec } from "./_chunks/iceberg-schema.mjs";
+import { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "./_chunks/engine.mjs";
 import { assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import "./planner.mjs";
 import { createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "./_chunks/resolver.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "icebird";
-const ICEBERG_TYPE_MAP = {
-	STRING: "string",
-	INT: "int",
-	LONG: "long",
-	DOUBLE: "double",
-	DATE: "date"
-};
-function icebergSchemaFor(table) {
-	return {
-		"type": "struct",
-		"schema-id": 0,
-		"fields": ICEBERG_SCHEMAS[table].columns.map((col) => ({
-			id: col.fieldId,
-			name: col.name,
-			required: col.required,
-			type: ICEBERG_TYPE_MAP[col.type]
-		}))
-	};
-}
-function icebergPartitionSpecFor(table) {
-	const fields = ICEBERG_SCHEMAS[table].columns;
-	const fieldId = (name) => {
-		const col = fields.find((c) => c.name === name);
-		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
-		return col.fieldId;
-	};
-	return {
-		"spec-id": 0,
-		"fields": ICEBERG_PARTITION_SPEC.map((p, i) => ({
-			"source-id": fieldId(p.sourceColumn),
-			"field-id": 1e3 + i,
-			"name": p.name,
-			"transform": p.transform
-		}))
-	};
-}
-async function connectIcebergCatalog(config) {
-	return {
-		catalog: await restCatalogConnect({
-			url: config.catalogUri,
-			warehouse: config.warehouse,
-			requestInit: { headers: { Authorization: `Bearer ${config.catalogToken}` } }
-		}),
-		resolver: s3SignedResolver({
-			accessKeyId: config.s3.accessKeyId,
-			secretAccessKey: config.s3.secretAccessKey,
-			region: config.s3.region ?? "auto",
-			endpoint: config.s3.endpoint,
-			pathStyle: true
-		}),
-		namespace: config.namespace
-	};
-}
-function isCommitRateLimited(err) {
-	if (err && typeof err === "object" && err.status === 429) return true;
-	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
-	return msg.includes("429") || msg.includes("too many commits") || msg.includes("rate limit");
-}
-function defaultCommitSleep(ms) {
-	return new Promise((resolve) => setTimeout(resolve, ms));
-}
-async function icebergAppendRetrying(args, options = {}) {
-	const maxAttempts = options.maxAttempts ?? 6;
-	const baseDelayMs = options.baseDelayMs ?? 1e3;
-	const maxDelayMs = options.maxDelayMs ?? 2e4;
-	const sleep = options.sleep ?? defaultCommitSleep;
-	const random = options.random ?? Math.random;
-	for (let attempt = 0; attempt < maxAttempts; attempt++) {
-		const err = await icebergAppend(args).then(() => void 0, (e) => e);
-		if (err === void 0) return;
-		if (!isCommitRateLimited(err) || attempt === maxAttempts - 1) throw err;
-		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
-		await sleep(Math.floor(random() * ceiling));
-	}
-}
-async function ensureIcebergNamespace(conn) {
-	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
-}
-async function createIcebergTables(conn, tables = ICEBERG_TABLES) {
-	const results = [];
-	for (const table of tables) await icebergCreateTable({
-		catalog: conn.catalog,
-		namespace: conn.namespace,
-		table,
-		schema: icebergSchemaFor(table),
-		partitionSpec: icebergPartitionSpecFor(table)
-	}).then(() => results.push({
-		table,
-		ok: true
-	}), (e) => results.push({
-		table,
-		ok: false,
-		error: String(e)
-	}));
-	return results;
-}
-async function listIcebergTables(conn) {
-	return restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name).sort(), () => []);
-}
-function monthsInRange(range) {
-	const [sy, sm] = range.start.split("-").map(Number);
-	const [ey, em] = range.end.split("-").map(Number);
-	const out = [];
-	let y = sy;
-	let m = sm;
-	while (y < ey || y === ey && m <= em) {
-		out.push(`${y}-${String(m).padStart(2, "0")}`);
-		m++;
-		if (m > 12) {
-			m = 1;
-			y++;
-		}
-	}
-	return out;
-}
-function monthsSinceEpoch(ym) {
-	const [y, m] = ym.split("-").map(Number);
-	return (y - 1970) * 12 + (m - 1);
-}
-function stripBucket(filePath) {
-	if (!filePath.startsWith("s3://")) return filePath;
-	const rest = filePath.slice(5);
-	const slash = rest.indexOf("/");
-	return slash >= 0 ? rest.slice(slash + 1) : rest;
-}
-async function listIcebergDataFiles(conn, opts) {
-	const { metadata } = await restCatalogLoadTable(conn.catalog, {
-		namespace: conn.namespace,
-		table: opts.table
-	});
-	if (metadata["current-snapshot-id"] == null) return [];
-	const wantedMonths = new Set(monthsInRange(opts.range).map(monthsSinceEpoch));
-	const manifests = await icebergManifests({
-		metadata,
-		resolver: conn.resolver
-	});
-	const out = [];
-	for (const m of manifests) for (const entry of m.entries) {
-		if (entry.status === 2) continue;
-		const df = entry.data_file;
-		if (df.content !== 0) continue;
-		const part = df.partition;
-		if (part.site_id !== opts.siteId) continue;
-		if (part.search_type !== opts.searchType) continue;
-		const month = part.date_month;
-		if (typeof month !== "number" || !wantedMonths.has(month)) continue;
-		out.push({
-			filePath: df.file_path,
-			objectKey: stripBucket(df.file_path),
-			bytes: Number(df.file_size_in_bytes),
-			rowCount: Number(df.record_count)
-		});
-	}
-	return out;
-}
-async function dropIcebergTables(conn, tables) {
-	const targets = tables ?? await restCatalogListTables(conn.catalog, { namespace: conn.namespace }).then((list) => list.map((t) => t.name), () => []);
-	const results = [];
-	for (const table of targets) await icebergDropTable({
-		catalog: conn.catalog,
-		namespace: conn.namespace,
-		table,
-		purgeRequested: true
-	}).then(() => results.push({
-		table,
-		ok: true
-	}), (e) => results.push({
-		table,
-		ok: false,
-		error: String(e)
-	}));
-	return results;
-}
 const NOOP_RESULT = {
 	flushed: 0,
 	recovered: 0,
@@ -299,99 +124,6 @@ function createIngestAccumulator(opts) {
 		}
 	};
 }
-const DAY_MILLIS = 864e5;
-function toIcebergDate(value) {
-	if (typeof value === "string") {
-		const ms = Date.parse(`${value}T00:00:00Z`);
-		if (Number.isNaN(ms)) throw new TypeError(`toIcebergDate: invalid date string '${value}'`);
-		return Math.floor(ms / DAY_MILLIS);
-	}
-	if (value instanceof Date) {
-		const ms = value.getTime();
-		if (Number.isNaN(ms)) throw new TypeError("toIcebergDate: invalid Date (NaN)");
-		return Math.floor(ms / DAY_MILLIS);
-	}
-	return value;
-}
-function coerceJsonSafe(value) {
-	if (typeof value === "bigint") return Number(value);
-	return value;
-}
-function toRecords(slice, rows) {
-	const siteId = slice.ctx.siteId ?? "";
-	return rows.map((row) => {
-		const out = {};
-		for (const k in row) out[k] = coerceJsonSafe(row[k]);
-		out.date = toIcebergDate(out.date);
-		out.site_id = siteId;
-		out.search_type = slice.searchType;
-		return out;
-	});
-}
-function createIcebergAppendSink(options) {
-	let connection;
-	const buffers = /* @__PURE__ */ new Map();
-	function connect() {
-		connection ??= connectIcebergCatalog(options.catalog);
-		return connection;
-	}
-	return {
-		capabilities: { appendOnly: true },
-		async emit(slice, rows) {
-			if (rows.length === 0) return { rowCount: 0 };
-			const records = toRecords(slice, rows);
-			const buffer = buffers.get(slice.table);
-			if (buffer) for (let i = 0; i < records.length; i++) buffer.push(records[i]);
-			else buffers.set(slice.table, records);
-			return { rowCount: records.length };
-		},
-		async close() {
-			const flushed = [];
-			const failed = [];
-			if (buffers.size === 0) return {
-				flushed,
-				failed
-			};
-			const conn = await connect().then((c) => c, (err) => {
-				connection = void 0;
-				return { error: String(err) };
-			});
-			if ("error" in conn) {
-				for (const [table, records] of buffers) if (records.length > 0) failed.push({
-					table,
-					error: conn.error
-				});
-				buffers.clear();
-				return {
-					flushed,
-					failed
-				};
-			}
-			for (const [table, records] of buffers) {
-				if (records.length === 0) continue;
-				await icebergAppendRetrying({
-					catalog: conn.catalog,
-					namespace: conn.namespace,
-					table,
-					resolver: conn.resolver,
-					records
-				}, options.commitRetry).then(() => {
-					flushed.push(table);
-				}, (err) => {
-					failed.push({
-						table,
-						error: String(err)
-					});
-				});
-			}
-			buffers.clear();
-			return {
-				flushed,
-				failed
-			};
-		}
-	};
-}
 const KEY_SEP = "\0";
 function partitionKey(slice) {
 	return [
@@ -544,4 +276,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
 const MIN_COUNTRY_IMPRESSIONS = 10;
 const MAX_SITEMAP_URLS_PER_SITE = 5e4;
 const MAX_TRACKED_URLS_PER_SITE = 2e5;
-export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, RAW_DAILY_COMPACT_THRESHOLD, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, connectIcebergCatalog, countRawDailies, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergAppendSink, createIcebergResolverAdapter, createIcebergTables, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dedupeOverlappingTiers, dimensionToColumn, drizzleSchema, dropIcebergTables, ensureIcebergNamespace, enumeratePartitions, fixedPolicy, formatLiteral, gcOrphansImpl, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, icebergAppendRetrying, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isCommitRateLimited, listIcebergDataFiles, listIcebergTables, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, splitOverlappingTiers, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };
+export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes };

package/dist/rollups.mjs

@@ -1,4 +1,4 @@
-import "./_chunks/storage.mjs";
+import "./_chunks/layout.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump";

package/dist/sink-node.d.mts

@@ -20,4 +20,4 @@ interface LocalIcebergSink extends Sink {
  * use this sink must skip when the stack is unreachable.
  */
 declare function createLocalIcebergSink(options: LocalIcebergSinkFullOptions): LocalIcebergSink;
-export { type LocalIcebergSink, type LocalIcebergSinkFullOptions, createLocalIcebergSink };
+export { type LocalIcebergSink, type LocalIcebergSinkFullOptions, type LocalIcebergSinkOptions, createLocalIcebergSink };

package/dist/sink-node.mjs

@@ -1,4 +1,4 @@
-import { ICEBERG_SCHEMAS } from "./_chunks/iceberg-schema.mjs";
+import { ICEBERG_SCHEMAS } from "./_chunks/schema2.mjs";
 import { execFile } from "node:child_process";
 import { dirname, join } from "node:path";
 import process from "node:process";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.24.0",
+  "version": "0.25.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -66,11 +66,6 @@
       "import": "./dist/sql-fragments.mjs",
       "default": "./dist/sql-fragments.mjs"
     },
-    "./schedule": {
-      "types": "./dist/schedule.d.mts",
-      "import": "./dist/schedule.mjs",
-      "default": "./dist/schedule.mjs"
-    },
     "./entities": {
       "types": "./dist/entities.d.mts",
       "import": "./dist/entities.mjs",
@@ -81,6 +76,11 @@
       "import": "./dist/rollups.mjs",
       "default": "./dist/rollups.mjs"
     },
+    "./iceberg": {
+      "types": "./dist/iceberg/index.d.mts",
+      "import": "./dist/iceberg/index.mjs",
+      "default": "./dist/iceberg/index.mjs"
+    },
     "./node": {
       "types": "./dist/adapters/node.d.mts",
       "import": "./dist/adapters/node.mjs",
@@ -180,8 +180,8 @@
     "drizzle-orm": "1.0.0-rc.3",
     "icebird": "^0.8.6",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.24.0",
-    "gscdump": "0.24.0"
+    "gscdump": "0.25.0",
+    "@gscdump/contracts": "0.25.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",

package/dist/_chunks/{storage.mjs → layout.mjs}

@@ -1,13 +1,4 @@
 import { MS_PER_DAY, toIsoDate } from "gscdump";
-const DEFAULT_SEARCH_TYPE = "web";
-function inferSearchType(entry) {
-	return entry.searchType ?? "web";
-}
-function inferLegacyTier(entry) {
-	if (entry.tier !== void 0) return entry.tier;
-	if (entry.partition.startsWith("daily/")) return "raw";
-	if (entry.partition.startsWith("monthly/")) return "d30";
-}
 function dayPartition(date) {
 	return `daily/${date}`;
 }
@@ -33,10 +24,19 @@ function quarterOfMonth(month) {
 	const [y, m] = month.split("-").map(Number);
 	return `${y}-Q${Math.floor((m - 1) / 3) + 1}`;
 }
+function tenantPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/` : `u_${ctx.userId}/`;
+}
+const DEFAULT_SEARCH_TYPE = "web";
 function objectKey(ctx, table, partition, version, searchType) {
 	return `${ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/${table}` : `u_${ctx.userId}/${table}`}/${searchType !== void 0 && searchType !== "web" ? `${searchType}/` : ""}${partition}__v${version}.parquet`;
 }
-function tenantPrefix(ctx) {
-	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/` : `u_${ctx.userId}/`;
+function inferSearchType(entry) {
+	return entry.searchType ?? "web";
+}
+function inferLegacyTier(entry) {
+	if (entry.tier !== void 0) return entry.tier;
+	if (entry.partition.startsWith("daily/")) return "raw";
+	if (entry.partition.startsWith("monthly/")) return "d30";
 }
 export { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, mondayOfWeek, monthPartition, objectKey, quarterOfMonth, quarterPartition, tenantPrefix, weekPartition };