@gscdump/engine 1.3.2 → 1.4.1

package/dist/index.mjs

@@ -1,646 +1,11 @@
-import { engineErrorToException, engineErrors } from "./errors.mjs";
-import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./_chunks/layout.mjs";
-import { matchesManifestEntryFilter, matchesSyncStateFilter, matchesWatermarkFilter } from "./_chunks/manifest-store-utils.mjs";
-import { coerceRow, coerceRows } from "./_chunks/coerce.mjs";
-import { MAX_DAY_BYTES, createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "./_chunks/engine.mjs";
-import { err, ok, unwrapResult } from "gscdump/result";
-const SHARD_RE = /^u_[^/]+\/manifest\/(?<siteId>[^/]+)\/(?<table>[^/]+)\/HEAD$/;
-const CAS_BACKOFF_BASE_MS = 5;
-const CAS_BACKOFF_CAP_MS = 250;
-const SHARD_IO_CONCURRENCY = 8;
-const R2_DELETE_BATCH_SIZE = 1e3;
-async function casBackoff(attempt) {
-	const ceil = Math.min(CAS_BACKOFF_CAP_MS, CAS_BACKOFF_BASE_MS * 2 ** attempt);
-	await new Promise((resolve) => setTimeout(resolve, Math.random() * ceil));
-}
-async function mapWithConcurrency(items, concurrency, fn) {
-	if (items.length === 0) return [];
-	const workerCount = Math.max(1, Math.min(items.length, Math.floor(concurrency)));
-	const results = Array.from({ length: items.length }, () => void 0);
-	let nextIndex = 0;
-	async function worker() {
-		while (true) {
-			const index = nextIndex++;
-			if (index >= items.length) return;
-			results[index] = await fn(items[index], index);
-		}
-	}
-	await Promise.all(Array.from({ length: workerCount }, worker));
-	return results;
-}
-function defaultSnapshotId() {
-	return `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`;
-}
-function shardPrefix(userId, siteId, table) {
-	return `u_${userId}/manifest/${siteId}/${table}/`;
-}
-function headKey(userId, siteId, table) {
-	return `${shardPrefix(userId, siteId, table)}HEAD`;
-}
-function snapshotKey(userId, siteId, table, snapshotId) {
-	return `${shardPrefix(userId, siteId, table)}v${snapshotId}.json`;
-}
-function emptySnapshot() {
-	return {
-		version: 1,
-		entries: [],
-		watermarks: [],
-		syncStates: []
-	};
-}
-function shardScopesFromEntries(entries) {
-	const out = /* @__PURE__ */ new Set();
-	for (const e of entries) {
-		if (e.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId; cross-site entries are unshardable");
-		out.add(`${e.siteId}\0${e.table}`);
-	}
-	return out;
-}
-function createR2ManifestStore(opts) {
-	const { bucket, userId } = opts;
-	const newSnapshotId = opts.newSnapshotId ?? defaultSnapshotId;
-	const now = opts.now ?? (() => Date.now());
-	const maxRetries = opts.maxRetries ?? 16;
-	const onEvent = opts.onEvent;
-	async function readShard(siteId, table) {
-		const head = await bucket.get(headKey(userId, siteId, table));
-		if (!head) return {
-			snapshot: emptySnapshot(),
-			headEtag: void 0
-		};
-		const snapshotId = (await head.text()).trim();
-		if (!snapshotId) return {
-			snapshot: emptySnapshot(),
-			headEtag: head.etag
-		};
-		const snap = await bucket.get(snapshotKey(userId, siteId, table, snapshotId));
-		if (!snap) return {
-			snapshot: emptySnapshot(),
-			headEtag: head.etag
-		};
-		const parsed = JSON.parse(await snap.text());
-		if (parsed.version !== 1) throw new Error(`unsupported manifest snapshot version: ${parsed.version}`);
-		return {
-			snapshot: parsed,
-			headEtag: head.etag
-		};
-	}
-	async function writeShardResult(siteId, table, snapshot, headEtag, attempt) {
-		const id = newSnapshotId();
-		const snapKey = snapshotKey(userId, siteId, table, id);
-		await bucket.put(snapKey, JSON.stringify(snapshot));
-		const conditional = headEtag ? { onlyIf: { etagMatches: headEtag } } : { onlyIf: { etagDoesNotMatch: "*" } };
-		return await bucket.put(headKey(userId, siteId, table), id, conditional) !== null ? ok(void 0) : err(engineErrors.manifestCasRoundLost(siteId, table, attempt));
-	}
-	async function mutateShardResult(siteId, table, mutate) {
-		let attempt = 0;
-		while (attempt < maxRetries) {
-			onEvent?.({
-				kind: "cas-attempt",
-				siteId,
-				table,
-				attempt
-			});
-			const { snapshot, headEtag } = await readShard(siteId, table);
-			await mutate(snapshot);
-			const round = await writeShardResult(siteId, table, snapshot, headEtag, attempt);
-			if (round.ok) {
-				onEvent?.({
-					kind: "cas-committed",
-					siteId,
-					table,
-					attempts: attempt + 1
-				});
-				return round;
-			}
-			onEvent?.({
-				kind: "cas-rejected",
-				siteId,
-				table,
-				attempt
-			});
-			attempt++;
-			if (attempt < maxRetries) await casBackoff(attempt);
-		}
-		return err(engineErrors.manifestCasExhausted(siteId, table, maxRetries));
-	}
-	async function mutateShard(siteId, table, mutate) {
-		return unwrapResult(await mutateShardResult(siteId, table, mutate), engineErrorToException);
-	}
-	async function listShards(siteId) {
-		const shards = [];
-		let cursor;
-		const prefix = siteId === void 0 ? `u_${userId}/manifest/` : `u_${userId}/manifest/${siteId}/`;
-		do {
-			const res = await bucket.list({
-				prefix,
-				cursor,
-				limit: 1e3
-			});
-			for (const obj of res.objects) {
-				const m = SHARD_RE.exec(obj.key);
-				if (m?.groups) shards.push({
-					siteId: m.groups.siteId,
-					table: m.groups.table
-				});
-			}
-			cursor = res.truncated ? res.cursor : void 0;
-		} while (cursor);
-		return shards;
-	}
-	async function shardsForFilter(filter) {
-		if (filter.siteId !== void 0 && filter.table !== void 0) return [{
-			siteId: filter.siteId,
-			table: filter.table
-		}];
-		return (await listShards(filter.siteId)).filter((s) => (filter.siteId === void 0 || s.siteId === filter.siteId) && (filter.table === void 0 || s.table === filter.table));
-	}
-	function assertScopedUser(got, op) {
-		if (got !== userId) throw new Error(`${op}: R2 manifest store is scoped to userId=${userId}, got ${got}`);
-	}
-	async function readEntriesAcrossShards(filter, includeRetired) {
-		assertScopedUser(filter.userId, includeRetired ? "listAll" : "listLive");
-		return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-			const { snapshot } = await readShard(siteId, table);
-			const entries = [];
-			for (const entry of snapshot.entries) {
-				if (!includeRetired && entry.retiredAt !== void 0) continue;
-				if (matchesManifestEntryFilter(entry, filter, { ignoreUserId: true })) entries.push(entry);
-			}
-			return entries;
-		})).flat();
-	}
-	function groupBySiteTable(entries) {
-		const out = /* @__PURE__ */ new Map();
-		for (const e of entries) {
-			const key = `${e.siteId}\0${e.table}`;
-			if (!out.has(key)) out.set(key, []);
-			out.get(key).push(e);
-		}
-		return out;
-	}
-	async function registerVersionsImpl(newEntries, superseding) {
-		if (newEntries.length === 0 && (!superseding || superseding.length === 0)) return;
-		const supersededAt = newEntries[0]?.createdAt ?? now();
-		const byShard = /* @__PURE__ */ new Map();
-		function bucket(entry, kind) {
-			assertScopedUser(entry.userId, "registerVersions");
-			if (entry.siteId === void 0) throw new Error("R2 manifest store requires entries to carry siteId");
-			const key = `${entry.siteId}\0${entry.table}`;
-			let bag = byShard.get(key);
-			if (!bag) {
-				bag = {
-					newEntries: [],
-					superseding: []
-				};
-				byShard.set(key, bag);
-			}
-			if (kind === "new") bag.newEntries.push(entry);
-			else bag.superseding.push(entry);
-		}
-		for (const e of newEntries) bucket(e, "new");
-		if (superseding) for (const e of superseding) bucket(e, "super");
-		await mapWithConcurrency([...byShard], SHARD_IO_CONCURRENCY, async ([shardKey, { newEntries: news, superseding: supers }]) => {
-			const [siteId, table] = shardKey.split("\0");
-			await mutateShard(siteId, table, (snap) => {
-				const byObjectKey = new Map(snap.entries.map((e) => [e.objectKey, e]));
-				for (const s of supers) {
-					const existing = byObjectKey.get(s.objectKey);
-					if (existing && existing.retiredAt === void 0) byObjectKey.set(s.objectKey, {
-						...existing,
-						retiredAt: supersededAt
-					});
-				}
-				for (const n of news) byObjectKey.set(n.objectKey, n);
-				snap.entries = Array.from(byObjectKey.values());
-			});
-		});
-	}
-	return {
-		async listLive(filter) {
-			return readEntriesAcrossShards(filter, false);
-		},
-		async listAll(filter) {
-			return readEntriesAcrossShards(filter, true);
-		},
-		async registerVersion(entry, superseding) {
-			return registerVersionsImpl([entry], superseding);
-		},
-		async registerVersions(entries, superseding) {
-			shardScopesFromEntries(entries);
-			return registerVersionsImpl(entries, superseding);
-		},
-		async listRetired(olderThan) {
-			return (await mapWithConcurrency(await listShards(), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const retired = [];
-				for (const e of snapshot.entries) if (e.retiredAt !== void 0 && e.retiredAt <= olderThan) retired.push(e);
-				return retired;
-			})).flat();
-		},
-		async delete(toDelete) {
-			await mapWithConcurrency([...groupBySiteTable(toDelete)], SHARD_IO_CONCURRENCY, async ([shardKey, entries]) => {
-				const [siteId, table] = shardKey.split("\0");
-				await mutateShard(siteId, table, (snap) => {
-					const drop = new Set(entries.map((e) => e.objectKey));
-					snap.entries = snap.entries.filter((e) => !drop.has(e.objectKey));
-				});
-			});
-		},
-		async getWatermarks(filter) {
-			assertScopedUser(filter.userId, "getWatermarks");
-			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const watermarks = [];
-				for (const w of snapshot.watermarks) if (matchesWatermarkFilter(w, filter, { ignoreUserId: true })) watermarks.push(w);
-				return watermarks;
-			})).flat();
-		},
-		async bumpWatermark(scope, date, at) {
-			assertScopedUser(scope.userId, "bumpWatermark");
-			if (scope.siteId === void 0) throw new Error("R2 manifest store requires watermarks to carry siteId");
-			const ts = at ?? now();
-			const scopeSearchType = inferSearchType(scope);
-			await mutateShard(scope.siteId, scope.table, (snap) => {
-				const idx = snap.watermarks.findIndex((w) => w.userId === userId && w.siteId === scope.siteId && w.table === scope.table && inferSearchType(w) === scopeSearchType);
-				if (idx === -1) {
-					snap.watermarks.push({
-						userId,
-						siteId: scope.siteId,
-						table: scope.table,
-						...scope.searchType !== void 0 ? { searchType: scope.searchType } : {},
-						newestDateSynced: date,
-						oldestDateSynced: date,
-						lastSyncAt: ts
-					});
-					return;
-				}
-				const w = snap.watermarks[idx];
-				const newest = date > w.newestDateSynced ? date : w.newestDateSynced;
-				const oldest = date < w.oldestDateSynced ? date : w.oldestDateSynced;
-				const lastSyncAt = ts > w.lastSyncAt ? ts : w.lastSyncAt;
-				snap.watermarks[idx] = {
-					...w,
-					newestDateSynced: newest,
-					oldestDateSynced: oldest,
-					lastSyncAt
-				};
-			});
-		},
-		async getSyncStates(filter) {
-			assertScopedUser(filter.userId, "getSyncStates");
-			return (await mapWithConcurrency(await shardsForFilter(filter), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const states = [];
-				for (const s of snapshot.syncStates) if (matchesSyncStateFilter(s, filter, { ignoreUserId: true })) states.push(s);
-				return states;
-			})).flat();
-		},
-		async setSyncState(scope, state, detail) {
-			assertScopedUser(scope.userId, "setSyncState");
-			if (scope.siteId === void 0) throw new Error("R2 manifest store requires sync states to carry siteId");
-			const at = detail?.at ?? now();
-			const scopeSearchType = inferSearchType(scope);
-			await mutateShard(scope.siteId, scope.table, (snap) => {
-				const idx = snap.syncStates.findIndex((s) => s.userId === userId && s.siteId === scope.siteId && s.table === scope.table && s.date === scope.date && inferSearchType(s) === scopeSearchType);
-				if (idx === -1) {
-					snap.syncStates.push({
-						userId,
-						siteId: scope.siteId,
-						table: scope.table,
-						date: scope.date,
-						state,
-						updatedAt: at,
-						attempts: 1,
-						error: detail?.error,
-						...scope.searchType !== void 0 ? { searchType: scope.searchType } : {}
-					});
-					return;
-				}
-				const prev = snap.syncStates[idx];
-				const attempts = state === "inflight" && prev.state !== "inflight" ? prev.attempts + 1 : prev.attempts;
-				const error = state === "done" ? void 0 : detail?.error ?? prev.error;
-				snap.syncStates[idx] = {
-					...prev,
-					state,
-					updatedAt: at,
-					attempts,
-					error
-				};
-			});
-		},
-		async withLock(_scope, fn) {
-			return fn();
-		},
-		async purgeTenant(filter) {
-			if (filter.userId !== userId) throw new Error(`purgeTenant: store is scoped to userId=${userId}, got ${filter.userId}`);
-			const purged = await mapWithConcurrency(await shardsForFilter({ siteId: filter.siteId }), SHARD_IO_CONCURRENCY, async ({ siteId, table }) => {
-				const { snapshot } = await readShard(siteId, table);
-				const prefix = shardPrefix(userId, siteId, table);
-				const keys = [];
-				let cursor;
-				do {
-					const res = await bucket.list({
-						prefix,
-						cursor,
-						limit: 1e3
-					});
-					for (const obj of res.objects) keys.push(obj.key);
-					cursor = res.truncated ? res.cursor : void 0;
-				} while (cursor);
-				for (let offset = 0; offset < keys.length; offset += R2_DELETE_BATCH_SIZE) await bucket.delete(keys.slice(offset, offset + R2_DELETE_BATCH_SIZE));
-				return {
-					entriesRemoved: snapshot.entries.length,
-					watermarksRemoved: snapshot.watermarks.length,
-					syncStatesRemoved: snapshot.syncStates.length
-				};
-			});
-			return {
-				entriesRemoved: purged.reduce((sum, result) => sum + result.entriesRemoved, 0),
-				watermarksRemoved: purged.reduce((sum, result) => sum + result.watermarksRemoved, 0),
-				syncStatesRemoved: purged.reduce((sum, result) => sum + result.syncStatesRemoved, 0)
-			};
-		}
-	};
-}
-function createQueryProfiler(sink, now = () => Date.now()) {
-	return { start(name, meta) {
-		const t0 = now();
-		return (extra) => {
-			const merged = meta || extra ? {
-				...meta,
-				...extra
-			} : void 0;
-			sink(merged ? {
-				name,
-				ms: now() - t0,
-				meta: merged
-			} : {
-				name,
-				ms: now() - t0
-			});
-		};
-	} };
-}
-function collectSpans(now) {
-	const spans = [];
-	return {
-		profiler: createQueryProfiler((s) => spans.push(s), now),
-		spans
-	};
-}
-const DAY = 24 * (3600 * 1e3);
-function isDue(state, now) {
-	return now >= state.nextAt;
-}
-function sitemapCadenceMs(consecutiveUnchanged) {
-	if (consecutiveUnchanged >= 7) return 30 * DAY;
-	if (consecutiveUnchanged >= 3) return 7 * DAY;
-	return DAY;
-}
-const SITEMAP_VERSION = 1;
-const sitemapPolicy = {
-	version: SITEMAP_VERSION,
-	initial(now) {
-		return {
-			nextAt: now + DAY,
-			consecutiveUnchanged: 0,
-			policyVersion: SITEMAP_VERSION
-		};
-	},
-	observe(prev, evt) {
-		if (prev.policyVersion !== SITEMAP_VERSION) return {
-			nextAt: evt.at + sitemapCadenceMs(0),
-			consecutiveUnchanged: 0,
-			policyVersion: SITEMAP_VERSION
-		};
-		if (evt.changed) return {
-			nextAt: evt.at + DAY,
-			consecutiveUnchanged: 0,
-			policyVersion: SITEMAP_VERSION
-		};
-		const next = prev.consecutiveUnchanged + 1;
-		return {
-			nextAt: evt.at + sitemapCadenceMs(next),
-			consecutiveUnchanged: next,
-			policyVersion: SITEMAP_VERSION
-		};
-	},
-	isDue
-};
-const INSPECTION_VERSION = 1;
-function inspectionCadenceMs(verdict) {
-	if (verdict === "PASS") return 30 * DAY;
-	if (verdict === "FAIL") return 7 * DAY;
-	return 14 * DAY;
-}
-function inspectionPolicy(verdict) {
-	const cadence = inspectionCadenceMs(verdict);
-	return {
-		version: INSPECTION_VERSION,
-		initial(now) {
-			return {
-				nextAt: now + cadence,
-				consecutiveUnchanged: 0,
-				policyVersion: INSPECTION_VERSION
-			};
-		},
-		observe(prev, evt) {
-			if (prev.policyVersion !== INSPECTION_VERSION) return {
-				nextAt: evt.at + cadence,
-				consecutiveUnchanged: 0,
-				policyVersion: INSPECTION_VERSION
-			};
-			const next = evt.changed ? 0 : prev.consecutiveUnchanged + 1;
-			return {
-				nextAt: evt.at + cadence,
-				consecutiveUnchanged: next,
-				policyVersion: INSPECTION_VERSION
-			};
-		},
-		isDue
-	};
-}
-const FIXED_VERSION = 1;
-function fixedPolicy(intervalMs) {
-	return {
-		version: FIXED_VERSION,
-		initial(now) {
-			return {
-				nextAt: now + intervalMs,
-				consecutiveUnchanged: 0,
-				policyVersion: FIXED_VERSION
-			};
-		},
-		observe(prev, evt) {
-			if (prev.policyVersion !== FIXED_VERSION) return {
-				nextAt: evt.at + intervalMs,
-				consecutiveUnchanged: 0,
-				policyVersion: FIXED_VERSION
-			};
-			const next = evt.changed ? 0 : prev.consecutiveUnchanged + 1;
-			return {
-				nextAt: evt.at + intervalMs,
-				consecutiveUnchanged: next,
-				policyVersion: FIXED_VERSION
-			};
-		},
-		isDue
-	};
-}
-const KEY_SEP = "\0";
-function partitionKey(slice) {
-	return [
-		slice.table,
-		slice.ctx.siteId ?? "",
-		slice.searchType,
-		slice.date
-	].join(KEY_SEP);
-}
-function tableOfKey(key) {
-	return key.slice(0, key.indexOf(KEY_SEP));
-}
-function withIdentity(slice, rows) {
-	return rows.map((r) => ({
-		...r,
-		site_id: slice.ctx.siteId ?? "",
-		search_type: slice.searchType
-	}));
-}
-function createInMemorySink() {
-	const partitions = /* @__PURE__ */ new Map();
-	let closed = false;
-	function allRows() {
-		return [...partitions.values()].flat();
-	}
-	return {
-		capabilities: { appendOnly: true },
-		async emit(slice, rows) {
-			const key = partitionKey(slice);
-			const stored = withIdentity(slice, rows);
-			const existing = partitions.get(key);
-			if (existing) existing.push(...stored);
-			else partitions.set(key, stored);
-			return { rowCount: stored.length };
-		},
-		async close() {
-			closed = true;
-			return {
-				flushed: [...new Set([...partitions.keys()].map((k) => tableOfKey(k)))],
-				failed: []
-			};
-		},
-		get rows() {
-			return allRows();
-		},
-		get closed() {
-			return closed;
-		},
-		rowsFor(table) {
-			return [...partitions.entries()].filter(([k]) => tableOfKey(k) === table).flatMap(([, v]) => v);
-		},
-		rowsForSlice(slice) {
-			return [...partitions.get(partitionKey(slice)) ?? []];
-		},
-		reset() {
-			partitions.clear();
-			closed = false;
-		}
-	};
-}
-const TABLES_BY_SEARCH_TYPE = {
-	web: [
-		"pages",
-		"queries",
-		"countries",
-		"page_queries",
-		"dates"
-	],
-	discover: [
-		"pages",
-		"countries",
-		"dates"
-	],
-	news: [
-		"pages",
-		"countries",
-		"dates"
-	],
-	googleNews: [
-		"pages",
-		"countries",
-		"dates"
-	],
-	image: [
-		"pages",
-		"countries",
-		"dates"
-	],
-	video: [
-		"pages",
-		"countries",
-		"dates"
-	]
-};
-function parseEnabledSearchTypes(raw) {
-	if (!raw) return ["web"];
-	const parsed = JSON.parse(raw);
-	if (!Array.isArray(parsed) || parsed.length === 0) return ["web"];
-	const valid = parsed.filter((v) => typeof v === "string" && v in TABLES_BY_SEARCH_TYPE);
-	if (valid.length === 0) return ["web"];
-	if (!valid.includes("web")) valid.unshift("web");
-	return valid;
-}
-function validateEnabledSearchTypesResult(value) {
-	if (!Array.isArray(value) || value.length === 0) return err(engineErrors.searchTypesNotArray());
-	const seen = /* @__PURE__ */ new Set();
-	const out = [];
-	for (const v of value) {
-		if (typeof v !== "string" || !(v in TABLES_BY_SEARCH_TYPE)) return err(engineErrors.unknownSearchType(v));
-		if (seen.has(v)) continue;
-		seen.add(v);
-		out.push(v);
-	}
-	if (!out.includes("web")) return err(engineErrors.searchTypesMissingWeb());
-	return ok(out);
-}
-function validateEnabledSearchTypes(value) {
-	return unwrapResult(validateEnabledSearchTypesResult(value), engineErrorToException);
-}
-const TABLE_TIERS = {
-	pages: "critical",
-	queries: "critical",
-	countries: "standard",
-	dates: "standard",
-	page_queries: "extended"
-};
-function getTableTier(table) {
-	return TABLE_TIERS[table] || "extended";
-}
-function getTablesForTier(tier) {
-	return Object.entries(TABLE_TIERS).filter(([_, t]) => t === tier).map(([name]) => name);
-}
-function getDateWeight(date, now = /* @__PURE__ */ new Date()) {
-	const target = new Date(date);
-	const daysAgo = Math.floor((now.getTime() - target.getTime()) / (1e3 * 60 * 60 * 24));
-	if (daysAgo <= 3) return "fresh";
-	if (daysAgo <= 60) return "recent";
-	return "historical";
-}
-const TIER_PRIORITY = {
-	critical: 0,
-	standard: 1,
-	extended: 2
-};
-const WEIGHT_PRIORITY = {
-	fresh: 0,
-	recent: 1,
-	historical: 2
-};
-const MAX_GSC_PAGES_R2 = 40;
-const ROW_LIMIT_R2 = 25e3;
-const MIN_SYNC_IMPRESSIONS = 1;
-const MIN_COUNTRY_IMPRESSIONS = 10;
-const MAX_SITEMAP_URLS_PER_SITE = 5e4;
-const MAX_TRACKED_URLS_PER_SITE = 2e5;
+import { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey } from "./layout.mjs";
+import { createR2ManifestStore } from "./adapters/r2-manifest.mjs";
+import { coerceRow, coerceRows } from "./coerce.mjs";
+import { createDuckDBCodec, createDuckDBExecutor } from "./duckdb.mjs";
+import { MAX_DAY_BYTES, createStorageEngine } from "./engine.mjs";
+import { collectSpans, createQueryProfiler } from "./profile.mjs";
+import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
+import { createInMemorySink } from "./sinks/in-memory-sink.mjs";
+import "./sinks/index.mjs";
+import { MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, getDateWeight, getTableTier, getTablesForTier, parseEnabledSearchTypes, validateEnabledSearchTypes, validateEnabledSearchTypesResult } from "./sync-config.mjs";
 export { DEFAULT_SEARCH_TYPE, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, TABLES_BY_SEARCH_TYPE, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, coerceRow, coerceRows, collectSpans, createDuckDBCodec, createDuckDBExecutor, createInMemorySink, createQueryProfiler, createR2ManifestStore, createStorageEngine, dayPartition, fixedPolicy, getDateWeight, getTableTier, getTablesForTier, hourPartition, inferLegacyTier, inferSearchType, inspectionPolicy, objectKey, parseEnabledSearchTypes, sitemapPolicy, validateEnabledSearchTypes, validateEnabledSearchTypesResult };

package/dist/ingest-accumulator.d.mts

@@ -1,4 +1,4 @@
-import { SearchType, TenantCtx as TenantCtx$1 } from "./_chunks/storage.mjs";
+import { SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { GscApiRow, RowAccumulatorOptions } from "./ingest.mjs";
 import { Grain, Row, TableName } from "@gscdump/contracts";
 interface IngestAccumulatorEngine {

package/dist/ingest.d.mts

@@ -1,4 +1,4 @@
-import { Row, TableName } from "./_chunks/storage.mjs";
+import { Row, TableName } from "./storage.mjs";
 /**
  * Canonical GSC API dimension order per table. Consumers hitting the raw
  * `searchanalytics.query` endpoint must request dimensions in this order so

package/dist/layout.d.mts

@@ -0,0 +1,31 @@
+import { CompactionTier, ManifestEntry } from "./storage.mjs";
+import { SearchType } from "gscdump/query";
+import { TableName, TenantCtx } from "@gscdump/contracts";
+declare function dayPartition(date: string): string;
+/**
+ * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet
+ * per day holds 24 hourly buckets — read-merge-write keeps `(url, hour)`
+ * idempotency across retries. Names sort lexically alongside daily ones but
+ * never collide because of the `hourly/` prefix.
+ */
+declare function hourPartition(date: string): string;
+/**
+ * Default `searchType` for entries written before the field landed and for
+ * sync paths that don't request a specific type. GSC's own default; the
+ * vast majority of stored data is web-search.
+ */
+declare const DEFAULT_SEARCH_TYPE: SearchType;
+declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
+/**
+ * Resolve the search type for an entry, defaulting legacy entries to `web`.
+ * Use this anywhere code needs to bucket entries by searchType.
+ */
+declare function inferSearchType(entry: Pick<ManifestEntry, 'searchType'>): SearchType;
+/**
+ * Infer the tier for an entry that pre-dates the `tier` field. Daily files
+ * are `raw`; monthly files are `d30`. Anything else (already migrated, or
+ * a partition shape we haven't seen) returns undefined and the caller must
+ * decide how to handle it.
+ */
+declare function inferLegacyTier(entry: Pick<ManifestEntry, 'partition' | 'tier'>): CompactionTier | undefined;
+export { DEFAULT_SEARCH_TYPE, dayPartition, hourPartition, inferLegacyTier, inferSearchType, objectKey };