@gscdump/engine 1.4.10 → 1.5.0

package/dist/entities.mjs

@@ -1,1382 +1,9 @@
 import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
-import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { readOptional } from "./adapters/read-optional.mjs";
-import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
-import { buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
-import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
-import { GSCDUMP_INDEXING_TRANSITION_FIELDS } from "@gscdump/contracts";
-import { classifyCanonicalDifference } from "gscdump";
-const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
-const ENTITY_IO_CONCURRENCY = 8;
-async function mapEntityIo(items, fn) {
-	if (items.length === 0) return [];
-	const results = Array.from({ length: items.length }, () => void 0);
-	let next = 0;
-	async function worker() {
-		while (true) {
-			const index = next++;
-			if (index >= items.length) return;
-			results[index] = await fn(items[index], index);
-		}
-	}
-	await Promise.all(Array.from({ length: Math.min(ENTITY_IO_CONCURRENCY, items.length) }, worker));
-	return results;
-}
-const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
-const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
-const INSPECTION_PARQUET_COLUMNS = [
-	{
-		name: "urlHash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "inspectedAt",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "indexStatus",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "lastCrawlTime",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "googleCanonical",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "userCanonical",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "coverageState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "robotsTxtState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "indexingState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "pageFetchState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "mobileUsabilityVerdict",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "richResultsVerdict",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "scheduleNextAt",
-		type: "BIGINT",
-		nullable: true
-	},
-	{
-		name: "scheduleConsecutiveUnchanged",
-		type: "INTEGER",
-		nullable: true
-	},
-	{
-		name: "schedulePolicyVersion",
-		type: "INTEGER",
-		nullable: true
-	}
-];
-const INSPECTION_EVENT_COLUMNS = [
-	...INSPECTION_PARQUET_COLUMNS,
-	{
-		name: "canonicalMismatchKind",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "crawlingUserAgent",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "richResultsItems",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "sitemaps",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "referringUrls",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "mobileIssues",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "inspectionResultLink",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "firstCheckedAt",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "checkCount",
-		type: "INTEGER",
-		nullable: true
-	},
-	{
-		name: "nextCheckAfter",
-		type: "BIGINT",
-		nullable: true
-	},
-	{
-		name: "nextCheckPriority",
-		type: "VARCHAR",
-		nullable: true
-	}
-];
-const CANONICAL_DIFFERENCE_KINDS = /* @__PURE__ */ new Set([
-	"none",
-	"formatting",
-	"path",
-	"cross_domain"
-]);
-function populateCanonicalMismatchKind(row) {
-	const current = row.canonicalMismatchKind;
-	const expected = classifyCanonicalDifference(typeof row.userCanonical === "string" ? row.userCanonical : null, typeof row.googleCanonical === "string" ? row.googleCanonical : null);
-	const missingOrInvalid = typeof current !== "string" || !CANONICAL_DIFFERENCE_KINDS.has(current);
-	row.canonicalMismatchKind = expected;
-	return missingOrInvalid || current !== expected;
-}
-const INSPECTION_TRANSITION_COLUMNS = [
-	{
-		name: "urlHash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "changedAfter",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "changedBefore",
-		type: "VARCHAR",
-		nullable: false
-	},
-	...GSCDUMP_INDEXING_TRANSITION_FIELDS.flatMap((field) => {
-		const capped = field[0].toUpperCase() + field.slice(1);
-		return [{
-			name: `from${capped}`,
-			type: "VARCHAR",
-			nullable: true
-		}, {
-			name: `to${capped}`,
-			type: "VARCHAR",
-			nullable: true
-		}];
-	})
-];
-function isStateTransition(before, after) {
-	return GSCDUMP_INDEXING_TRANSITION_FIELDS.some((field) => (before[field] ?? null) !== (after[field] ?? null));
-}
-function buildTransitionRow(before, after) {
-	const row = {
-		urlHash: String(after.urlHash),
-		url: String(after.url ?? before.url ?? ""),
-		changedAfter: String(before.inspectedAt ?? ""),
-		changedBefore: String(after.inspectedAt ?? "")
-	};
-	for (const field of GSCDUMP_INDEXING_TRANSITION_FIELDS) {
-		const capped = field[0].toUpperCase() + field.slice(1);
-		row[`from${capped}`] = before[field] ?? null;
-		row[`to${capped}`] = after[field] ?? null;
-	}
-	return row;
-}
-async function appendTransitions(ds, ctx, rows) {
-	if (rows.length === 0) return 0;
-	const byMonth = /* @__PURE__ */ new Map();
-	for (const row of rows) {
-		const month = transitionMonth(row);
-		const bucket = byMonth.get(month);
-		if (bucket) bucket.push(row);
-		else byMonth.set(month, [row]);
-	}
-	let written = 0;
-	for (const [month, monthRows] of byMonth) {
-		const key = inspectionTransitionsMonthKey(ctx, month);
-		const existingBytes = await readOptional(ds, key);
-		const existing = existingBytes ? await decodeParquetToRows(existingBytes) : [];
-		const seen = new Set(existing.map((r) => `${String(r.urlHash)}\u0000${String(r.changedBefore)}`));
-		const fresh = monthRows.filter((r) => {
-			const id = `${String(r.urlHash)}\u0000${String(r.changedBefore)}`;
-			if (seen.has(id)) return false;
-			seen.add(id);
-			return true;
-		});
-		if (fresh.length === 0) continue;
-		const merged = [...existing, ...fresh];
-		await ds.write(key, encodeRowsToParquetFlex(merged, {
-			columns: INSPECTION_TRANSITION_COLUMNS,
-			sortKey: ["urlHash"]
-		}));
-		written += fresh.length;
-	}
-	return written;
-}
-function transitionMonth(row) {
-	return String(row.changedBefore ?? "").slice(0, 7) || "unknown";
-}
-function createInspectionStore(opts) {
-	const ds = opts.dataSource;
-	function shardFor(record) {
-		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
-		return m ? `${m[1]}-${m[2]}` : "unknown";
-	}
-	function randomBatchId() {
-		return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
-	}
-	return {
-		async appendHistory(ctx, records, options) {
-			if (records.length === 0) return;
-			const batchId = options?.batchId ?? randomBatchId();
-			const byMonth = /* @__PURE__ */ new Map();
-			for (const r of records) {
-				const month = shardFor(r);
-				if (!byMonth.has(month)) byMonth.set(month, []);
-				byMonth.get(month).push(r);
-			}
-			await mapEntityIo([...byMonth].map(([yearMonth, batch]) => {
-				const bytes = encodeJsonBigintSafe({
-					version: 1,
-					records: batch
-				});
-				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
-				return {
-					key: inspectionHistoryShardKey(ctx, yearMonth, batchId),
-					bytes
-				};
-			}), (shard) => ds.write(shard.key, shard.bytes));
-		},
-		async loadHistory(ctx, yearMonth) {
-			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));
-			if (keys.length === 0) return void 0;
-			return {
-				version: 1,
-				records: (await mapEntityIo(keys, async (key) => {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) return [];
-					return (await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
-						console.warn("[inspection.loadHistory] failed to decode shard", {
-							key,
-							error: err.message
-						});
-					}))?.records ?? [];
-				})).flat()
-			};
-		},
-		async materialize(ctx, rowIter) {
-			const rows = Array.from(rowIter);
-			rows.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-			const bytes = encodeRowsToParquetFlex(rows, {
-				columns: INSPECTION_PARQUET_COLUMNS,
-				sortKey: ["urlHash"]
-			});
-			const key = inspectionParquetKey(ctx);
-			await ds.write(key, bytes);
-			return {
-				key,
-				rowCount: rows.length,
-				bytes: bytes.byteLength
-			};
-		},
-		async appendInspectionEvents(ctx, rows, options) {
-			if (rows.length === 0) return {
-				keys: [],
-				rowCount: 0
-			};
-			const batchId = options?.batchId ?? randomBatchId();
-			const byMonth = /* @__PURE__ */ new Map();
-			for (const r of rows) {
-				const m = YEAR_MONTH_RE.exec(r.inspectedAt);
-				const month = m ? `${m[1]}-${m[2]}` : "unknown";
-				const bucket = byMonth.get(month) ?? [];
-				bucket.push(r);
-				byMonth.set(month, bucket);
-			}
-			const files = [...byMonth].map(([month, batch]) => {
-				const bytes = encodeRowsToParquetFlex(batch, {
-					columns: INSPECTION_EVENT_COLUMNS,
-					sortKey: ["urlHash"]
-				});
-				return {
-					key: inspectionEventKey(ctx, month, batchId),
-					bytes
-				};
-			});
-			await mapEntityIo(files, (file) => ds.write(file.key, file.bytes));
-			return {
-				keys: files.map((file) => file.key),
-				rowCount: rows.length
-			};
-		},
-		async compactInspections(ctx, opts) {
-			const eventKeys = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((k) => INSPECTION_EVENT_KEY_RE.test(k));
-			if (eventKeys.length === 0) return {
-				baseRowCount: 0,
-				eventsFolded: 0,
-				eventFilesDeleted: 0,
-				transitionsWritten: 0
-			};
-			const baseKey = inspectionBaseKey(ctx);
-			const baseBytes = await readOptional(ds, baseKey);
-			const baseRows = baseBytes ? await decodeParquetToRows(baseBytes) : [];
-			const latest = /* @__PURE__ */ new Map();
-			const earliestChecked = /* @__PURE__ */ new Map();
-			const transitions = [];
-			const consider = (row) => {
-				const h = String(row.urlHash);
-				const prev = latest.get(h);
-				if (!prev || String(row.inspectedAt ?? "") > String(prev.inspectedAt ?? "")) {
-					if (opts?.transitions && prev && isStateTransition(prev, row)) transitions.push(buildTransitionRow(prev, row));
-					latest.set(h, row);
-				}
-				const fc = row.firstCheckedAt;
-				if (fc != null) {
-					const fcStr = String(fc);
-					const cur = earliestChecked.get(h);
-					if (cur === void 0 || fcStr < cur) earliestChecked.set(h, fcStr);
-				}
-			};
-			for (const row of baseRows) consider(row);
-			let eventsFolded = 0;
-			const consumed = [];
-			const eventRows = [];
-			const eventFiles = await mapEntityIo(eventKeys.sort(), async (key) => {
-				const bytes = await readOptional(ds, key);
-				if (!bytes) return void 0;
-				return {
-					key,
-					rows: await decodeParquetToRows(bytes)
-				};
-			});
-			for (const file of eventFiles) {
-				if (!file) continue;
-				const { key, rows } = file;
-				consumed.push(key);
-				eventRows.push(...rows);
-				eventsFolded += rows.length;
-			}
-			eventRows.sort((a, b) => String(a.urlHash).localeCompare(String(b.urlHash)) || String(a.inspectedAt ?? "").localeCompare(String(b.inspectedAt ?? "")));
-			for (const row of eventRows) consider(row);
-			const merged = [];
-			for (const [h, row] of latest) {
-				const fc = earliestChecked.get(h);
-				if (fc !== void 0) row.firstCheckedAt = fc;
-				populateCanonicalMismatchKind(row);
-				merged.push(row);
-			}
-			const bytes = encodeRowsToParquetFlex(merged, {
-				columns: INSPECTION_EVENT_COLUMNS,
-				sortKey: ["urlHash"]
-			});
-			const transitionsWritten = opts?.transitions ? await appendTransitions(ds, ctx, transitions) : 0;
-			await ds.write(baseKey, bytes);
-			if (consumed.length > 0) await ds.delete(consumed);
-			return {
-				baseRowCount: merged.length,
-				eventsFolded,
-				eventFilesDeleted: consumed.length,
-				transitionsWritten
-			};
-		},
-		async backfillCanonicalMismatchKinds(ctx) {
-			const baseKey = inspectionBaseKey(ctx);
-			const baseBytes = await readOptional(ds, baseKey);
-			if (!baseBytes) return {
-				baseRowCount: 0,
-				rowsBackfilled: 0,
-				rewritten: false
-			};
-			const rows = await decodeParquetToRows(baseBytes);
-			let rowsBackfilled = 0;
-			for (const row of rows) if (populateCanonicalMismatchKind(row)) rowsBackfilled++;
-			if (rowsBackfilled === 0) return {
-				baseRowCount: rows.length,
-				rowsBackfilled: 0,
-				rewritten: false
-			};
-			await ds.write(baseKey, encodeRowsToParquetFlex(rows, {
-				columns: INSPECTION_EVENT_COLUMNS,
-				sortKey: ["urlHash"]
-			}));
-			return {
-				baseRowCount: rows.length,
-				rowsBackfilled,
-				rewritten: true
-			};
-		},
-		parquetUri(ctx) {
-			return ds.uri?.(inspectionParquetKey(ctx));
-		}
-	};
-}
-const SITEMAP_URLS_EVENT_PREFIX_RE = /\/urls\/events\/(\d{4}-\d{2}-\d{2})__[0-9a-f]+__\d+__[0-9a-f]+\.parquet$/;
-const SITEMAP_URLS_PENDING_GENERATION_RE = /\/urls\/generations\/pending\/([0-9a-f]+)\.json$/;
-const URLS_INDEX_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "first_seen_at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "last_seen_at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "removed_at",
-		type: "BIGINT",
-		nullable: true
-	}
-];
-const URLS_DELTA_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "op",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "generation_id",
-		type: "VARCHAR",
-		nullable: true
-	}
-];
-const URLS_EVENT_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "op",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "generation_id",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "observed_at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "sequence",
-		type: "INTEGER",
-		nullable: false
-	},
-	{
-		name: "projects_state",
-		type: "INTEGER",
-		nullable: false
-	},
-	{
-		name: "seed_generation",
-		type: "INTEGER",
-		nullable: false
-	},
-	{
-		name: "generation_kind",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "content_hash",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "input_count",
-		type: "INTEGER",
-		nullable: true
-	}
-];
-function rowToUrlRecord(row) {
-	return {
-		feedpath: String(row.feedpath),
-		feedpathHash: String(row.feedpath_hash),
-		urlHash: String(row.url_hash),
-		loc: String(row.loc),
-		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-		firstSeenAt: Number(row.first_seen_at),
-		lastSeenAt: Number(row.last_seen_at),
-		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
-	};
-}
-function urlRecordToRow(r) {
-	return {
-		feedpath: r.feedpath,
-		feedpath_hash: r.feedpathHash,
-		url_hash: r.urlHash,
-		loc: r.loc,
-		lastmod: r.lastmod ?? null,
-		first_seen_at: r.firstSeenAt,
-		last_seen_at: r.lastSeenAt,
-		removed_at: r.removedAt ?? null
-	};
-}
-function createSitemapUrlState(indexRows) {
-	const state = {
-		live: /* @__PURE__ */ new Map(),
-		removed: /* @__PURE__ */ new Map()
-	};
-	for (const row of indexRows) {
-		const record = rowToUrlRecord(row);
-		if (record.removedAt == null) state.live.set(record.urlHash, record);
-		else state.removed.set(record.urlHash, record);
-	}
-	return state;
-}
-function applySitemapDeltaRows(state, rows) {
-	for (const row of rows) {
-		const urlHash = String(row.url_hash);
-		const observedAt = Number(row.at);
-		const op = String(row.op);
-		if (op === "added") {
-			const previous = state.live.get(urlHash) ?? state.removed.get(urlHash);
-			state.removed.delete(urlHash);
-			state.live.set(urlHash, {
-				feedpath: String(row.feedpath),
-				feedpathHash: String(row.feedpath_hash),
-				urlHash,
-				loc: String(row.loc),
-				lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-				firstSeenAt: previous?.firstSeenAt ?? observedAt,
-				lastSeenAt: observedAt
-			});
-		} else if (op === "removed") {
-			const previous = state.live.get(urlHash);
-			state.live.delete(urlHash);
-			if (previous) state.removed.set(urlHash, {
-				...previous,
-				removedAt: observedAt
-			});
-		}
-	}
-}
-function applySitemapDeltaFiles(state, files) {
-	for (const file of files) applySitemapDeltaRows(state, file.rows);
-}
-function sortSitemapDeltaFiles(files) {
-	return files.sort((a, b) => a.key.localeCompare(b.key));
-}
-async function readSitemapDeltaFiles(ds, keys) {
-	return sortSitemapDeltaFiles((await mapEntityIo(keys, async (key) => {
-		const bytes = await readOptional(ds, key);
-		return bytes ? {
-			key,
-			rows: await decodeParquetToRows(bytes)
-		} : void 0;
-	})).filter((file) => file !== void 0));
-}
-function dateInRange(date, range) {
-	return (!range?.from || date >= range.from) && (!range?.to || date <= range.to);
-}
-function createSitemapReadStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	async function readJson(key) {
-		const bytes = await readOptional(ds, key);
-		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
-	}
-	return {
-		async loadIndex(ctx) {
-			return await readJson(sitemapIndexKey(ctx)) ?? {
-				version: 1,
-				records: {}
-			};
-		},
-		async getLatest(ctx, path) {
-			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
-		},
-		async *loadUrls(ctx, feedpath, loadOpts) {
-			const feedpathHash = hash(feedpath);
-			const listedDeltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).filter((key) => parseSitemapUrlsDeltaKey(key)?.feedpathHash === feedpathHash);
-			const listedDeltaFiles = await readSitemapDeltaFiles(ds, listedDeltaKeys);
-			const manifestBytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
-			const manifest = manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0;
-			const indexRows = await readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []);
-			const currentDeltaKeys = new Set(selectSitemapProjectionFiles([], listedDeltaKeys, manifest).deltaKeys);
-			const state = createSitemapUrlState(indexRows);
-			applySitemapDeltaFiles(state, listedDeltaFiles.filter((file) => currentDeltaKeys.has(file.key)));
-			for (const record of state.live.values()) yield record;
-			if (loadOpts?.includeRemoved) for (const record of state.removed.values()) yield record;
-		},
-		async *loadDeltas(ctx, dateRange) {
-			const [listedKeys, manifestBytes] = await Promise.all([ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`), readOptional(ds, sitemapUrlsProjectionManifestKey(ctx))]);
-			const keys = selectSitemapProjectionFiles([], listedKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0).deltaKeys.filter((key) => {
-				const parsed = parseSitemapUrlsDeltaKey(key);
-				return Boolean(parsed && dateInRange(parsed.date, dateRange));
-			});
-			const files = await readSitemapDeltaFiles(ds, keys);
-			for (const file of files) for (const row of file.rows) {
-				const op = String(row.op);
-				if (op !== "added" && op !== "removed") continue;
-				yield {
-					feedpath: String(row.feedpath),
-					feedpathHash: String(row.feedpath_hash),
-					urlHash: String(row.url_hash),
-					op,
-					loc: String(row.loc),
-					lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-					at: Number(row.at)
-				};
-			}
-		},
-		async *loadEvents(ctx, dateRange) {
-			const listedEventKeys = await ds.list(`${sitemapUrlsEventsPrefix(ctx)}/`);
-			const pendingKeys = await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`);
-			const pendingEvents = new Set((await mapEntityIo(pendingKeys, (key) => readJson(key))).filter((pending) => pending !== void 0).map((pending) => pending.eventKey));
-			const keys = listedEventKeys.filter((key) => {
-				const match = SITEMAP_URLS_EVENT_PREFIX_RE.exec(key);
-				return !pendingEvents.has(key) && Boolean(match?.[1] && dateInRange(match[1], dateRange));
-			}).sort();
-			for (const key of keys) {
-				const bytes = await readOptional(ds, key);
-				if (!bytes) continue;
-				const rows = await decodeParquetToRows(bytes);
-				rows.sort((a, b) => Number(a.observed_at) - Number(b.observed_at) || Number(a.sequence) - Number(b.sequence) || String(a.url_hash).localeCompare(String(b.url_hash)));
-				for (const row of rows) {
-					const op = String(row.op);
-					if (op !== "added" && op !== "removed") continue;
-					yield {
-						feedpath: String(row.feedpath),
-						feedpathHash: String(row.feedpath_hash),
-						urlHash: String(row.url_hash),
-						op,
-						loc: String(row.loc),
-						lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-						generationId: String(row.generation_id),
-						observedAt: Number(row.observed_at),
-						sequence: Number(row.sequence),
-						projectsState: Boolean(row.projects_state)
-					};
-				}
-			}
-		}
-	};
-}
-function createSitemapStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	const now = opts.now ?? (() => Date.now());
-	const withMutation = opts.withMutation;
-	const readStore = createSitemapReadStore({
-		dataSource: ds,
-		hash
-	});
-	async function readJson(key) {
-		const bytes = await readOptional(ds, key);
-		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
-	}
-	function writeJson(key, value) {
-		return ds.write(key, encodeJsonBigintSafe(value));
-	}
-	async function readProjectionManifest(ctx) {
-		const bytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
-		return bytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(bytes)) : emptySitemapProjectionManifest();
-	}
-	async function deleteExpiredProjectionDeltas(manifest, deltaKeys, currentTime) {
-		const expired = deltaKeys.filter((key) => {
-			const parsed = parseSitemapUrlsDeltaKey(key);
-			const feed = parsed ? manifest.feeds[parsed.feedpathHash] : void 0;
-			return Boolean(feed && key <= feed.compactedThrough && currentTime - feed.publishedAt >= 9e5);
-		});
-		if (expired.length > 0) await ds.delete(expired);
-	}
-	async function publishCurrentProjection(ctx, feedpathHash, rows, consumedDeltas, manifest, publishedAt) {
-		const bytes = encodeRowsToParquetFlex(rows.map(urlRecordToRow), {
-			columns: URLS_INDEX_COLUMNS,
-			sortKey: ["feedpath_hash", "url_hash"]
-		});
-		await ds.write(sitemapUrlsIndexKey(ctx, feedpathHash), bytes);
-		const compactedThrough = consumedDeltas.at(-1)?.key;
-		if (!compactedThrough) return manifest;
-		const next = withSitemapProjectionFeed(manifest, feedpathHash, {
-			compactedThrough,
-			publishedAt
-		});
-		await writeJson(sitemapUrlsProjectionManifestKey(ctx), next);
-		return next;
-	}
-	function normalizedEventRows(rows) {
-		return rows.map((row) => ({
-			feedpath: String(row.feedpath),
-			feedpathHash: String(row.feedpath_hash),
-			urlHash: String(row.url_hash),
-			op: String(row.op),
-			loc: String(row.loc),
-			lastmod: row.lastmod == null ? null : String(row.lastmod),
-			generationId: String(row.generation_id),
-			observedAt: Number(row.observed_at),
-			sequence: Number(row.sequence),
-			projectsState: Boolean(row.projects_state),
-			seedGeneration: Boolean(row.seed_generation),
-			generationKind: String(row.generation_kind),
-			contentHash: row.content_hash == null ? null : String(row.content_hash),
-			inputCount: row.input_count == null ? null : Number(row.input_count)
-		})).sort((a, b) => a.observedAt - b.observedAt || a.sequence - b.sequence || a.urlHash.localeCompare(b.urlHash) || a.op.localeCompare(b.op));
-	}
-	function eventDigest(rows) {
-		return hashUrl(JSON.stringify(normalizedEventRows(rows)));
-	}
-	function assertGenerationAccepted(generation, checkpoint, scope) {
-		if (!checkpoint) return "newer";
-		if (generation.id === checkpoint.generationId) {
-			if (generation.observedAt === checkpoint.observedAt) return "same";
-			throw new Error(`sitemap generation conflict for ${scope}: generation ${generation.id} changed observedAt`);
-		}
-		if (generation.observedAt > checkpoint.observedAt) return "newer";
-		const reason = generation.observedAt === checkpoint.observedAt ? "ambiguous timestamp" : "stale generation";
-		throw new Error(`sitemap generation conflict for ${scope}: ${reason}`);
-	}
-	async function ensureEvents(ctx, feedpathHash, generation, rows) {
-		const expectedDigest = eventDigest(rows);
-		if (rows.length === 0) return {
-			rows: [],
-			digest: expectedDigest
-		};
-		const key = sitemapUrlsEventKey(ctx, feedpathHash, generation);
-		await writeJson(sitemapUrlsPendingGenerationKey(ctx, feedpathHash), {
-			version: 1,
-			generationId: generation.id,
-			observedAt: generation.observedAt,
-			eventKey: key,
-			eventDigest: expectedDigest
-		});
-		const existing = await readOptional(ds, key);
-		if (existing) {
-			const existingRows = await decodeParquetToRows(existing);
-			const existingDigest = eventDigest(existingRows);
-			if (existingDigest !== expectedDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: immutable event digest changed`);
-			return {
-				rows: existingRows,
-				digest: existingDigest
-			};
-		}
-		const bytes = encodeRowsToParquetFlex(rows, {
-			columns: URLS_EVENT_COLUMNS,
-			sortKey: ["sequence", "url_hash"]
-		});
-		await ds.write(key, bytes);
-		return {
-			rows: [...rows],
-			digest: expectedDigest
-		};
-	}
-	function eventRow(generation, row, metadata) {
-		return {
-			feedpath: row.feedpath,
-			feedpath_hash: row.feedpath_hash,
-			url_hash: row.url_hash,
-			op: row.op,
-			loc: row.loc,
-			lastmod: row.lastmod,
-			generation_id: generation.id,
-			observed_at: generation.observedAt,
-			sequence: metadata.sequence,
-			projects_state: metadata.projectsState ? 1 : 0,
-			seed_generation: metadata.seedGeneration ? 1 : 0,
-			generation_kind: metadata.generationKind,
-			content_hash: metadata.contentHash ?? null,
-			input_count: metadata.inputCount ?? null
-		};
-	}
-	function stateRowsFromEvents(rows) {
-		return rows.filter((row) => Boolean(row.projects_state)).map((row) => ({
-			feedpath: row.feedpath,
-			feedpath_hash: row.feedpath_hash,
-			url_hash: row.url_hash,
-			op: row.op,
-			loc: row.loc,
-			lastmod: row.lastmod,
-			at: row.observed_at,
-			generation_id: row.generation_id
-		}));
-	}
-	function checkpointFromEvents(rows, digest) {
-		const first = rows[0];
-		if (!first) throw new Error("cannot checkpoint an empty sitemap event file");
-		const generationId = String(first.generation_id);
-		const observedAt = Number(first.observed_at);
-		for (const row of rows) if (String(row.generation_id) !== generationId || Number(row.observed_at) !== observedAt) throw new Error("sitemap event file contains multiple generations");
-		if (String(first.generation_kind) === "snapshot") {
-			const contentHash = String(first.content_hash);
-			const inputCount = Number(first.input_count);
-			const stateRows = stateRowsFromEvents(rows);
-			const added = stateRows.filter((row) => String(row.op) === "added").length;
-			return {
-				_tag: "snapshot",
-				version: 1,
-				generationId,
-				observedAt,
-				eventDigest: digest,
-				result: {
-					added,
-					removed: stateRows.filter((row) => String(row.op) === "removed").length,
-					kept: Math.max(0, inputCount - added),
-					contentHash,
-					unchanged: stateRows.length === 0
-				}
-			};
-		}
-		return {
-			_tag: "reconcile",
-			version: 1,
-			generationId,
-			observedAt,
-			eventDigest: digest
-		};
-	}
-	async function projectEvents(ctx, feedpathHash, rows, digest) {
-		const checkpoint = checkpointFromEvents(rows, digest);
-		const generation = {
-			id: checkpoint.generationId,
-			observedAt: checkpoint.observedAt
-		};
-		const stateRows = stateRowsFromEvents(rows);
-		if (stateRows.length > 0) {
-			const bytes = encodeRowsToParquetFlex(stateRows, {
-				columns: URLS_DELTA_COLUMNS,
-				sortKey: ["url_hash"]
-			});
-			await ds.write(sitemapUrlsDeltaKey(ctx, feedpathHash, generation), bytes);
-		}
-		if (rows.some((row) => Boolean(row.seed_generation))) await writeJson(sitemapUrlsEventSeedKey(ctx, feedpathHash), {
-			version: 1,
-			generationId: checkpoint.generationId,
-			observedAt: checkpoint.observedAt
-		});
-		await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), checkpoint);
-		return checkpoint;
-	}
-	async function repairFeedProjection(ctx, feedpathHash) {
-		const pendingKey = sitemapUrlsPendingGenerationKey(ctx, feedpathHash);
-		const [checkpoint, pending] = await Promise.all([readJson(sitemapUrlsGenerationKey(ctx, feedpathHash)), readJson(pendingKey)]);
-		if (!pending) return checkpoint;
-		const pendingGeneration = {
-			_tag: "complete",
-			id: pending.generationId,
-			observedAt: pending.observedAt
-		};
-		if (checkpoint?.generationId === pending.generationId && checkpoint.observedAt !== pending.observedAt) throw new Error(`sitemap generation conflict for ${feedpathHash}: generation ${pending.generationId} changed observedAt`);
-		if (checkpoint && pending.observedAt < checkpoint.observedAt) {
-			await ds.delete([pendingKey]);
-			return checkpoint;
-		}
-		const position = assertGenerationAccepted(pendingGeneration, checkpoint, feedpathHash);
-		const eventBytes = await readOptional(ds, pending.eventKey);
-		if (!eventBytes) {
-			await ds.delete([pendingKey]);
-			return checkpoint;
-		}
-		const rows = await decodeParquetToRows(eventBytes);
-		const digest = eventDigest(rows);
-		if (digest !== pending.eventDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: pending event digest changed`);
-		if (position === "same") {
-			if (checkpoint?.eventDigest !== digest) throw new Error(`sitemap generation conflict for ${feedpathHash}: checkpoint digest changed`);
-			await ds.delete([pendingKey]);
-			return checkpoint;
-		}
-		const repaired = await projectEvents(ctx, feedpathHash, rows, digest);
-		await ds.delete([pendingKey]);
-		return repaired;
-	}
-	return {
-		...readStore,
-		writeSnapshot(ctx, records) {
-			return withMutation(ctx, async () => {
-				if (records.length === 0) return;
-				const indexKey = sitemapIndexKey(ctx);
-				const index = await readJson(indexKey) ?? {
-					version: 1,
-					records: {}
-				};
-				const stamp = now();
-				const historyDocs = /* @__PURE__ */ new Map();
-				for (const record of records) {
-					const feedpathHash = hash(record.path);
-					index.records[feedpathHash] = record;
-					historyDocs.set(sitemapHistoryKey(ctx, feedpathHash, stamp), {
-						version: 1,
-						path: record.path,
-						capturedAt: record.capturedAt,
-						record
-					});
-				}
-				await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
-				await writeJson(indexKey, index);
-			});
-		},
-		snapshotUrls(ctx, generation, feedpath, urls) {
-			return withMutation(ctx, async () => {
-				const feedpathHash = hash(feedpath);
-				const contentHash = hashUrlList(urls);
-				const checkpoint = await repairFeedProjection(ctx, feedpathHash);
-				if (assertGenerationAccepted(generation, checkpoint, feedpathHash) === "same") {
-					if (checkpoint?._tag !== "snapshot" || checkpoint.result.contentHash !== contentHash) throw new Error(`sitemap generation conflict for ${feedpathHash}: snapshot input changed`);
-					return checkpoint.result;
-				}
-				if (assertGenerationAccepted(generation, await readJson(sitemapUrlsReconcileGenerationKey(ctx)), "site reconciliation") === "same") throw new Error("sitemap generation conflict: site generation was already reconciled");
-				const priorByHash = /* @__PURE__ */ new Map();
-				for await (const record of readStore.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(record.urlHash, record);
-				const livePrior = [...priorByHash.values()].filter((record) => record.removedAt == null);
-				const incomingByHash = /* @__PURE__ */ new Map();
-				for (const url of urls) incomingByHash.set(hash(url.loc), url);
-				const deltaRows = [];
-				let added = 0;
-				let removed = 0;
-				let kept = 0;
-				for (const [urlHash, url] of incomingByHash) {
-					const previous = priorByHash.get(urlHash);
-					if (!previous || previous.removedAt != null) {
-						added++;
-						deltaRows.push({
-							feedpath,
-							feedpath_hash: feedpathHash,
-							url_hash: urlHash,
-							op: "added",
-							loc: url.loc,
-							lastmod: url.lastmod ?? null,
-							at: generation.observedAt,
-							generation_id: generation.id
-						});
-					} else kept++;
-				}
-				for (const [urlHash, previous] of priorByHash) if (previous.removedAt == null && !incomingByHash.has(urlHash)) {
-					removed++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: feedpathHash,
-						url_hash: urlHash,
-						op: "removed",
-						loc: previous.loc,
-						lastmod: previous.lastmod ?? null,
-						at: generation.observedAt,
-						generation_id: generation.id
-					});
-				}
-				const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
-				const seed = await readJson(seedKey);
-				const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
-				const seedRows = isSeedGeneration ? livePrior.map((record) => ({
-					feedpath,
-					feedpath_hash: feedpathHash,
-					url_hash: record.urlHash,
-					op: "added",
-					loc: record.loc,
-					lastmod: record.lastmod ?? null
-				})) : [];
-				const actualSequence = seedRows.length > 0 ? 1 : 0;
-				const inputCount = incomingByHash.size;
-				const persistedEvents = await ensureEvents(ctx, feedpathHash, generation, [...seedRows.map((row) => eventRow(generation, row, {
-					sequence: 0,
-					projectsState: false,
-					seedGeneration: isSeedGeneration,
-					generationKind: "snapshot",
-					contentHash,
-					inputCount
-				})), ...deltaRows.map((row) => eventRow(generation, row, {
-					sequence: actualSequence,
-					projectsState: true,
-					seedGeneration: isSeedGeneration,
-					generationKind: "snapshot",
-					contentHash,
-					inputCount
-				}))]);
-				const result = {
-					added,
-					removed,
-					kept,
-					contentHash,
-					unchanged: deltaRows.length === 0
-				};
-				if (persistedEvents.rows.length > 0) {
-					if ((await projectEvents(ctx, feedpathHash, persistedEvents.rows, persistedEvents.digest))._tag !== "snapshot") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected snapshot event`);
-					await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
-				} else {
-					if (isSeedGeneration) await writeJson(seedKey, {
-						version: 1,
-						generationId: generation.id,
-						observedAt: generation.observedAt
-					});
-					await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), {
-						_tag: "snapshot",
-						version: 1,
-						generationId: generation.id,
-						observedAt: generation.observedAt,
-						eventDigest: persistedEvents.digest,
-						result
-					});
-				}
-				return result;
-			});
-		},
-		compactUrls(ctx, compactOpts = {}) {
-			return withMutation(ctx, async () => {
-				const startedAt = now();
-				const deadlineMs = compactOpts.deadlineMs ?? Number.POSITIVE_INFINITY;
-				const maxFeedpaths = compactOpts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
-				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
-					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
-					if (feedpathHash) await repairFeedProjection(ctx, feedpathHash);
-				}
-				const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
-				let projectionManifest = await readProjectionManifest(ctx);
-				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, startedAt);
-				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
-				const deltasByFeed = /* @__PURE__ */ new Map();
-				for (const key of activeDeltaKeys) {
-					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
-					if (!feedpathHash) continue;
-					const keys = deltasByFeed.get(feedpathHash) ?? [];
-					keys.push(key);
-					deltasByFeed.set(feedpathHash, keys);
-				}
-				const totalFeedpaths = deltasByFeed.size;
-				let compactedFeedpaths = 0;
-				for (const [feedpathHash, feedDeltaKeys] of deltasByFeed) {
-					if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
-					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, feedDeltaKeys)]);
-					const state = createSitemapUrlState(indexRows);
-					applySitemapDeltaFiles(state, deltaFiles);
-					projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
-					compactedFeedpaths++;
-				}
-				return {
-					compactedFeedpaths,
-					remainingFeedpaths: totalFeedpaths - compactedFeedpaths
-				};
-			});
-		},
-		reconcile(ctx, generation, { liveFeedpaths }) {
-			return withMutation(ctx, async () => {
-				const liveHashes = new Set(liveFeedpaths.map((feedpath) => hash(feedpath)));
-				const inputDigest = hashSortedUrlList([...liveHashes]);
-				const siteCheckpoint = await readJson(sitemapUrlsReconcileGenerationKey(ctx));
-				if (assertGenerationAccepted(generation, siteCheckpoint, "site reconciliation") === "same") {
-					if (siteCheckpoint?.inputDigest !== inputDigest) throw new Error("sitemap generation conflict: reconcile input changed");
-					return {
-						feedpathsPruned: 0,
-						urlsRemoved: 0
-					};
-				}
-				const pendingFeedHashes = /* @__PURE__ */ new Set();
-				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
-					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
-					if (feedpathHash) pendingFeedHashes.add(feedpathHash);
-				}
-				const repairedCheckpoints = /* @__PURE__ */ new Map();
-				for (const feedpathHash of pendingFeedHashes) repairedCheckpoints.set(feedpathHash, await repairFeedProjection(ctx, feedpathHash));
-				const present = /* @__PURE__ */ new Set();
-				const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
-				let projectionManifest = await readProjectionManifest(ctx);
-				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, now());
-				for (const feedpathHash of Object.keys(projectionManifest.feeds)) present.add(feedpathHash);
-				for (const key of indexKeys) {
-					const match = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
-					if (match?.[1]) present.add(match[1]);
-				}
-				const deltasByFeed = /* @__PURE__ */ new Map();
-				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
-				for (const key of activeDeltaKeys) {
-					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
-					if (!feedpathHash) continue;
-					present.add(feedpathHash);
-					const keys = deltasByFeed.get(feedpathHash) ?? [];
-					keys.push(key);
-					deltasByFeed.set(feedpathHash, keys);
-				}
-				let feedpathsPruned = 0;
-				let urlsRemoved = 0;
-				for (const feedpathHash of present) {
-					if (liveHashes.has(feedpathHash)) continue;
-					const checkpoint = repairedCheckpoints.has(feedpathHash) ? repairedCheckpoints.get(feedpathHash) : await repairFeedProjection(ctx, feedpathHash);
-					const feedPosition = assertGenerationAccepted(generation, checkpoint, feedpathHash);
-					if (feedPosition === "same" && checkpoint?._tag !== "reconcile") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected reconcile event`);
-					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, deltasByFeed.get(feedpathHash) ?? [])]);
-					const state = createSitemapUrlState(indexRows);
-					applySitemapDeltaFiles(state, deltaFiles);
-					const live = [...state.live.values()];
-					const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
-					const seed = await readJson(seedKey);
-					const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
-					const seedRows = isSeedGeneration ? live.map((record) => ({
-						feedpath: record.feedpath,
-						feedpath_hash: feedpathHash,
-						url_hash: record.urlHash,
-						op: "added",
-						loc: record.loc,
-						lastmod: record.lastmod ?? null
-					})) : [];
-					const removalRows = live.map((record) => ({
-						feedpath: record.feedpath,
-						feedpath_hash: feedpathHash,
-						url_hash: record.urlHash,
-						op: "removed",
-						loc: record.loc,
-						lastmod: record.lastmod ?? null
-					}));
-					const proposedEvents = feedPosition === "newer" ? [...seedRows.map((row) => eventRow(generation, row, {
-						sequence: 0,
-						projectsState: false,
-						seedGeneration: isSeedGeneration,
-						generationKind: "reconcile"
-					})), ...removalRows.map((row) => eventRow(generation, row, {
-						sequence: seedRows.length > 0 ? 1 : 0,
-						projectsState: true,
-						seedGeneration: isSeedGeneration,
-						generationKind: "reconcile"
-					}))] : [];
-					const persistedEvents = feedPosition === "newer" ? await ensureEvents(ctx, feedpathHash, generation, proposedEvents) : {
-						rows: [],
-						digest: checkpoint.eventDigest
-					};
-					if (isSeedGeneration && feedPosition === "newer") await writeJson(seedKey, {
-						version: 1,
-						generationId: generation.id,
-						observedAt: generation.observedAt
-					});
-					const removedHashes = new Set(persistedEvents.rows.filter((row) => Boolean(row.projects_state) && String(row.op) === "removed").map((row) => String(row.url_hash)));
-					const removedBeforeFeed = urlsRemoved;
-					for (const record of live.filter((record) => removedHashes.has(record.urlHash))) {
-						state.live.delete(record.urlHash);
-						state.removed.set(record.urlHash, {
-							...record,
-							removedAt: generation.observedAt
-						});
-						urlsRemoved++;
-					}
-					if (live.length > 0 || deltaFiles.length > 0) projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
-					if (feedPosition === "newer") {
-						const feedCheckpoint = persistedEvents.rows.length > 0 ? checkpointFromEvents(persistedEvents.rows, persistedEvents.digest) : {
-							_tag: "reconcile",
-							version: 1,
-							generationId: generation.id,
-							observedAt: generation.observedAt,
-							eventDigest: persistedEvents.digest
-						};
-						await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), feedCheckpoint);
-						if (persistedEvents.rows.length > 0) await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
-					}
-					if (urlsRemoved > removedBeforeFeed) feedpathsPruned++;
-				}
-				await writeJson(sitemapUrlsReconcileGenerationKey(ctx), {
-					version: 1,
-					generationId: generation.id,
-					observedAt: generation.observedAt,
-					inputDigest
-				});
-				return {
-					feedpathsPruned,
-					urlsRemoved
-				};
-			});
-		}
-	};
-}
-function createIndexingMetadataStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	async function readIndex(key) {
-		const bytes = await readOptional(ds, key);
-		if (bytes === void 0) return {
-			version: 1,
-			records: {}
-		};
-		return JSON.parse(new TextDecoder().decode(bytes));
-	}
-	return {
-		async writeBatch(ctx, records) {
-			if (records.length === 0) return;
-			const key = indexingMetadataIndexKey(ctx);
-			const index = await readIndex(key);
-			for (const r of records) index.records[hash(r.url)] = r;
-			await ds.write(key, encodeJsonBigintSafe(index));
-		},
-		async loadIndex(ctx) {
-			return readIndex(indexingMetadataIndexKey(ctx));
-		},
-		async getLatest(ctx, url) {
-			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
-		}
-	};
-}
-function createEmptyTypesStore(opts) {
-	const ds = opts.dataSource;
-	const now = opts.now ?? (() => Date.now());
-	async function readDoc(key) {
-		const bytes = await readOptional(ds, key);
-		if (bytes === void 0) return {
-			version: 1,
-			emptyTypes: [],
-			markedAt: {}
-		};
-		return JSON.parse(new TextDecoder().decode(bytes));
-	}
-	async function writeDoc(key, doc) {
-		await ds.write(key, encodeJsonBigintSafe(doc));
-	}
-	return {
-		async load(ctx) {
-			return readDoc(emptyTypesKey(ctx));
-		},
-		async mark(ctx, types, at) {
-			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
-			const key = emptyTypesKey(ctx);
-			const doc = await readDoc(key);
-			const stamp = at ?? now();
-			let changed = false;
-			for (const t of types) {
-				if (!doc.emptyTypes.includes(t)) {
-					doc.emptyTypes.push(t);
-					changed = true;
-				}
-				if (doc.markedAt[t] === void 0) {
-					doc.markedAt[t] = stamp;
-					changed = true;
-				}
-			}
-			if (changed) {
-				doc.emptyTypes.sort();
-				await writeDoc(key, doc);
-			}
-			return doc;
-		},
-		async clear(ctx, types) {
-			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
-			const key = emptyTypesKey(ctx);
-			const doc = await readDoc(key);
-			const drop = new Set(types);
-			const before = doc.emptyTypes.length;
-			doc.emptyTypes = doc.emptyTypes.filter((t) => !drop.has(t));
-			for (const t of drop) delete doc.markedAt[t];
-			if (doc.emptyTypes.length !== before) await writeDoc(key, doc);
-			return doc;
-		}
-	};
-}
+import { createEmptyTypesStore } from "./entities/empty-types.mjs";
+import { createIndexingMetadataStore } from "./entities/indexing-metadata.mjs";
+import { INSPECTION_HISTORY_MAX_BYTES, createInspectionStore } from "./entities/inspection.mjs";
+import { buildQueryDimRecords, createQueryDimStore } from "./entities/query-dim.mjs";
+import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./entities/sitemap-projection.mjs";
+import { createSitemapReadStore } from "./entities/sitemap.mjs";
+import { createSitemapStore } from "./entities/sitemap-write.mjs";
 export { INSPECTION_HISTORY_MAX_BYTES, SITEMAP_PROJECTION_GRACE_MS, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };