@gscdump/engine 1.4.3 → 1.4.4

package/dist/entities.mjs

@@ -1,6 +1,7 @@
-import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
 import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { readOptional } from "./adapters/read-optional.mjs";
+import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
 import { buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
@@ -315,7 +316,8 @@ function createInspectionStore(opts) {
 		}
 	};
 }
-const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
+const SITEMAP_URLS_EVENT_PREFIX_RE = /\/urls\/events\/(\d{4}-\d{2}-\d{2})__[0-9a-f]+__\d+__[0-9a-f]+\.parquet$/;
+const SITEMAP_URLS_PENDING_GENERATION_RE = /\/urls\/generations\/pending\/([0-9a-f]+)\.json$/;
 const URLS_INDEX_COLUMNS = [
 	{
 		name: "feedpath",
@@ -393,6 +395,83 @@ const URLS_DELTA_COLUMNS = [
 		name: "at",
 		type: "BIGINT",
 		nullable: false
+	},
+	{
+		name: "generation_id",
+		type: "VARCHAR",
+		nullable: true
+	}
+];
+const URLS_EVENT_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "op",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "generation_id",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "observed_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "sequence",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "projects_state",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "seed_generation",
+		type: "INTEGER",
+		nullable: false
+	},
+	{
+		name: "generation_kind",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "content_hash",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "input_count",
+		type: "INTEGER",
+		nullable: true
 	}
 ];
 function rowToUrlRecord(row) {
@@ -419,44 +498,71 @@ function urlRecordToRow(r) {
 		removed_at: r.removedAt ?? null
 	};
 }
-function isoDate(ms) {
-	return new Date(ms).toISOString().slice(0, 10);
+function createSitemapUrlState(indexRows) {
+	const state = {
+		live: /* @__PURE__ */ new Map(),
+		removed: /* @__PURE__ */ new Map()
+	};
+	for (const row of indexRows) {
+		const record = rowToUrlRecord(row);
+		if (record.removedAt == null) state.live.set(record.urlHash, record);
+		else state.removed.set(record.urlHash, record);
+	}
+	return state;
 }
-function createSitemapStore(opts) {
+function applySitemapDeltaRows(state, rows) {
+	for (const row of rows) {
+		const urlHash = String(row.url_hash);
+		const observedAt = Number(row.at);
+		const op = String(row.op);
+		if (op === "added") {
+			const previous = state.live.get(urlHash) ?? state.removed.get(urlHash);
+			state.removed.delete(urlHash);
+			state.live.set(urlHash, {
+				feedpath: String(row.feedpath),
+				feedpathHash: String(row.feedpath_hash),
+				urlHash,
+				loc: String(row.loc),
+				lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+				firstSeenAt: previous?.firstSeenAt ?? observedAt,
+				lastSeenAt: observedAt
+			});
+		} else if (op === "removed") {
+			const previous = state.live.get(urlHash);
+			state.live.delete(urlHash);
+			if (previous) state.removed.set(urlHash, {
+				...previous,
+				removedAt: observedAt
+			});
+		}
+	}
+}
+function applySitemapDeltaFiles(state, files) {
+	for (const file of files) applySitemapDeltaRows(state, file.rows);
+}
+function sortSitemapDeltaFiles(files) {
+	return files.sort((a, b) => a.key.localeCompare(b.key));
+}
+async function readSitemapDeltaFiles(ds, keys) {
+	return sortSitemapDeltaFiles((await mapEntityIo(keys, async (key) => {
+		const bytes = await readOptional(ds, key);
+		return bytes ? {
+			key,
+			rows: await decodeParquetToRows(bytes)
+		} : void 0;
+	})).filter((file) => file !== void 0));
+}
+function dateInRange(date, range) {
+	return (!range?.from || date >= range.from) && (!range?.to || date <= range.to);
+}
+function createSitemapReadStore(opts) {
 	const ds = opts.dataSource;
 	const hash = opts.hash ?? hashUrl;
-	const now = opts.now ?? (() => Date.now());
 	async function readJson(key) {
 		const bytes = await readOptional(ds, key);
-		if (bytes === void 0) return void 0;
-		return JSON.parse(new TextDecoder().decode(bytes));
-	}
-	async function writeJson(key, value) {
-		await ds.write(key, encodeJsonBigintSafe(value));
+		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
 	}
 	return {
-		async writeSnapshot(ctx, records) {
-			if (records.length === 0) return;
-			const indexKey = sitemapIndexKey(ctx);
-			const index = await readJson(indexKey) ?? {
-				version: 1,
-				records: {}
-			};
-			const stamp = now();
-			const historyDocs = /* @__PURE__ */ new Map();
-			for (const r of records) {
-				const h = hash(r.path);
-				index.records[h] = r;
-				historyDocs.set(sitemapHistoryKey(ctx, h, stamp), {
-					version: 1,
-					path: r.path,
-					capturedAt: r.capturedAt,
-					record: r
-				});
-			}
-			await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
-			await writeJson(indexKey, index);
-		},
 		async loadIndex(ctx) {
 			return await readJson(sitemapIndexKey(ctx)) ?? {
 				version: 1,
@@ -466,322 +572,589 @@ function createSitemapStore(opts) {
 		async getLatest(ctx, path) {
 			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
 		},
-		async snapshotUrls(ctx, feedpath, urls) {
-			const fpHash = hash(feedpath);
-			const contentHash = hashUrlList(urls);
-			const at = now();
-			const priorByHash = /* @__PURE__ */ new Map();
-			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
-			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
-			if (livePrior.length > 0) {
-				if (hashSortedUrlList(livePrior.map((r) => String(r.loc)).sort()) === contentHash) return {
-					added: 0,
-					removed: 0,
-					kept: livePrior.length,
-					contentHash,
-					unchanged: true
-				};
-			}
-			const incomingByHash = /* @__PURE__ */ new Map();
-			for (const u of urls) incomingByHash.set(hash(u.loc), u);
-			const deltaRows = [];
-			let added = 0;
-			let removed = 0;
-			let kept = 0;
-			const date = isoDate(at);
-			for (const [urlHash, u] of incomingByHash) {
-				const prev = priorByHash.get(urlHash);
-				if (!prev || prev.removedAt != null) {
-					added++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: fpHash,
-						url_hash: urlHash,
-						op: "added",
-						loc: u.loc,
-						lastmod: u.lastmod ?? null,
-						at
-					});
-				} else kept++;
-			}
-			for (const [urlHash, prev] of priorByHash) {
-				if (prev.removedAt != null) continue;
-				if (!incomingByHash.has(urlHash)) {
-					removed++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: fpHash,
-						url_hash: urlHash,
-						op: "removed",
-						loc: prev.loc,
-						lastmod: prev.lastmod ?? null,
-						at
-					});
-				}
-			}
-			if (deltaRows.length > 0) {
-				const bytes = encodeRowsToParquetFlex(deltaRows, {
-					columns: URLS_DELTA_COLUMNS,
-					sortKey: ["url_hash"]
-				});
-				await ds.write(sitemapUrlsDeltaKey(ctx, fpHash, date), bytes);
-			}
-			return {
-				added,
-				removed,
-				kept,
-				contentHash,
-				unchanged: false
-			};
+		async *loadUrls(ctx, feedpath, loadOpts) {
+			const feedpathHash = hash(feedpath);
+			const listedDeltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).filter((key) => parseSitemapUrlsDeltaKey(key)?.feedpathHash === feedpathHash);
+			const listedDeltaFiles = await readSitemapDeltaFiles(ds, listedDeltaKeys);
+			const manifestBytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
+			const manifest = manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0;
+			const indexRows = await readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []);
+			const currentDeltaKeys = new Set(selectSitemapProjectionFiles([], listedDeltaKeys, manifest).deltaKeys);
+			const state = createSitemapUrlState(indexRows);
+			applySitemapDeltaFiles(state, listedDeltaFiles.filter((file) => currentDeltaKeys.has(file.key)));
+			for (const record of state.live.values()) yield record;
+			if (loadOpts?.includeRemoved) for (const record of state.removed.values()) yield record;
 		},
-		async *loadUrls(ctx, feedpath, opts) {
-			const fpHash = hash(feedpath);
-			const includeRemoved = opts?.includeRemoved ?? false;
-			const [indexRows, listedDeltaKeys] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, fpHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
-			const deltaKeys = listedDeltaKeys.filter((key) => {
-				return SITEMAP_URLS_DELTA_PREFIX_RE.exec(key)?.[2] === fpHash;
-			}).sort();
-			const live = /* @__PURE__ */ new Map();
-			const removedMap = /* @__PURE__ */ new Map();
-			for (const row of indexRows) {
-				const rec = rowToUrlRecord(row);
-				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
-				else live.set(rec.urlHash, rec);
-			}
-			const deltas = await mapEntityIo(deltaKeys, async (key) => {
-				const dBytes = await readOptional(ds, key);
-				if (!dBytes) return [];
-				return decodeParquetToRows(dBytes);
+		async *loadDeltas(ctx, dateRange) {
+			const [listedKeys, manifestBytes] = await Promise.all([ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`), readOptional(ds, sitemapUrlsProjectionManifestKey(ctx))]);
+			const keys = selectSitemapProjectionFiles([], listedKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0).deltaKeys.filter((key) => {
+				const parsed = parseSitemapUrlsDeltaKey(key);
+				return Boolean(parsed && dateInRange(parsed.date, dateRange));
 			});
-			for (const dRows of deltas) for (const r of dRows) {
-				const op = String(r.op);
-				const urlHash = String(r.url_hash);
-				const at = Number(r.at);
-				if (op === "added") {
-					const prev = live.get(urlHash) ?? removedMap.get(urlHash);
-					removedMap.delete(urlHash);
-					live.set(urlHash, {
-						feedpath,
-						feedpathHash: fpHash,
-						urlHash,
-						loc: String(r.loc),
-						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-						firstSeenAt: prev?.firstSeenAt ?? at,
-						lastSeenAt: at
-					});
-				} else if (op === "removed") {
-					const prev = live.get(urlHash);
-					live.delete(urlHash);
-					if (prev) removedMap.set(urlHash, {
-						...prev,
-						removedAt: at
-					});
-				}
+			const files = await readSitemapDeltaFiles(ds, keys);
+			for (const file of files) for (const row of file.rows) {
+				const op = String(row.op);
+				if (op !== "added" && op !== "removed") continue;
+				yield {
+					feedpath: String(row.feedpath),
+					feedpathHash: String(row.feedpath_hash),
+					urlHash: String(row.url_hash),
+					op,
+					loc: String(row.loc),
+					lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+					at: Number(row.at)
+				};
 			}
-			for (const rec of live.values()) yield rec;
-			if (includeRemoved) for (const rec of removedMap.values()) yield rec;
 		},
-		async *loadDeltas(ctx, dateRange) {
-			const from = dateRange?.from;
-			const to = dateRange?.to;
-			const keys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+		async *loadEvents(ctx, dateRange) {
+			const listedEventKeys = await ds.list(`${sitemapUrlsEventsPrefix(ctx)}/`);
+			const pendingKeys = await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`);
+			const pendingEvents = new Set((await mapEntityIo(pendingKeys, (key) => readJson(key))).filter((pending) => pending !== void 0).map((pending) => pending.eventKey));
+			const keys = listedEventKeys.filter((key) => {
+				const match = SITEMAP_URLS_EVENT_PREFIX_RE.exec(key);
+				return !pendingEvents.has(key) && Boolean(match?.[1] && dateInRange(match[1], dateRange));
+			}).sort();
 			for (const key of keys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				const date = m[1];
-				if (!date) continue;
-				if (from && date < from) continue;
-				if (to && date > to) continue;
 				const bytes = await readOptional(ds, key);
 				if (!bytes) continue;
 				const rows = await decodeParquetToRows(bytes);
-				for (const r of rows) {
-					const op = String(r.op);
+				rows.sort((a, b) => Number(a.observed_at) - Number(b.observed_at) || Number(a.sequence) - Number(b.sequence) || String(a.url_hash).localeCompare(String(b.url_hash)));
+				for (const row of rows) {
+					const op = String(row.op);
 					if (op !== "added" && op !== "removed") continue;
 					yield {
-						feedpath: String(r.feedpath),
-						feedpathHash: String(r.feedpath_hash),
-						urlHash: String(r.url_hash),
+						feedpath: String(row.feedpath),
+						feedpathHash: String(row.feedpath_hash),
+						urlHash: String(row.url_hash),
 						op,
-						loc: String(r.loc),
-						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-						at: Number(r.at)
+						loc: String(row.loc),
+						lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+						generationId: String(row.generation_id),
+						observedAt: Number(row.observed_at),
+						sequence: Number(row.sequence),
+						projectsState: Boolean(row.projects_state)
 					};
 				}
 			}
+		}
+	};
+}
+function createSitemapStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	const now = opts.now ?? (() => Date.now());
+	const withMutation = opts.withMutation;
+	const readStore = createSitemapReadStore({
+		dataSource: ds,
+		hash
+	});
+	async function readJson(key) {
+		const bytes = await readOptional(ds, key);
+		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
+	}
+	function writeJson(key, value) {
+		return ds.write(key, encodeJsonBigintSafe(value));
+	}
+	async function readProjectionManifest(ctx) {
+		const bytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
+		return bytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(bytes)) : emptySitemapProjectionManifest();
+	}
+	async function deleteExpiredProjectionDeltas(manifest, deltaKeys, currentTime) {
+		const expired = deltaKeys.filter((key) => {
+			const parsed = parseSitemapUrlsDeltaKey(key);
+			const feed = parsed ? manifest.feeds[parsed.feedpathHash] : void 0;
+			return Boolean(feed && key <= feed.compactedThrough && currentTime - feed.publishedAt >= 9e5);
+		});
+		if (expired.length > 0) await ds.delete(expired);
+	}
+	async function publishCurrentProjection(ctx, feedpathHash, rows, consumedDeltas, manifest, publishedAt) {
+		const bytes = encodeRowsToParquetFlex(rows.map(urlRecordToRow), {
+			columns: URLS_INDEX_COLUMNS,
+			sortKey: ["feedpath_hash", "url_hash"]
+		});
+		await ds.write(sitemapUrlsIndexKey(ctx, feedpathHash), bytes);
+		const compactedThrough = consumedDeltas.at(-1)?.key;
+		if (!compactedThrough) return manifest;
+		const next = withSitemapProjectionFeed(manifest, feedpathHash, {
+			compactedThrough,
+			publishedAt
+		});
+		await writeJson(sitemapUrlsProjectionManifestKey(ctx), next);
+		return next;
+	}
+	function normalizedEventRows(rows) {
+		return rows.map((row) => ({
+			feedpath: String(row.feedpath),
+			feedpathHash: String(row.feedpath_hash),
+			urlHash: String(row.url_hash),
+			op: String(row.op),
+			loc: String(row.loc),
+			lastmod: row.lastmod == null ? null : String(row.lastmod),
+			generationId: String(row.generation_id),
+			observedAt: Number(row.observed_at),
+			sequence: Number(row.sequence),
+			projectsState: Boolean(row.projects_state),
+			seedGeneration: Boolean(row.seed_generation),
+			generationKind: String(row.generation_kind),
+			contentHash: row.content_hash == null ? null : String(row.content_hash),
+			inputCount: row.input_count == null ? null : Number(row.input_count)
+		})).sort((a, b) => a.observedAt - b.observedAt || a.sequence - b.sequence || a.urlHash.localeCompare(b.urlHash) || a.op.localeCompare(b.op));
+	}
+	function eventDigest(rows) {
+		return hashUrl(JSON.stringify(normalizedEventRows(rows)));
+	}
+	function assertGenerationAccepted(generation, checkpoint, scope) {
+		if (!checkpoint) return "newer";
+		if (generation.id === checkpoint.generationId) {
+			if (generation.observedAt === checkpoint.observedAt) return "same";
+			throw new Error(`sitemap generation conflict for ${scope}: generation ${generation.id} changed observedAt`);
+		}
+		if (generation.observedAt > checkpoint.observedAt) return "newer";
+		const reason = generation.observedAt === checkpoint.observedAt ? "ambiguous timestamp" : "stale generation";
+		throw new Error(`sitemap generation conflict for ${scope}: ${reason}`);
+	}
+	async function ensureEvents(ctx, feedpathHash, generation, rows) {
+		const expectedDigest = eventDigest(rows);
+		if (rows.length === 0) return {
+			rows: [],
+			digest: expectedDigest
+		};
+		const key = sitemapUrlsEventKey(ctx, feedpathHash, generation);
+		await writeJson(sitemapUrlsPendingGenerationKey(ctx, feedpathHash), {
+			version: 1,
+			generationId: generation.id,
+			observedAt: generation.observedAt,
+			eventKey: key,
+			eventDigest: expectedDigest
+		});
+		const existing = await readOptional(ds, key);
+		if (existing) {
+			const existingRows = await decodeParquetToRows(existing);
+			const existingDigest = eventDigest(existingRows);
+			if (existingDigest !== expectedDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: immutable event digest changed`);
+			return {
+				rows: existingRows,
+				digest: existingDigest
+			};
+		}
+		const bytes = encodeRowsToParquetFlex(rows, {
+			columns: URLS_EVENT_COLUMNS,
+			sortKey: ["sequence", "url_hash"]
+		});
+		await ds.write(key, bytes);
+		return {
+			rows: [...rows],
+			digest: expectedDigest
+		};
+	}
+	function eventRow(generation, row, metadata) {
+		return {
+			feedpath: row.feedpath,
+			feedpath_hash: row.feedpath_hash,
+			url_hash: row.url_hash,
+			op: row.op,
+			loc: row.loc,
+			lastmod: row.lastmod,
+			generation_id: generation.id,
+			observed_at: generation.observedAt,
+			sequence: metadata.sequence,
+			projects_state: metadata.projectsState ? 1 : 0,
+			seed_generation: metadata.seedGeneration ? 1 : 0,
+			generation_kind: metadata.generationKind,
+			content_hash: metadata.contentHash ?? null,
+			input_count: metadata.inputCount ?? null
+		};
+	}
+	function stateRowsFromEvents(rows) {
+		return rows.filter((row) => Boolean(row.projects_state)).map((row) => ({
+			feedpath: row.feedpath,
+			feedpath_hash: row.feedpath_hash,
+			url_hash: row.url_hash,
+			op: row.op,
+			loc: row.loc,
+			lastmod: row.lastmod,
+			at: row.observed_at,
+			generation_id: row.generation_id
+		}));
+	}
+	function checkpointFromEvents(rows, digest) {
+		const first = rows[0];
+		if (!first) throw new Error("cannot checkpoint an empty sitemap event file");
+		const generationId = String(first.generation_id);
+		const observedAt = Number(first.observed_at);
+		for (const row of rows) if (String(row.generation_id) !== generationId || Number(row.observed_at) !== observedAt) throw new Error("sitemap event file contains multiple generations");
+		if (String(first.generation_kind) === "snapshot") {
+			const contentHash = String(first.content_hash);
+			const inputCount = Number(first.input_count);
+			const stateRows = stateRowsFromEvents(rows);
+			const added = stateRows.filter((row) => String(row.op) === "added").length;
+			return {
+				_tag: "snapshot",
+				version: 1,
+				generationId,
+				observedAt,
+				eventDigest: digest,
+				result: {
+					added,
+					removed: stateRows.filter((row) => String(row.op) === "removed").length,
+					kept: Math.max(0, inputCount - added),
+					contentHash,
+					unchanged: stateRows.length === 0
+				}
+			};
+		}
+		return {
+			_tag: "reconcile",
+			version: 1,
+			generationId,
+			observedAt,
+			eventDigest: digest
+		};
+	}
+	async function projectEvents(ctx, feedpathHash, rows, digest) {
+		const checkpoint = checkpointFromEvents(rows, digest);
+		const generation = {
+			id: checkpoint.generationId,
+			observedAt: checkpoint.observedAt
+		};
+		const stateRows = stateRowsFromEvents(rows);
+		if (stateRows.length > 0) {
+			const bytes = encodeRowsToParquetFlex(stateRows, {
+				columns: URLS_DELTA_COLUMNS,
+				sortKey: ["url_hash"]
+			});
+			await ds.write(sitemapUrlsDeltaKey(ctx, feedpathHash, generation), bytes);
+		}
+		if (rows.some((row) => Boolean(row.seed_generation))) await writeJson(sitemapUrlsEventSeedKey(ctx, feedpathHash), {
+			version: 1,
+			generationId: checkpoint.generationId,
+			observedAt: checkpoint.observedAt
+		});
+		await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), checkpoint);
+		return checkpoint;
+	}
+	async function repairFeedProjection(ctx, feedpathHash) {
+		const pendingKey = sitemapUrlsPendingGenerationKey(ctx, feedpathHash);
+		const [checkpoint, pending] = await Promise.all([readJson(sitemapUrlsGenerationKey(ctx, feedpathHash)), readJson(pendingKey)]);
+		if (!pending) return checkpoint;
+		const pendingGeneration = {
+			_tag: "complete",
+			id: pending.generationId,
+			observedAt: pending.observedAt
+		};
+		if (checkpoint?.generationId === pending.generationId && checkpoint.observedAt !== pending.observedAt) throw new Error(`sitemap generation conflict for ${feedpathHash}: generation ${pending.generationId} changed observedAt`);
+		if (checkpoint && pending.observedAt < checkpoint.observedAt) {
+			await ds.delete([pendingKey]);
+			return checkpoint;
+		}
+		const position = assertGenerationAccepted(pendingGeneration, checkpoint, feedpathHash);
+		const eventBytes = await readOptional(ds, pending.eventKey);
+		if (!eventBytes) {
+			await ds.delete([pendingKey]);
+			return checkpoint;
+		}
+		const rows = await decodeParquetToRows(eventBytes);
+		const digest = eventDigest(rows);
+		if (digest !== pending.eventDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: pending event digest changed`);
+		if (position === "same") {
+			if (checkpoint?.eventDigest !== digest) throw new Error(`sitemap generation conflict for ${feedpathHash}: checkpoint digest changed`);
+			await ds.delete([pendingKey]);
+			return checkpoint;
+		}
+		const repaired = await projectEvents(ctx, feedpathHash, rows, digest);
+		await ds.delete([pendingKey]);
+		return repaired;
+	}
+	return {
+		...readStore,
+		writeSnapshot(ctx, records) {
+			return withMutation(ctx, async () => {
+				if (records.length === 0) return;
+				const indexKey = sitemapIndexKey(ctx);
+				const index = await readJson(indexKey) ?? {
+					version: 1,
+					records: {}
+				};
+				const stamp = now();
+				const historyDocs = /* @__PURE__ */ new Map();
+				for (const record of records) {
+					const feedpathHash = hash(record.path);
+					index.records[feedpathHash] = record;
+					historyDocs.set(sitemapHistoryKey(ctx, feedpathHash, stamp), {
+						version: 1,
+						path: record.path,
+						capturedAt: record.capturedAt,
+						record
+					});
+				}
+				await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
+				await writeJson(indexKey, index);
+			});
 		},
-		async compactUrls(ctx, opts = {}) {
-			const startedAt = now();
-			const deadlineMs = opts.deadlineMs ?? Number.POSITIVE_INFINITY;
-			const maxFeedpaths = opts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
-			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
-			const deltasByFeed = /* @__PURE__ */ new Map();
-			for (const key of deltaKeys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				const feedpathHash = m[2];
-				if (!feedpathHash) continue;
-				const list = deltasByFeed.get(feedpathHash) ?? [];
-				list.push(key);
-				deltasByFeed.set(feedpathHash, list);
-			}
-			const totalFeedpaths = deltasByFeed.size;
-			let compactedFeedpaths = 0;
-			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
-				if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
-				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
-				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo(feedDeltaKeys.sort(), async (key) => {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) return void 0;
-					return {
-						key,
-						rows: await decodeParquetToRows(bytes)
-					};
-				})]);
-				const live = /* @__PURE__ */ new Map();
-				const removed = /* @__PURE__ */ new Map();
-				for (const row of indexRows) {
-					const rec = rowToUrlRecord(row);
-					if (rec.removedAt != null) removed.set(rec.urlHash, rec);
-					else live.set(rec.urlHash, rec);
+		snapshotUrls(ctx, generation, feedpath, urls) {
+			return withMutation(ctx, async () => {
+				const feedpathHash = hash(feedpath);
+				const contentHash = hashUrlList(urls);
+				const checkpoint = await repairFeedProjection(ctx, feedpathHash);
+				if (assertGenerationAccepted(generation, checkpoint, feedpathHash) === "same") {
+					if (checkpoint?._tag !== "snapshot" || checkpoint.result.contentHash !== contentHash) throw new Error(`sitemap generation conflict for ${feedpathHash}: snapshot input changed`);
+					return checkpoint.result;
 				}
-				const consumed = [];
-				for (const file of deltaFiles) {
-					if (!file) continue;
-					const { key, rows } = file;
-					consumed.push(key);
-					for (const r of rows) {
-						const urlHash = String(r.url_hash);
-						const at = Number(r.at);
-						const op = String(r.op);
-						if (op === "added") {
-							const prev = live.get(urlHash) ?? removed.get(urlHash);
-							removed.delete(urlHash);
-							live.set(urlHash, {
-								feedpath: String(r.feedpath),
-								feedpathHash: fpHash,
-								urlHash,
-								loc: String(r.loc),
-								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-								firstSeenAt: prev?.firstSeenAt ?? at,
-								lastSeenAt: at
-							});
-						} else if (op === "removed") {
-							const prev = live.get(urlHash);
-							live.delete(urlHash);
-							if (prev) removed.set(urlHash, {
-								...prev,
-								removedAt: at
-							});
-						}
-					}
+				if (assertGenerationAccepted(generation, await readJson(sitemapUrlsReconcileGenerationKey(ctx)), "site reconciliation") === "same") throw new Error("sitemap generation conflict: site generation was already reconciled");
+				const priorByHash = /* @__PURE__ */ new Map();
+				for await (const record of readStore.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(record.urlHash, record);
+				const livePrior = [...priorByHash.values()].filter((record) => record.removedAt == null);
+				const incomingByHash = /* @__PURE__ */ new Map();
+				for (const url of urls) incomingByHash.set(hash(url.loc), url);
+				const deltaRows = [];
+				let added = 0;
+				let removed = 0;
+				let kept = 0;
+				for (const [urlHash, url] of incomingByHash) {
+					const previous = priorByHash.get(urlHash);
+					if (!previous || previous.removedAt != null) {
+						added++;
+						deltaRows.push({
+							feedpath,
+							feedpath_hash: feedpathHash,
+							url_hash: urlHash,
+							op: "added",
+							loc: url.loc,
+							lastmod: url.lastmod ?? null,
+							at: generation.observedAt,
+							generation_id: generation.id
+						});
+					} else kept++;
 				}
-				const merged = [...live.values(), ...removed.values()];
-				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
-					columns: URLS_INDEX_COLUMNS,
-					sortKey: ["feedpath_hash", "url_hash"]
-				});
-				await ds.write(indexKey, bytes);
-				if (consumed.length > 0) await ds.delete(consumed);
-				compactedFeedpaths++;
-			}
-			return {
-				compactedFeedpaths,
-				remainingFeedpaths: totalFeedpaths - compactedFeedpaths
-			};
+				for (const [urlHash, previous] of priorByHash) if (previous.removedAt == null && !incomingByHash.has(urlHash)) {
+					removed++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: feedpathHash,
+						url_hash: urlHash,
+						op: "removed",
+						loc: previous.loc,
+						lastmod: previous.lastmod ?? null,
+						at: generation.observedAt,
+						generation_id: generation.id
+					});
+				}
+				const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
+				const seed = await readJson(seedKey);
+				const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
+				const seedRows = isSeedGeneration ? livePrior.map((record) => ({
+					feedpath,
+					feedpath_hash: feedpathHash,
+					url_hash: record.urlHash,
+					op: "added",
+					loc: record.loc,
+					lastmod: record.lastmod ?? null
+				})) : [];
+				const actualSequence = seedRows.length > 0 ? 1 : 0;
+				const inputCount = incomingByHash.size;
+				const persistedEvents = await ensureEvents(ctx, feedpathHash, generation, [...seedRows.map((row) => eventRow(generation, row, {
+					sequence: 0,
+					projectsState: false,
+					seedGeneration: isSeedGeneration,
+					generationKind: "snapshot",
+					contentHash,
+					inputCount
+				})), ...deltaRows.map((row) => eventRow(generation, row, {
+					sequence: actualSequence,
+					projectsState: true,
+					seedGeneration: isSeedGeneration,
+					generationKind: "snapshot",
+					contentHash,
+					inputCount
+				}))]);
+				const result = {
+					added,
+					removed,
+					kept,
+					contentHash,
+					unchanged: deltaRows.length === 0
+				};
+				if (persistedEvents.rows.length > 0) {
+					if ((await projectEvents(ctx, feedpathHash, persistedEvents.rows, persistedEvents.digest))._tag !== "snapshot") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected snapshot event`);
+					await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
+				} else {
+					if (isSeedGeneration) await writeJson(seedKey, {
+						version: 1,
+						generationId: generation.id,
+						observedAt: generation.observedAt
+					});
+					await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), {
+						_tag: "snapshot",
+						version: 1,
+						generationId: generation.id,
+						observedAt: generation.observedAt,
+						eventDigest: persistedEvents.digest,
+						result
+					});
+				}
+				return result;
+			});
 		},
-		async reconcile(ctx, { liveFeedpaths, at: atOpt }) {
-			const at = atOpt ?? now();
-			const liveHashes = new Set(liveFeedpaths.map((fp) => hash(fp)));
-			const present = /* @__PURE__ */ new Set();
-			const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
-			for (const key of indexKeys) {
-				const m = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
-				if (m) present.add(m[1]);
-			}
-			const deltasByFeed = /* @__PURE__ */ new Map();
-			for (const key of deltaKeys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				present.add(m[2]);
-				const list = deltasByFeed.get(m[2]) ?? [];
-				list.push(key);
-				deltasByFeed.set(m[2], list);
-			}
-			let feedpathsPruned = 0;
-			let urlsRemoved = 0;
-			for (const fpHash of present) {
-				if (liveHashes.has(fpHash)) continue;
-				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
-				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo((deltasByFeed.get(fpHash) ?? []).sort(), async (key) => {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) return void 0;
+		compactUrls(ctx, compactOpts = {}) {
+			return withMutation(ctx, async () => {
+				const startedAt = now();
+				const deadlineMs = compactOpts.deadlineMs ?? Number.POSITIVE_INFINITY;
+				const maxFeedpaths = compactOpts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
+				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
+					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
+					if (feedpathHash) await repairFeedProjection(ctx, feedpathHash);
+				}
+				const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
+				let projectionManifest = await readProjectionManifest(ctx);
+				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, startedAt);
+				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
+				const deltasByFeed = /* @__PURE__ */ new Map();
+				for (const key of activeDeltaKeys) {
+					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
+					if (!feedpathHash) continue;
+					const keys = deltasByFeed.get(feedpathHash) ?? [];
+					keys.push(key);
+					deltasByFeed.set(feedpathHash, keys);
+				}
+				const totalFeedpaths = deltasByFeed.size;
+				let compactedFeedpaths = 0;
+				for (const [feedpathHash, feedDeltaKeys] of deltasByFeed) {
+					if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
+					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, feedDeltaKeys)]);
+					const state = createSitemapUrlState(indexRows);
+					applySitemapDeltaFiles(state, deltaFiles);
+					projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
+					compactedFeedpaths++;
+				}
+				return {
+					compactedFeedpaths,
+					remainingFeedpaths: totalFeedpaths - compactedFeedpaths
+				};
+			});
+		},
+		reconcile(ctx, generation, { liveFeedpaths }) {
+			return withMutation(ctx, async () => {
+				const liveHashes = new Set(liveFeedpaths.map((feedpath) => hash(feedpath)));
+				const inputDigest = hashSortedUrlList([...liveHashes]);
+				const siteCheckpoint = await readJson(sitemapUrlsReconcileGenerationKey(ctx));
+				if (assertGenerationAccepted(generation, siteCheckpoint, "site reconciliation") === "same") {
+					if (siteCheckpoint?.inputDigest !== inputDigest) throw new Error("sitemap generation conflict: reconcile input changed");
 					return {
-						key,
-						rows: await decodeParquetToRows(bytes)
+						feedpathsPruned: 0,
+						urlsRemoved: 0
 					};
-				})]);
-				const live = /* @__PURE__ */ new Map();
-				const removed = /* @__PURE__ */ new Map();
-				for (const row of indexRows) {
-					const r = rowToUrlRecord(row);
-					if (r.removedAt != null) removed.set(r.urlHash, r);
-					else live.set(r.urlHash, r);
 				}
-				const consumed = [];
-				for (const file of deltaFiles) {
-					if (!file) continue;
-					const { key, rows } = file;
-					consumed.push(key);
-					for (const r of rows) {
-						const urlHash = String(r.url_hash);
-						const dat = Number(r.at);
-						if (String(r.op) === "added") {
-							const prev = live.get(urlHash) ?? removed.get(urlHash);
-							removed.delete(urlHash);
-							live.set(urlHash, {
-								feedpath: String(r.feedpath),
-								feedpathHash: fpHash,
-								urlHash,
-								loc: String(r.loc),
-								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-								firstSeenAt: prev?.firstSeenAt ?? dat,
-								lastSeenAt: dat
-							});
-						} else if (String(r.op) === "removed") {
-							const prev = live.get(urlHash);
-							live.delete(urlHash);
-							if (prev) removed.set(urlHash, {
-								...prev,
-								removedAt: dat
-							});
-						}
-					}
+				const pendingFeedHashes = /* @__PURE__ */ new Set();
+				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
+					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
+					if (feedpathHash) pendingFeedHashes.add(feedpathHash);
+				}
+				const repairedCheckpoints = /* @__PURE__ */ new Map();
+				for (const feedpathHash of pendingFeedHashes) repairedCheckpoints.set(feedpathHash, await repairFeedProjection(ctx, feedpathHash));
+				const present = /* @__PURE__ */ new Set();
+				const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
+				let projectionManifest = await readProjectionManifest(ctx);
+				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, now());
+				for (const feedpathHash of Object.keys(projectionManifest.feeds)) present.add(feedpathHash);
+				for (const key of indexKeys) {
+					const match = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
+					if (match?.[1]) present.add(match[1]);
 				}
-				const hadLive = live.size > 0;
-				if (!hadLive && consumed.length === 0) continue;
-				for (const [urlHash, r] of live) {
-					removed.set(urlHash, {
-						...r,
-						removedAt: at
+				const deltasByFeed = /* @__PURE__ */ new Map();
+				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
+				for (const key of activeDeltaKeys) {
+					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
+					if (!feedpathHash) continue;
+					present.add(feedpathHash);
+					const keys = deltasByFeed.get(feedpathHash) ?? [];
+					keys.push(key);
+					deltasByFeed.set(feedpathHash, keys);
+				}
+				let feedpathsPruned = 0;
+				let urlsRemoved = 0;
+				for (const feedpathHash of present) {
+					if (liveHashes.has(feedpathHash)) continue;
+					const checkpoint = repairedCheckpoints.has(feedpathHash) ? repairedCheckpoints.get(feedpathHash) : await repairFeedProjection(ctx, feedpathHash);
+					const feedPosition = assertGenerationAccepted(generation, checkpoint, feedpathHash);
+					if (feedPosition === "same" && checkpoint?._tag !== "reconcile") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected reconcile event`);
+					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, deltasByFeed.get(feedpathHash) ?? [])]);
+					const state = createSitemapUrlState(indexRows);
+					applySitemapDeltaFiles(state, deltaFiles);
+					const live = [...state.live.values()];
+					const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
+					const seed = await readJson(seedKey);
+					const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
+					const seedRows = isSeedGeneration ? live.map((record) => ({
+						feedpath: record.feedpath,
+						feedpath_hash: feedpathHash,
+						url_hash: record.urlHash,
+						op: "added",
+						loc: record.loc,
+						lastmod: record.lastmod ?? null
+					})) : [];
+					const removalRows = live.map((record) => ({
+						feedpath: record.feedpath,
+						feedpath_hash: feedpathHash,
+						url_hash: record.urlHash,
+						op: "removed",
+						loc: record.loc,
+						lastmod: record.lastmod ?? null
+					}));
+					const proposedEvents = feedPosition === "newer" ? [...seedRows.map((row) => eventRow(generation, row, {
+						sequence: 0,
+						projectsState: false,
+						seedGeneration: isSeedGeneration,
+						generationKind: "reconcile"
+					})), ...removalRows.map((row) => eventRow(generation, row, {
+						sequence: seedRows.length > 0 ? 1 : 0,
+						projectsState: true,
+						seedGeneration: isSeedGeneration,
+						generationKind: "reconcile"
+					}))] : [];
+					const persistedEvents = feedPosition === "newer" ? await ensureEvents(ctx, feedpathHash, generation, proposedEvents) : {
+						rows: [],
+						digest: checkpoint.eventDigest
+					};
+					if (isSeedGeneration && feedPosition === "newer") await writeJson(seedKey, {
+						version: 1,
+						generationId: generation.id,
+						observedAt: generation.observedAt
 					});
-					urlsRemoved++;
+					const removedHashes = new Set(persistedEvents.rows.filter((row) => Boolean(row.projects_state) && String(row.op) === "removed").map((row) => String(row.url_hash)));
+					const removedBeforeFeed = urlsRemoved;
+					for (const record of live.filter((record) => removedHashes.has(record.urlHash))) {
+						state.live.delete(record.urlHash);
+						state.removed.set(record.urlHash, {
+							...record,
+							removedAt: generation.observedAt
+						});
+						urlsRemoved++;
+					}
+					if (live.length > 0 || deltaFiles.length > 0) projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
+					if (feedPosition === "newer") {
+						const feedCheckpoint = persistedEvents.rows.length > 0 ? checkpointFromEvents(persistedEvents.rows, persistedEvents.digest) : {
+							_tag: "reconcile",
+							version: 1,
+							generationId: generation.id,
+							observedAt: generation.observedAt,
+							eventDigest: persistedEvents.digest
+						};
+						await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), feedCheckpoint);
+						if (persistedEvents.rows.length > 0) await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
+					}
+					if (urlsRemoved > removedBeforeFeed) feedpathsPruned++;
 				}
-				const merged = [...removed.values()];
-				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
-					columns: URLS_INDEX_COLUMNS,
-					sortKey: ["feedpath_hash", "url_hash"]
+				await writeJson(sitemapUrlsReconcileGenerationKey(ctx), {
+					version: 1,
+					generationId: generation.id,
+					observedAt: generation.observedAt,
+					inputDigest
 				});
-				await ds.write(indexKey, bytes);
-				if (consumed.length > 0) await ds.delete(consumed);
-				if (hadLive) feedpathsPruned++;
-			}
-			return {
-				feedpathsPruned,
-				urlsRemoved
-			};
+				return {
+					feedpathsPruned,
+					urlsRemoved
+				};
+			});
 		}
 	};
 }
@@ -866,4 +1239,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };
+export { INSPECTION_HISTORY_MAX_BYTES, SITEMAP_PROJECTION_GRACE_MS, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };