@gscdump/engine 1.5.4 → 2.0.1

package/dist/entities/sitemap-write.mjs

@@ -1,528 +1,9 @@
-import { hashSortedUrlList, hashUrl, hashUrlList, parseSitemapUrlsDeltaKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "../entity-keys.mjs";
-import { readOptional } from "../adapters/read-optional.mjs";
-import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
-import { mapEntityIo } from "./io.mjs";
-import { decodeSitemapProjectionManifest, emptySitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
-import { SITEMAP_URLS_PENDING_GENERATION_RE, URLS_DELTA_COLUMNS, URLS_EVENT_COLUMNS, URLS_INDEX_COLUMNS, applySitemapDeltaFiles, createSitemapUrlState, readSitemapDeltaFiles, urlRecordToRow } from "./sitemap-shared.mjs";
+import { createSitemapGenerationWriteMethods } from "./sitemap-generation.mjs";
 import { createSitemapReadStore } from "./sitemap.mjs";
-import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 function createSitemapStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	const now = opts.now ?? (() => Date.now());
-	const withMutation = opts.withMutation;
-	const readStore = createSitemapReadStore({
-		dataSource: ds,
-		hash
-	});
-	async function readJson(key) {
-		const bytes = await readOptional(ds, key);
-		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
-	}
-	function writeJson(key, value) {
-		return ds.write(key, encodeJsonBigintSafe(value));
-	}
-	async function readProjectionManifest(ctx) {
-		const bytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
-		return bytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(bytes)) : emptySitemapProjectionManifest();
-	}
-	async function deleteExpiredProjectionDeltas(manifest, deltaKeys, currentTime) {
-		const expired = deltaKeys.filter((key) => {
-			const parsed = parseSitemapUrlsDeltaKey(key);
-			const feed = parsed ? manifest.feeds[parsed.feedpathHash] : void 0;
-			return Boolean(feed && key <= feed.compactedThrough && currentTime - feed.publishedAt >= 9e5);
-		});
-		if (expired.length > 0) await ds.delete(expired);
-	}
-	async function publishCurrentProjection(ctx, feedpathHash, rows, consumedDeltas, manifest, publishedAt) {
-		const bytes = encodeRowsToParquetFlex(rows.map(urlRecordToRow), {
-			columns: URLS_INDEX_COLUMNS,
-			sortKey: ["feedpath_hash", "url_hash"]
-		});
-		await ds.write(sitemapUrlsIndexKey(ctx, feedpathHash), bytes);
-		const compactedThrough = consumedDeltas.at(-1)?.key;
-		if (!compactedThrough) return manifest;
-		const next = withSitemapProjectionFeed(manifest, feedpathHash, {
-			compactedThrough,
-			publishedAt
-		});
-		await writeJson(sitemapUrlsProjectionManifestKey(ctx), next);
-		return next;
-	}
-	function normalizedEventRows(rows) {
-		return rows.map((row) => ({
-			feedpath: String(row.feedpath),
-			feedpathHash: String(row.feedpath_hash),
-			urlHash: String(row.url_hash),
-			op: String(row.op),
-			loc: String(row.loc),
-			lastmod: row.lastmod == null ? null : String(row.lastmod),
-			generationId: String(row.generation_id),
-			observedAt: Number(row.observed_at),
-			sequence: Number(row.sequence),
-			projectsState: Boolean(row.projects_state),
-			seedGeneration: Boolean(row.seed_generation),
-			generationKind: String(row.generation_kind),
-			contentHash: row.content_hash == null ? null : String(row.content_hash),
-			inputCount: row.input_count == null ? null : Number(row.input_count)
-		})).sort((a, b) => a.observedAt - b.observedAt || a.sequence - b.sequence || a.urlHash.localeCompare(b.urlHash) || a.op.localeCompare(b.op));
-	}
-	function eventDigest(rows) {
-		return hashUrl(JSON.stringify(normalizedEventRows(rows)));
-	}
-	function assertGenerationAccepted(generation, checkpoint, scope) {
-		if (!checkpoint) return "newer";
-		if (generation.id === checkpoint.generationId) {
-			if (generation.observedAt === checkpoint.observedAt) return "same";
-			throw new Error(`sitemap generation conflict for ${scope}: generation ${generation.id} changed observedAt`);
-		}
-		if (generation.observedAt > checkpoint.observedAt) return "newer";
-		const reason = generation.observedAt === checkpoint.observedAt ? "ambiguous timestamp" : "stale generation";
-		throw new Error(`sitemap generation conflict for ${scope}: ${reason}`);
-	}
-	async function ensureEvents(ctx, feedpathHash, generation, rows) {
-		const expectedDigest = eventDigest(rows);
-		if (rows.length === 0) return {
-			rows: [],
-			digest: expectedDigest
-		};
-		const key = sitemapUrlsEventKey(ctx, feedpathHash, generation);
-		await writeJson(sitemapUrlsPendingGenerationKey(ctx, feedpathHash), {
-			version: 1,
-			generationId: generation.id,
-			observedAt: generation.observedAt,
-			eventKey: key,
-			eventDigest: expectedDigest
-		});
-		const existing = await readOptional(ds, key);
-		if (existing) {
-			const existingRows = await decodeParquetToRows(existing);
-			const existingDigest = eventDigest(existingRows);
-			if (existingDigest !== expectedDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: immutable event digest changed`);
-			return {
-				rows: existingRows,
-				digest: existingDigest
-			};
-		}
-		const bytes = encodeRowsToParquetFlex(rows, {
-			columns: URLS_EVENT_COLUMNS,
-			sortKey: ["sequence", "url_hash"]
-		});
-		await ds.write(key, bytes);
-		return {
-			rows: [...rows],
-			digest: expectedDigest
-		};
-	}
-	function eventRow(generation, row, metadata) {
-		return {
-			feedpath: row.feedpath,
-			feedpath_hash: row.feedpath_hash,
-			url_hash: row.url_hash,
-			op: row.op,
-			loc: row.loc,
-			lastmod: row.lastmod,
-			generation_id: generation.id,
-			observed_at: generation.observedAt,
-			sequence: metadata.sequence,
-			projects_state: metadata.projectsState ? 1 : 0,
-			seed_generation: metadata.seedGeneration ? 1 : 0,
-			generation_kind: metadata.generationKind,
-			content_hash: metadata.contentHash ?? null,
-			input_count: metadata.inputCount ?? null
-		};
-	}
-	function stateRowsFromEvents(rows) {
-		return rows.filter((row) => Boolean(row.projects_state)).map((row) => ({
-			feedpath: row.feedpath,
-			feedpath_hash: row.feedpath_hash,
-			url_hash: row.url_hash,
-			op: row.op,
-			loc: row.loc,
-			lastmod: row.lastmod,
-			at: row.observed_at,
-			generation_id: row.generation_id
-		}));
-	}
-	function checkpointFromEvents(rows, digest) {
-		const first = rows[0];
-		if (!first) throw new Error("cannot checkpoint an empty sitemap event file");
-		const generationId = String(first.generation_id);
-		const observedAt = Number(first.observed_at);
-		for (const row of rows) if (String(row.generation_id) !== generationId || Number(row.observed_at) !== observedAt) throw new Error("sitemap event file contains multiple generations");
-		if (String(first.generation_kind) === "snapshot") {
-			const contentHash = String(first.content_hash);
-			const inputCount = Number(first.input_count);
-			const stateRows = stateRowsFromEvents(rows);
-			const added = stateRows.filter((row) => String(row.op) === "added").length;
-			return {
-				_tag: "snapshot",
-				version: 1,
-				generationId,
-				observedAt,
-				eventDigest: digest,
-				result: {
-					added,
-					removed: stateRows.filter((row) => String(row.op) === "removed").length,
-					kept: Math.max(0, inputCount - added),
-					contentHash,
-					unchanged: stateRows.length === 0
-				}
-			};
-		}
-		return {
-			_tag: "reconcile",
-			version: 1,
-			generationId,
-			observedAt,
-			eventDigest: digest
-		};
-	}
-	async function projectEvents(ctx, feedpathHash, rows, digest) {
-		const checkpoint = checkpointFromEvents(rows, digest);
-		const generation = {
-			id: checkpoint.generationId,
-			observedAt: checkpoint.observedAt
-		};
-		const stateRows = stateRowsFromEvents(rows);
-		if (stateRows.length > 0) {
-			const bytes = encodeRowsToParquetFlex(stateRows, {
-				columns: URLS_DELTA_COLUMNS,
-				sortKey: ["url_hash"]
-			});
-			await ds.write(sitemapUrlsDeltaKey(ctx, feedpathHash, generation), bytes);
-		}
-		if (rows.some((row) => Boolean(row.seed_generation))) await writeJson(sitemapUrlsEventSeedKey(ctx, feedpathHash), {
-			version: 1,
-			generationId: checkpoint.generationId,
-			observedAt: checkpoint.observedAt
-		});
-		await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), checkpoint);
-		return checkpoint;
-	}
-	async function repairFeedProjection(ctx, feedpathHash) {
-		const pendingKey = sitemapUrlsPendingGenerationKey(ctx, feedpathHash);
-		const [checkpoint, pending] = await Promise.all([readJson(sitemapUrlsGenerationKey(ctx, feedpathHash)), readJson(pendingKey)]);
-		if (!pending) return checkpoint;
-		const pendingGeneration = {
-			_tag: "complete",
-			id: pending.generationId,
-			observedAt: pending.observedAt
-		};
-		if (checkpoint?.generationId === pending.generationId && checkpoint.observedAt !== pending.observedAt) throw new Error(`sitemap generation conflict for ${feedpathHash}: generation ${pending.generationId} changed observedAt`);
-		if (checkpoint && pending.observedAt < checkpoint.observedAt) {
-			await ds.delete([pendingKey]);
-			return checkpoint;
-		}
-		const position = assertGenerationAccepted(pendingGeneration, checkpoint, feedpathHash);
-		const eventBytes = await readOptional(ds, pending.eventKey);
-		if (!eventBytes) {
-			await ds.delete([pendingKey]);
-			return checkpoint;
-		}
-		const rows = await decodeParquetToRows(eventBytes);
-		const digest = eventDigest(rows);
-		if (digest !== pending.eventDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: pending event digest changed`);
-		if (position === "same") {
-			if (checkpoint?.eventDigest !== digest) throw new Error(`sitemap generation conflict for ${feedpathHash}: checkpoint digest changed`);
-			await ds.delete([pendingKey]);
-			return checkpoint;
-		}
-		const repaired = await projectEvents(ctx, feedpathHash, rows, digest);
-		await ds.delete([pendingKey]);
-		return repaired;
-	}
 	return {
-		...readStore,
-		writeSnapshot(ctx, records) {
-			return withMutation(ctx, async () => {
-				if (records.length === 0) return;
-				const indexKey = sitemapIndexKey(ctx);
-				const index = await readJson(indexKey) ?? {
-					version: 1,
-					records: {}
-				};
-				const stamp = now();
-				const historyDocs = /* @__PURE__ */ new Map();
-				for (const record of records) {
-					const feedpathHash = hash(record.path);
-					index.records[feedpathHash] = record;
-					historyDocs.set(sitemapHistoryKey(ctx, feedpathHash, stamp), {
-						version: 1,
-						path: record.path,
-						capturedAt: record.capturedAt,
-						record
-					});
-				}
-				await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
-				await writeJson(indexKey, index);
-			});
-		},
-		snapshotUrls(ctx, generation, feedpath, urls) {
-			return withMutation(ctx, async () => {
-				const feedpathHash = hash(feedpath);
-				const contentHash = hashUrlList(urls);
-				const checkpoint = await repairFeedProjection(ctx, feedpathHash);
-				if (assertGenerationAccepted(generation, checkpoint, feedpathHash) === "same") {
-					if (checkpoint?._tag !== "snapshot" || checkpoint.result.contentHash !== contentHash) throw new Error(`sitemap generation conflict for ${feedpathHash}: snapshot input changed`);
-					return checkpoint.result;
-				}
-				if (assertGenerationAccepted(generation, await readJson(sitemapUrlsReconcileGenerationKey(ctx)), "site reconciliation") === "same") throw new Error("sitemap generation conflict: site generation was already reconciled");
-				const priorByHash = /* @__PURE__ */ new Map();
-				for await (const record of readStore.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(record.urlHash, record);
-				const livePrior = [...priorByHash.values()].filter((record) => record.removedAt == null);
-				const incomingByHash = /* @__PURE__ */ new Map();
-				for (const url of urls) incomingByHash.set(hash(url.loc), url);
-				const deltaRows = [];
-				let added = 0;
-				let removed = 0;
-				let kept = 0;
-				for (const [urlHash, url] of incomingByHash) {
-					const previous = priorByHash.get(urlHash);
-					if (!previous || previous.removedAt != null) {
-						added++;
-						deltaRows.push({
-							feedpath,
-							feedpath_hash: feedpathHash,
-							url_hash: urlHash,
-							op: "added",
-							loc: url.loc,
-							lastmod: url.lastmod ?? null,
-							at: generation.observedAt,
-							generation_id: generation.id
-						});
-					} else kept++;
-				}
-				for (const [urlHash, previous] of priorByHash) if (previous.removedAt == null && !incomingByHash.has(urlHash)) {
-					removed++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: feedpathHash,
-						url_hash: urlHash,
-						op: "removed",
-						loc: previous.loc,
-						lastmod: previous.lastmod ?? null,
-						at: generation.observedAt,
-						generation_id: generation.id
-					});
-				}
-				const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
-				const seed = await readJson(seedKey);
-				const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
-				const seedRows = isSeedGeneration ? livePrior.map((record) => ({
-					feedpath,
-					feedpath_hash: feedpathHash,
-					url_hash: record.urlHash,
-					op: "added",
-					loc: record.loc,
-					lastmod: record.lastmod ?? null
-				})) : [];
-				const actualSequence = seedRows.length > 0 ? 1 : 0;
-				const inputCount = incomingByHash.size;
-				const persistedEvents = await ensureEvents(ctx, feedpathHash, generation, [...seedRows.map((row) => eventRow(generation, row, {
-					sequence: 0,
-					projectsState: false,
-					seedGeneration: isSeedGeneration,
-					generationKind: "snapshot",
-					contentHash,
-					inputCount
-				})), ...deltaRows.map((row) => eventRow(generation, row, {
-					sequence: actualSequence,
-					projectsState: true,
-					seedGeneration: isSeedGeneration,
-					generationKind: "snapshot",
-					contentHash,
-					inputCount
-				}))]);
-				const result = {
-					added,
-					removed,
-					kept,
-					contentHash,
-					unchanged: deltaRows.length === 0
-				};
-				if (persistedEvents.rows.length > 0) {
-					if ((await projectEvents(ctx, feedpathHash, persistedEvents.rows, persistedEvents.digest))._tag !== "snapshot") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected snapshot event`);
-					await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
-				} else {
-					if (isSeedGeneration) await writeJson(seedKey, {
-						version: 1,
-						generationId: generation.id,
-						observedAt: generation.observedAt
-					});
-					await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), {
-						_tag: "snapshot",
-						version: 1,
-						generationId: generation.id,
-						observedAt: generation.observedAt,
-						eventDigest: persistedEvents.digest,
-						result
-					});
-				}
-				return result;
-			});
-		},
-		compactUrls(ctx, compactOpts = {}) {
-			return withMutation(ctx, async () => {
-				const startedAt = now();
-				const deadlineMs = compactOpts.deadlineMs ?? Number.POSITIVE_INFINITY;
-				const maxFeedpaths = compactOpts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
-				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
-					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
-					if (feedpathHash) await repairFeedProjection(ctx, feedpathHash);
-				}
-				const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
-				let projectionManifest = await readProjectionManifest(ctx);
-				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, startedAt);
-				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
-				const deltasByFeed = /* @__PURE__ */ new Map();
-				for (const key of activeDeltaKeys) {
-					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
-					if (!feedpathHash) continue;
-					const keys = deltasByFeed.get(feedpathHash) ?? [];
-					keys.push(key);
-					deltasByFeed.set(feedpathHash, keys);
-				}
-				const totalFeedpaths = deltasByFeed.size;
-				let compactedFeedpaths = 0;
-				for (const [feedpathHash, feedDeltaKeys] of deltasByFeed) {
-					if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
-					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, feedDeltaKeys)]);
-					const state = createSitemapUrlState(indexRows);
-					applySitemapDeltaFiles(state, deltaFiles);
-					projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
-					compactedFeedpaths++;
-				}
-				return {
-					compactedFeedpaths,
-					remainingFeedpaths: totalFeedpaths - compactedFeedpaths
-				};
-			});
-		},
-		reconcile(ctx, generation, { liveFeedpaths }) {
-			return withMutation(ctx, async () => {
-				const liveHashes = new Set(liveFeedpaths.map((feedpath) => hash(feedpath)));
-				const inputDigest = hashSortedUrlList([...liveHashes]);
-				const siteCheckpoint = await readJson(sitemapUrlsReconcileGenerationKey(ctx));
-				if (assertGenerationAccepted(generation, siteCheckpoint, "site reconciliation") === "same") {
-					if (siteCheckpoint?.inputDigest !== inputDigest) throw new Error("sitemap generation conflict: reconcile input changed");
-					return {
-						feedpathsPruned: 0,
-						urlsRemoved: 0
-					};
-				}
-				const pendingFeedHashes = /* @__PURE__ */ new Set();
-				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
-					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
-					if (feedpathHash) pendingFeedHashes.add(feedpathHash);
-				}
-				const repairedCheckpoints = /* @__PURE__ */ new Map();
-				for (const feedpathHash of pendingFeedHashes) repairedCheckpoints.set(feedpathHash, await repairFeedProjection(ctx, feedpathHash));
-				const present = /* @__PURE__ */ new Set();
-				const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
-				let projectionManifest = await readProjectionManifest(ctx);
-				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, now());
-				for (const feedpathHash of Object.keys(projectionManifest.feeds)) present.add(feedpathHash);
-				for (const key of indexKeys) {
-					const match = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
-					if (match?.[1]) present.add(match[1]);
-				}
-				const deltasByFeed = /* @__PURE__ */ new Map();
-				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
-				for (const key of activeDeltaKeys) {
-					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
-					if (!feedpathHash) continue;
-					present.add(feedpathHash);
-					const keys = deltasByFeed.get(feedpathHash) ?? [];
-					keys.push(key);
-					deltasByFeed.set(feedpathHash, keys);
-				}
-				let feedpathsPruned = 0;
-				let urlsRemoved = 0;
-				for (const feedpathHash of present) {
-					if (liveHashes.has(feedpathHash)) continue;
-					const checkpoint = repairedCheckpoints.has(feedpathHash) ? repairedCheckpoints.get(feedpathHash) : await repairFeedProjection(ctx, feedpathHash);
-					const feedPosition = assertGenerationAccepted(generation, checkpoint, feedpathHash);
-					if (feedPosition === "same" && checkpoint?._tag !== "reconcile") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected reconcile event`);
-					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, deltasByFeed.get(feedpathHash) ?? [])]);
-					const state = createSitemapUrlState(indexRows);
-					applySitemapDeltaFiles(state, deltaFiles);
-					const live = [...state.live.values()];
-					const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
-					const seed = await readJson(seedKey);
-					const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
-					const seedRows = isSeedGeneration ? live.map((record) => ({
-						feedpath: record.feedpath,
-						feedpath_hash: feedpathHash,
-						url_hash: record.urlHash,
-						op: "added",
-						loc: record.loc,
-						lastmod: record.lastmod ?? null
-					})) : [];
-					const removalRows = live.map((record) => ({
-						feedpath: record.feedpath,
-						feedpath_hash: feedpathHash,
-						url_hash: record.urlHash,
-						op: "removed",
-						loc: record.loc,
-						lastmod: record.lastmod ?? null
-					}));
-					const proposedEvents = feedPosition === "newer" ? [...seedRows.map((row) => eventRow(generation, row, {
-						sequence: 0,
-						projectsState: false,
-						seedGeneration: isSeedGeneration,
-						generationKind: "reconcile"
-					})), ...removalRows.map((row) => eventRow(generation, row, {
-						sequence: seedRows.length > 0 ? 1 : 0,
-						projectsState: true,
-						seedGeneration: isSeedGeneration,
-						generationKind: "reconcile"
-					}))] : [];
-					const persistedEvents = feedPosition === "newer" ? await ensureEvents(ctx, feedpathHash, generation, proposedEvents) : {
-						rows: [],
-						digest: checkpoint.eventDigest
-					};
-					if (isSeedGeneration && feedPosition === "newer") await writeJson(seedKey, {
-						version: 1,
-						generationId: generation.id,
-						observedAt: generation.observedAt
-					});
-					const removedHashes = new Set(persistedEvents.rows.filter((row) => Boolean(row.projects_state) && String(row.op) === "removed").map((row) => String(row.url_hash)));
-					const removedBeforeFeed = urlsRemoved;
-					for (const record of live.filter((record) => removedHashes.has(record.urlHash))) {
-						state.live.delete(record.urlHash);
-						state.removed.set(record.urlHash, {
-							...record,
-							removedAt: generation.observedAt
-						});
-						urlsRemoved++;
-					}
-					if (live.length > 0 || deltaFiles.length > 0) projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
-					if (feedPosition === "newer") {
-						const feedCheckpoint = persistedEvents.rows.length > 0 ? checkpointFromEvents(persistedEvents.rows, persistedEvents.digest) : {
-							_tag: "reconcile",
-							version: 1,
-							generationId: generation.id,
-							observedAt: generation.observedAt,
-							eventDigest: persistedEvents.digest
-						};
-						await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), feedCheckpoint);
-						if (persistedEvents.rows.length > 0) await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
-					}
-					if (urlsRemoved > removedBeforeFeed) feedpathsPruned++;
-				}
-				await writeJson(sitemapUrlsReconcileGenerationKey(ctx), {
-					version: 1,
-					generationId: generation.id,
-					observedAt: generation.observedAt,
-					inputDigest
-				});
-				return {
-					feedpathsPruned,
-					urlsRemoved
-				};
-			});
-		}
+		...createSitemapReadStore(opts),
+		...createSitemapGenerationWriteMethods(opts)
 	};
 }
 export { createSitemapStore };

package/dist/entities/sitemap.mjs

@@ -1,88 +1,64 @@
-import { hashUrl, parseSitemapUrlsDeltaKey, sitemapIndexKey, sitemapUrlsEventsPrefix, sitemapUrlsIndexKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey } from "../entity-keys.mjs";
+import { sitemapSiteManifestKey } from "../entity-keys.mjs";
 import { readOptional } from "../adapters/read-optional.mjs";
 import { decodeParquetToRows } from "../adapters/hyparquet.mjs";
-import { mapEntityIo } from "./io.mjs";
-import { decodeSitemapProjectionManifest, selectSitemapProjectionFiles } from "./sitemap-projection.mjs";
-import { SITEMAP_URLS_EVENT_PREFIX_RE, applySitemapDeltaFiles, createSitemapUrlState, dateInRange, readSitemapDeltaFiles } from "./sitemap-shared.mjs";
+import { createSitemapGenerationReadMethods } from "./sitemap-generation.mjs";
+function dateInRange(date, range) {
+	return (!range?.from || date >= range.from) && (!range?.to || date <= range.to);
+}
+function rowToMembershipEvent(row) {
+	const op = String(row.op);
+	if (op !== "added" && op !== "removed" && op !== "updated") return void 0;
+	return {
+		feedpath: String(row.feedpath),
+		feedpathHash: String(row.feedpath_hash),
+		urlHash: String(row.url_hash),
+		op,
+		loc: String(row.loc),
+		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+		previousLastmod: row.previous_lastmod == null ? void 0 : String(row.previous_lastmod),
+		generationId: String(row.generation_id),
+		observedAt: Number(row.observed_at),
+		sequence: Number(row.sequence),
+		projectsState: Boolean(row.projects_state)
+	};
+}
 function createSitemapReadStore(opts) {
 	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	async function readJson(key) {
+	const generationReads = createSitemapGenerationReadMethods(opts);
+	async function readManifest(key) {
 		const bytes = await readOptional(ds, key);
 		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
 	}
 	return {
-		async loadIndex(ctx) {
-			return await readJson(sitemapIndexKey(ctx)) ?? {
-				version: 1,
-				records: {}
-			};
-		},
-		async getLatest(ctx, path) {
-			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
-		},
-		async *loadUrls(ctx, feedpath, loadOpts) {
-			const feedpathHash = hash(feedpath);
-			const listedDeltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).filter((key) => parseSitemapUrlsDeltaKey(key)?.feedpathHash === feedpathHash);
-			const listedDeltaFiles = await readSitemapDeltaFiles(ds, listedDeltaKeys);
-			const manifestBytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
-			const manifest = manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0;
-			const indexRows = await readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []);
-			const currentDeltaKeys = new Set(selectSitemapProjectionFiles([], listedDeltaKeys, manifest).deltaKeys);
-			const state = createSitemapUrlState(indexRows);
-			applySitemapDeltaFiles(state, listedDeltaFiles.filter((file) => currentDeltaKeys.has(file.key)));
-			for (const record of state.live.values()) yield record;
-			if (loadOpts?.includeRemoved) for (const record of state.removed.values()) yield record;
-		},
-		async *loadDeltas(ctx, dateRange) {
-			const [listedKeys, manifestBytes] = await Promise.all([ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`), readOptional(ds, sitemapUrlsProjectionManifestKey(ctx))]);
-			const keys = selectSitemapProjectionFiles([], listedKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0).deltaKeys.filter((key) => {
-				const parsed = parseSitemapUrlsDeltaKey(key);
-				return Boolean(parsed && dateInRange(parsed.date, dateRange));
-			});
-			const files = await readSitemapDeltaFiles(ds, keys);
-			for (const file of files) for (const row of file.rows) {
-				const op = String(row.op);
-				if (op !== "added" && op !== "removed") continue;
-				yield {
-					feedpath: String(row.feedpath),
-					feedpathHash: String(row.feedpath_hash),
-					urlHash: String(row.url_hash),
-					op,
-					loc: String(row.loc),
-					lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-					at: Number(row.at)
-				};
-			}
-		},
+		...generationReads,
 		async *loadEvents(ctx, dateRange) {
-			const listedEventKeys = await ds.list(`${sitemapUrlsEventsPrefix(ctx)}/`);
-			const pendingKeys = await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`);
-			const pendingEvents = new Set((await mapEntityIo(pendingKeys, (key) => readJson(key))).filter((pending) => pending !== void 0).map((pending) => pending.eventKey));
-			const keys = listedEventKeys.filter((key) => {
-				const match = SITEMAP_URLS_EVENT_PREFIX_RE.exec(key);
-				return !pendingEvents.has(key) && Boolean(match?.[1] && dateInRange(match[1], dateRange));
-			}).sort();
-			for (const key of keys) {
-				const bytes = await readOptional(ds, key);
-				if (!bytes) continue;
-				const rows = await decodeParquetToRows(bytes);
-				rows.sort((a, b) => Number(a.observed_at) - Number(b.observed_at) || Number(a.sequence) - Number(b.sequence) || String(a.url_hash).localeCompare(String(b.url_hash)));
-				for (const row of rows) {
-					const op = String(row.op);
-					if (op !== "added" && op !== "removed") continue;
-					yield {
-						feedpath: String(row.feedpath),
-						feedpathHash: String(row.feedpath_hash),
-						urlHash: String(row.url_hash),
-						op,
-						loc: String(row.loc),
-						lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-						generationId: String(row.generation_id),
-						observedAt: Number(row.observed_at),
-						sequence: Number(row.sequence),
-						projectsState: Boolean(row.projects_state)
-					};
+			const current = await readManifest(sitemapSiteManifestKey(ctx));
+			if (!current) return;
+			const ancestry = [];
+			const seen = /* @__PURE__ */ new Set();
+			let cursor = current;
+			while (cursor) {
+				const observedDate = new Date(cursor.observedAt).toISOString().slice(0, 10);
+				if (dateRange?.from && observedDate < dateRange.from) break;
+				ancestry.push(cursor);
+				if (!cursor.previousManifestKey) break;
+				if (seen.has(cursor.previousManifestKey)) throw new Error("sitemap generation manifest ancestry cycle");
+				seen.add(cursor.previousManifestKey);
+				const previous = await readManifest(cursor.previousManifestKey);
+				if (!previous || previous.generationId !== cursor.previousGenerationId) throw new Error("sitemap generation manifest ancestry is incomplete");
+				cursor = previous;
+			}
+			for (const manifest of ancestry.reverse()) {
+				if (!dateInRange(new Date(manifest.observedAt).toISOString().slice(0, 10), dateRange)) continue;
+				for (const key of manifest.eventKeys) {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) throw new Error(`published sitemap event missing: ${key}`);
+					const rows = await decodeParquetToRows(bytes);
+					rows.sort((left, right) => Number(left.sequence) - Number(right.sequence) || String(left.url_hash).localeCompare(String(right.url_hash)));
+					for (const row of rows) {
+						const event = rowToMembershipEvent(row);
+						if (event) yield event;
+					}
 				}
 			}
 		}