@gscdump/engine 1.4.11 → 1.5.1

package/dist/entities/sitemap-write.mjs

@@ -0,0 +1,528 @@
+import { hashSortedUrlList, hashUrl, hashUrlList, parseSitemapUrlsDeltaKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
+import { mapEntityIo } from "./io.mjs";
+import { decodeSitemapProjectionManifest, emptySitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
+import { SITEMAP_URLS_PENDING_GENERATION_RE, URLS_DELTA_COLUMNS, URLS_EVENT_COLUMNS, URLS_INDEX_COLUMNS, applySitemapDeltaFiles, createSitemapUrlState, readSitemapDeltaFiles, urlRecordToRow } from "./sitemap-shared.mjs";
+import { createSitemapReadStore } from "./sitemap.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+function createSitemapStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	const now = opts.now ?? (() => Date.now());
+	const withMutation = opts.withMutation;
+	const readStore = createSitemapReadStore({
+		dataSource: ds,
+		hash
+	});
+	async function readJson(key) {
+		const bytes = await readOptional(ds, key);
+		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
+	}
+	function writeJson(key, value) {
+		return ds.write(key, encodeJsonBigintSafe(value));
+	}
+	async function readProjectionManifest(ctx) {
+		const bytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
+		return bytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(bytes)) : emptySitemapProjectionManifest();
+	}
+	async function deleteExpiredProjectionDeltas(manifest, deltaKeys, currentTime) {
+		const expired = deltaKeys.filter((key) => {
+			const parsed = parseSitemapUrlsDeltaKey(key);
+			const feed = parsed ? manifest.feeds[parsed.feedpathHash] : void 0;
+			return Boolean(feed && key <= feed.compactedThrough && currentTime - feed.publishedAt >= 9e5);
+		});
+		if (expired.length > 0) await ds.delete(expired);
+	}
+	async function publishCurrentProjection(ctx, feedpathHash, rows, consumedDeltas, manifest, publishedAt) {
+		const bytes = encodeRowsToParquetFlex(rows.map(urlRecordToRow), {
+			columns: URLS_INDEX_COLUMNS,
+			sortKey: ["feedpath_hash", "url_hash"]
+		});
+		await ds.write(sitemapUrlsIndexKey(ctx, feedpathHash), bytes);
+		const compactedThrough = consumedDeltas.at(-1)?.key;
+		if (!compactedThrough) return manifest;
+		const next = withSitemapProjectionFeed(manifest, feedpathHash, {
+			compactedThrough,
+			publishedAt
+		});
+		await writeJson(sitemapUrlsProjectionManifestKey(ctx), next);
+		return next;
+	}
+	function normalizedEventRows(rows) {
+		return rows.map((row) => ({
+			feedpath: String(row.feedpath),
+			feedpathHash: String(row.feedpath_hash),
+			urlHash: String(row.url_hash),
+			op: String(row.op),
+			loc: String(row.loc),
+			lastmod: row.lastmod == null ? null : String(row.lastmod),
+			generationId: String(row.generation_id),
+			observedAt: Number(row.observed_at),
+			sequence: Number(row.sequence),
+			projectsState: Boolean(row.projects_state),
+			seedGeneration: Boolean(row.seed_generation),
+			generationKind: String(row.generation_kind),
+			contentHash: row.content_hash == null ? null : String(row.content_hash),
+			inputCount: row.input_count == null ? null : Number(row.input_count)
+		})).sort((a, b) => a.observedAt - b.observedAt || a.sequence - b.sequence || a.urlHash.localeCompare(b.urlHash) || a.op.localeCompare(b.op));
+	}
+	function eventDigest(rows) {
+		return hashUrl(JSON.stringify(normalizedEventRows(rows)));
+	}
+	function assertGenerationAccepted(generation, checkpoint, scope) {
+		if (!checkpoint) return "newer";
+		if (generation.id === checkpoint.generationId) {
+			if (generation.observedAt === checkpoint.observedAt) return "same";
+			throw new Error(`sitemap generation conflict for ${scope}: generation ${generation.id} changed observedAt`);
+		}
+		if (generation.observedAt > checkpoint.observedAt) return "newer";
+		const reason = generation.observedAt === checkpoint.observedAt ? "ambiguous timestamp" : "stale generation";
+		throw new Error(`sitemap generation conflict for ${scope}: ${reason}`);
+	}
+	async function ensureEvents(ctx, feedpathHash, generation, rows) {
+		const expectedDigest = eventDigest(rows);
+		if (rows.length === 0) return {
+			rows: [],
+			digest: expectedDigest
+		};
+		const key = sitemapUrlsEventKey(ctx, feedpathHash, generation);
+		await writeJson(sitemapUrlsPendingGenerationKey(ctx, feedpathHash), {
+			version: 1,
+			generationId: generation.id,
+			observedAt: generation.observedAt,
+			eventKey: key,
+			eventDigest: expectedDigest
+		});
+		const existing = await readOptional(ds, key);
+		if (existing) {
+			const existingRows = await decodeParquetToRows(existing);
+			const existingDigest = eventDigest(existingRows);
+			if (existingDigest !== expectedDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: immutable event digest changed`);
+			return {
+				rows: existingRows,
+				digest: existingDigest
+			};
+		}
+		const bytes = encodeRowsToParquetFlex(rows, {
+			columns: URLS_EVENT_COLUMNS,
+			sortKey: ["sequence", "url_hash"]
+		});
+		await ds.write(key, bytes);
+		return {
+			rows: [...rows],
+			digest: expectedDigest
+		};
+	}
+	function eventRow(generation, row, metadata) {
+		return {
+			feedpath: row.feedpath,
+			feedpath_hash: row.feedpath_hash,
+			url_hash: row.url_hash,
+			op: row.op,
+			loc: row.loc,
+			lastmod: row.lastmod,
+			generation_id: generation.id,
+			observed_at: generation.observedAt,
+			sequence: metadata.sequence,
+			projects_state: metadata.projectsState ? 1 : 0,
+			seed_generation: metadata.seedGeneration ? 1 : 0,
+			generation_kind: metadata.generationKind,
+			content_hash: metadata.contentHash ?? null,
+			input_count: metadata.inputCount ?? null
+		};
+	}
+	function stateRowsFromEvents(rows) {
+		return rows.filter((row) => Boolean(row.projects_state)).map((row) => ({
+			feedpath: row.feedpath,
+			feedpath_hash: row.feedpath_hash,
+			url_hash: row.url_hash,
+			op: row.op,
+			loc: row.loc,
+			lastmod: row.lastmod,
+			at: row.observed_at,
+			generation_id: row.generation_id
+		}));
+	}
+	function checkpointFromEvents(rows, digest) {
+		const first = rows[0];
+		if (!first) throw new Error("cannot checkpoint an empty sitemap event file");
+		const generationId = String(first.generation_id);
+		const observedAt = Number(first.observed_at);
+		for (const row of rows) if (String(row.generation_id) !== generationId || Number(row.observed_at) !== observedAt) throw new Error("sitemap event file contains multiple generations");
+		if (String(first.generation_kind) === "snapshot") {
+			const contentHash = String(first.content_hash);
+			const inputCount = Number(first.input_count);
+			const stateRows = stateRowsFromEvents(rows);
+			const added = stateRows.filter((row) => String(row.op) === "added").length;
+			return {
+				_tag: "snapshot",
+				version: 1,
+				generationId,
+				observedAt,
+				eventDigest: digest,
+				result: {
+					added,
+					removed: stateRows.filter((row) => String(row.op) === "removed").length,
+					kept: Math.max(0, inputCount - added),
+					contentHash,
+					unchanged: stateRows.length === 0
+				}
+			};
+		}
+		return {
+			_tag: "reconcile",
+			version: 1,
+			generationId,
+			observedAt,
+			eventDigest: digest
+		};
+	}
+	async function projectEvents(ctx, feedpathHash, rows, digest) {
+		const checkpoint = checkpointFromEvents(rows, digest);
+		const generation = {
+			id: checkpoint.generationId,
+			observedAt: checkpoint.observedAt
+		};
+		const stateRows = stateRowsFromEvents(rows);
+		if (stateRows.length > 0) {
+			const bytes = encodeRowsToParquetFlex(stateRows, {
+				columns: URLS_DELTA_COLUMNS,
+				sortKey: ["url_hash"]
+			});
+			await ds.write(sitemapUrlsDeltaKey(ctx, feedpathHash, generation), bytes);
+		}
+		if (rows.some((row) => Boolean(row.seed_generation))) await writeJson(sitemapUrlsEventSeedKey(ctx, feedpathHash), {
+			version: 1,
+			generationId: checkpoint.generationId,
+			observedAt: checkpoint.observedAt
+		});
+		await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), checkpoint);
+		return checkpoint;
+	}
+	async function repairFeedProjection(ctx, feedpathHash) {
+		const pendingKey = sitemapUrlsPendingGenerationKey(ctx, feedpathHash);
+		const [checkpoint, pending] = await Promise.all([readJson(sitemapUrlsGenerationKey(ctx, feedpathHash)), readJson(pendingKey)]);
+		if (!pending) return checkpoint;
+		const pendingGeneration = {
+			_tag: "complete",
+			id: pending.generationId,
+			observedAt: pending.observedAt
+		};
+		if (checkpoint?.generationId === pending.generationId && checkpoint.observedAt !== pending.observedAt) throw new Error(`sitemap generation conflict for ${feedpathHash}: generation ${pending.generationId} changed observedAt`);
+		if (checkpoint && pending.observedAt < checkpoint.observedAt) {
+			await ds.delete([pendingKey]);
+			return checkpoint;
+		}
+		const position = assertGenerationAccepted(pendingGeneration, checkpoint, feedpathHash);
+		const eventBytes = await readOptional(ds, pending.eventKey);
+		if (!eventBytes) {
+			await ds.delete([pendingKey]);
+			return checkpoint;
+		}
+		const rows = await decodeParquetToRows(eventBytes);
+		const digest = eventDigest(rows);
+		if (digest !== pending.eventDigest) throw new Error(`sitemap generation conflict for ${feedpathHash}: pending event digest changed`);
+		if (position === "same") {
+			if (checkpoint?.eventDigest !== digest) throw new Error(`sitemap generation conflict for ${feedpathHash}: checkpoint digest changed`);
+			await ds.delete([pendingKey]);
+			return checkpoint;
+		}
+		const repaired = await projectEvents(ctx, feedpathHash, rows, digest);
+		await ds.delete([pendingKey]);
+		return repaired;
+	}
+	return {
+		...readStore,
+		writeSnapshot(ctx, records) {
+			return withMutation(ctx, async () => {
+				if (records.length === 0) return;
+				const indexKey = sitemapIndexKey(ctx);
+				const index = await readJson(indexKey) ?? {
+					version: 1,
+					records: {}
+				};
+				const stamp = now();
+				const historyDocs = /* @__PURE__ */ new Map();
+				for (const record of records) {
+					const feedpathHash = hash(record.path);
+					index.records[feedpathHash] = record;
+					historyDocs.set(sitemapHistoryKey(ctx, feedpathHash, stamp), {
+						version: 1,
+						path: record.path,
+						capturedAt: record.capturedAt,
+						record
+					});
+				}
+				await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
+				await writeJson(indexKey, index);
+			});
+		},
+		snapshotUrls(ctx, generation, feedpath, urls) {
+			return withMutation(ctx, async () => {
+				const feedpathHash = hash(feedpath);
+				const contentHash = hashUrlList(urls);
+				const checkpoint = await repairFeedProjection(ctx, feedpathHash);
+				if (assertGenerationAccepted(generation, checkpoint, feedpathHash) === "same") {
+					if (checkpoint?._tag !== "snapshot" || checkpoint.result.contentHash !== contentHash) throw new Error(`sitemap generation conflict for ${feedpathHash}: snapshot input changed`);
+					return checkpoint.result;
+				}
+				if (assertGenerationAccepted(generation, await readJson(sitemapUrlsReconcileGenerationKey(ctx)), "site reconciliation") === "same") throw new Error("sitemap generation conflict: site generation was already reconciled");
+				const priorByHash = /* @__PURE__ */ new Map();
+				for await (const record of readStore.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(record.urlHash, record);
+				const livePrior = [...priorByHash.values()].filter((record) => record.removedAt == null);
+				const incomingByHash = /* @__PURE__ */ new Map();
+				for (const url of urls) incomingByHash.set(hash(url.loc), url);
+				const deltaRows = [];
+				let added = 0;
+				let removed = 0;
+				let kept = 0;
+				for (const [urlHash, url] of incomingByHash) {
+					const previous = priorByHash.get(urlHash);
+					if (!previous || previous.removedAt != null) {
+						added++;
+						deltaRows.push({
+							feedpath,
+							feedpath_hash: feedpathHash,
+							url_hash: urlHash,
+							op: "added",
+							loc: url.loc,
+							lastmod: url.lastmod ?? null,
+							at: generation.observedAt,
+							generation_id: generation.id
+						});
+					} else kept++;
+				}
+				for (const [urlHash, previous] of priorByHash) if (previous.removedAt == null && !incomingByHash.has(urlHash)) {
+					removed++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: feedpathHash,
+						url_hash: urlHash,
+						op: "removed",
+						loc: previous.loc,
+						lastmod: previous.lastmod ?? null,
+						at: generation.observedAt,
+						generation_id: generation.id
+					});
+				}
+				const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
+				const seed = await readJson(seedKey);
+				const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
+				const seedRows = isSeedGeneration ? livePrior.map((record) => ({
+					feedpath,
+					feedpath_hash: feedpathHash,
+					url_hash: record.urlHash,
+					op: "added",
+					loc: record.loc,
+					lastmod: record.lastmod ?? null
+				})) : [];
+				const actualSequence = seedRows.length > 0 ? 1 : 0;
+				const inputCount = incomingByHash.size;
+				const persistedEvents = await ensureEvents(ctx, feedpathHash, generation, [...seedRows.map((row) => eventRow(generation, row, {
+					sequence: 0,
+					projectsState: false,
+					seedGeneration: isSeedGeneration,
+					generationKind: "snapshot",
+					contentHash,
+					inputCount
+				})), ...deltaRows.map((row) => eventRow(generation, row, {
+					sequence: actualSequence,
+					projectsState: true,
+					seedGeneration: isSeedGeneration,
+					generationKind: "snapshot",
+					contentHash,
+					inputCount
+				}))]);
+				const result = {
+					added,
+					removed,
+					kept,
+					contentHash,
+					unchanged: deltaRows.length === 0
+				};
+				if (persistedEvents.rows.length > 0) {
+					if ((await projectEvents(ctx, feedpathHash, persistedEvents.rows, persistedEvents.digest))._tag !== "snapshot") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected snapshot event`);
+					await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
+				} else {
+					if (isSeedGeneration) await writeJson(seedKey, {
+						version: 1,
+						generationId: generation.id,
+						observedAt: generation.observedAt
+					});
+					await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), {
+						_tag: "snapshot",
+						version: 1,
+						generationId: generation.id,
+						observedAt: generation.observedAt,
+						eventDigest: persistedEvents.digest,
+						result
+					});
+				}
+				return result;
+			});
+		},
+		compactUrls(ctx, compactOpts = {}) {
+			return withMutation(ctx, async () => {
+				const startedAt = now();
+				const deadlineMs = compactOpts.deadlineMs ?? Number.POSITIVE_INFINITY;
+				const maxFeedpaths = compactOpts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
+				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
+					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
+					if (feedpathHash) await repairFeedProjection(ctx, feedpathHash);
+				}
+				const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
+				let projectionManifest = await readProjectionManifest(ctx);
+				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, startedAt);
+				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
+				const deltasByFeed = /* @__PURE__ */ new Map();
+				for (const key of activeDeltaKeys) {
+					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
+					if (!feedpathHash) continue;
+					const keys = deltasByFeed.get(feedpathHash) ?? [];
+					keys.push(key);
+					deltasByFeed.set(feedpathHash, keys);
+				}
+				const totalFeedpaths = deltasByFeed.size;
+				let compactedFeedpaths = 0;
+				for (const [feedpathHash, feedDeltaKeys] of deltasByFeed) {
+					if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
+					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, feedDeltaKeys)]);
+					const state = createSitemapUrlState(indexRows);
+					applySitemapDeltaFiles(state, deltaFiles);
+					projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
+					compactedFeedpaths++;
+				}
+				return {
+					compactedFeedpaths,
+					remainingFeedpaths: totalFeedpaths - compactedFeedpaths
+				};
+			});
+		},
+		reconcile(ctx, generation, { liveFeedpaths }) {
+			return withMutation(ctx, async () => {
+				const liveHashes = new Set(liveFeedpaths.map((feedpath) => hash(feedpath)));
+				const inputDigest = hashSortedUrlList([...liveHashes]);
+				const siteCheckpoint = await readJson(sitemapUrlsReconcileGenerationKey(ctx));
+				if (assertGenerationAccepted(generation, siteCheckpoint, "site reconciliation") === "same") {
+					if (siteCheckpoint?.inputDigest !== inputDigest) throw new Error("sitemap generation conflict: reconcile input changed");
+					return {
+						feedpathsPruned: 0,
+						urlsRemoved: 0
+					};
+				}
+				const pendingFeedHashes = /* @__PURE__ */ new Set();
+				for (const key of await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`)) {
+					const feedpathHash = SITEMAP_URLS_PENDING_GENERATION_RE.exec(key)?.[1];
+					if (feedpathHash) pendingFeedHashes.add(feedpathHash);
+				}
+				const repairedCheckpoints = /* @__PURE__ */ new Map();
+				for (const feedpathHash of pendingFeedHashes) repairedCheckpoints.set(feedpathHash, await repairFeedProjection(ctx, feedpathHash));
+				const present = /* @__PURE__ */ new Set();
+				const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
+				let projectionManifest = await readProjectionManifest(ctx);
+				await deleteExpiredProjectionDeltas(projectionManifest, deltaKeys, now());
+				for (const feedpathHash of Object.keys(projectionManifest.feeds)) present.add(feedpathHash);
+				for (const key of indexKeys) {
+					const match = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
+					if (match?.[1]) present.add(match[1]);
+				}
+				const deltasByFeed = /* @__PURE__ */ new Map();
+				const activeDeltaKeys = selectSitemapProjectionFiles([], deltaKeys, projectionManifest).deltaKeys;
+				for (const key of activeDeltaKeys) {
+					const feedpathHash = parseSitemapUrlsDeltaKey(key)?.feedpathHash;
+					if (!feedpathHash) continue;
+					present.add(feedpathHash);
+					const keys = deltasByFeed.get(feedpathHash) ?? [];
+					keys.push(key);
+					deltasByFeed.set(feedpathHash, keys);
+				}
+				let feedpathsPruned = 0;
+				let urlsRemoved = 0;
+				for (const feedpathHash of present) {
+					if (liveHashes.has(feedpathHash)) continue;
+					const checkpoint = repairedCheckpoints.has(feedpathHash) ? repairedCheckpoints.get(feedpathHash) : await repairFeedProjection(ctx, feedpathHash);
+					const feedPosition = assertGenerationAccepted(generation, checkpoint, feedpathHash);
+					if (feedPosition === "same" && checkpoint?._tag !== "reconcile") throw new Error(`sitemap generation conflict for ${feedpathHash}: expected reconcile event`);
+					const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), readSitemapDeltaFiles(ds, deltasByFeed.get(feedpathHash) ?? [])]);
+					const state = createSitemapUrlState(indexRows);
+					applySitemapDeltaFiles(state, deltaFiles);
+					const live = [...state.live.values()];
+					const seedKey = sitemapUrlsEventSeedKey(ctx, feedpathHash);
+					const seed = await readJson(seedKey);
+					const isSeedGeneration = seed === void 0 || seed.generationId === generation.id;
+					const seedRows = isSeedGeneration ? live.map((record) => ({
+						feedpath: record.feedpath,
+						feedpath_hash: feedpathHash,
+						url_hash: record.urlHash,
+						op: "added",
+						loc: record.loc,
+						lastmod: record.lastmod ?? null
+					})) : [];
+					const removalRows = live.map((record) => ({
+						feedpath: record.feedpath,
+						feedpath_hash: feedpathHash,
+						url_hash: record.urlHash,
+						op: "removed",
+						loc: record.loc,
+						lastmod: record.lastmod ?? null
+					}));
+					const proposedEvents = feedPosition === "newer" ? [...seedRows.map((row) => eventRow(generation, row, {
+						sequence: 0,
+						projectsState: false,
+						seedGeneration: isSeedGeneration,
+						generationKind: "reconcile"
+					})), ...removalRows.map((row) => eventRow(generation, row, {
+						sequence: seedRows.length > 0 ? 1 : 0,
+						projectsState: true,
+						seedGeneration: isSeedGeneration,
+						generationKind: "reconcile"
+					}))] : [];
+					const persistedEvents = feedPosition === "newer" ? await ensureEvents(ctx, feedpathHash, generation, proposedEvents) : {
+						rows: [],
+						digest: checkpoint.eventDigest
+					};
+					if (isSeedGeneration && feedPosition === "newer") await writeJson(seedKey, {
+						version: 1,
+						generationId: generation.id,
+						observedAt: generation.observedAt
+					});
+					const removedHashes = new Set(persistedEvents.rows.filter((row) => Boolean(row.projects_state) && String(row.op) === "removed").map((row) => String(row.url_hash)));
+					const removedBeforeFeed = urlsRemoved;
+					for (const record of live.filter((record) => removedHashes.has(record.urlHash))) {
+						state.live.delete(record.urlHash);
+						state.removed.set(record.urlHash, {
+							...record,
+							removedAt: generation.observedAt
+						});
+						urlsRemoved++;
+					}
+					if (live.length > 0 || deltaFiles.length > 0) projectionManifest = await publishCurrentProjection(ctx, feedpathHash, [...state.live.values(), ...state.removed.values()].sort((a, b) => a.urlHash.localeCompare(b.urlHash)), deltaFiles, projectionManifest, now());
+					if (feedPosition === "newer") {
+						const feedCheckpoint = persistedEvents.rows.length > 0 ? checkpointFromEvents(persistedEvents.rows, persistedEvents.digest) : {
+							_tag: "reconcile",
+							version: 1,
+							generationId: generation.id,
+							observedAt: generation.observedAt,
+							eventDigest: persistedEvents.digest
+						};
+						await writeJson(sitemapUrlsGenerationKey(ctx, feedpathHash), feedCheckpoint);
+						if (persistedEvents.rows.length > 0) await ds.delete([sitemapUrlsPendingGenerationKey(ctx, feedpathHash)]);
+					}
+					if (urlsRemoved > removedBeforeFeed) feedpathsPruned++;
+				}
+				await writeJson(sitemapUrlsReconcileGenerationKey(ctx), {
+					version: 1,
+					generationId: generation.id,
+					observedAt: generation.observedAt,
+					inputDigest
+				});
+				return {
+					feedpathsPruned,
+					urlsRemoved
+				};
+			});
+		}
+	};
+}
+export { createSitemapStore };

package/dist/entities/sitemap.d.mts

@@ -0,0 +1,3 @@
+import { CreateSitemapReadStoreOptions, SitemapReadStore } from "./sitemap-shared.mjs";
+declare function createSitemapReadStore(opts: CreateSitemapReadStoreOptions): SitemapReadStore;
+export { createSitemapReadStore };

package/dist/entities/sitemap.mjs

@@ -0,0 +1,91 @@
+import { hashUrl, parseSitemapUrlsDeltaKey, sitemapIndexKey, sitemapUrlsEventsPrefix, sitemapUrlsIndexKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { decodeParquetToRows } from "../adapters/hyparquet.mjs";
+import { mapEntityIo } from "./io.mjs";
+import { decodeSitemapProjectionManifest, selectSitemapProjectionFiles } from "./sitemap-projection.mjs";
+import { SITEMAP_URLS_EVENT_PREFIX_RE, applySitemapDeltaFiles, createSitemapUrlState, dateInRange, readSitemapDeltaFiles } from "./sitemap-shared.mjs";
+function createSitemapReadStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	async function readJson(key) {
+		const bytes = await readOptional(ds, key);
+		return bytes === void 0 ? void 0 : JSON.parse(new TextDecoder().decode(bytes));
+	}
+	return {
+		async loadIndex(ctx) {
+			return await readJson(sitemapIndexKey(ctx)) ?? {
+				version: 1,
+				records: {}
+			};
+		},
+		async getLatest(ctx, path) {
+			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
+		},
+		async *loadUrls(ctx, feedpath, loadOpts) {
+			const feedpathHash = hash(feedpath);
+			const listedDeltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).filter((key) => parseSitemapUrlsDeltaKey(key)?.feedpathHash === feedpathHash);
+			const listedDeltaFiles = await readSitemapDeltaFiles(ds, listedDeltaKeys);
+			const manifestBytes = await readOptional(ds, sitemapUrlsProjectionManifestKey(ctx));
+			const manifest = manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0;
+			const indexRows = await readOptional(ds, sitemapUrlsIndexKey(ctx, feedpathHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []);
+			const currentDeltaKeys = new Set(selectSitemapProjectionFiles([], listedDeltaKeys, manifest).deltaKeys);
+			const state = createSitemapUrlState(indexRows);
+			applySitemapDeltaFiles(state, listedDeltaFiles.filter((file) => currentDeltaKeys.has(file.key)));
+			for (const record of state.live.values()) yield record;
+			if (loadOpts?.includeRemoved) for (const record of state.removed.values()) yield record;
+		},
+		async *loadDeltas(ctx, dateRange) {
+			const [listedKeys, manifestBytes] = await Promise.all([ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`), readOptional(ds, sitemapUrlsProjectionManifestKey(ctx))]);
+			const keys = selectSitemapProjectionFiles([], listedKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0).deltaKeys.filter((key) => {
+				const parsed = parseSitemapUrlsDeltaKey(key);
+				return Boolean(parsed && dateInRange(parsed.date, dateRange));
+			});
+			const files = await readSitemapDeltaFiles(ds, keys);
+			for (const file of files) for (const row of file.rows) {
+				const op = String(row.op);
+				if (op !== "added" && op !== "removed") continue;
+				yield {
+					feedpath: String(row.feedpath),
+					feedpathHash: String(row.feedpath_hash),
+					urlHash: String(row.url_hash),
+					op,
+					loc: String(row.loc),
+					lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+					at: Number(row.at)
+				};
+			}
+		},
+		async *loadEvents(ctx, dateRange) {
+			const listedEventKeys = await ds.list(`${sitemapUrlsEventsPrefix(ctx)}/`);
+			const pendingKeys = await ds.list(`${sitemapUrlsPendingGenerationsPrefix(ctx)}/`);
+			const pendingEvents = new Set((await mapEntityIo(pendingKeys, (key) => readJson(key))).filter((pending) => pending !== void 0).map((pending) => pending.eventKey));
+			const keys = listedEventKeys.filter((key) => {
+				const match = SITEMAP_URLS_EVENT_PREFIX_RE.exec(key);
+				return !pendingEvents.has(key) && Boolean(match?.[1] && dateInRange(match[1], dateRange));
+			}).sort();
+			for (const key of keys) {
+				const bytes = await readOptional(ds, key);
+				if (!bytes) continue;
+				const rows = await decodeParquetToRows(bytes);
+				rows.sort((a, b) => Number(a.observed_at) - Number(b.observed_at) || Number(a.sequence) - Number(b.sequence) || String(a.url_hash).localeCompare(String(b.url_hash)));
+				for (const row of rows) {
+					const op = String(row.op);
+					if (op !== "added" && op !== "removed") continue;
+					yield {
+						feedpath: String(row.feedpath),
+						feedpathHash: String(row.feedpath_hash),
+						urlHash: String(row.url_hash),
+						op,
+						loc: String(row.loc),
+						lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+						generationId: String(row.generation_id),
+						observedAt: Number(row.observed_at),
+						sequence: Number(row.sequence),
+						projectsState: Boolean(row.projects_state)
+					};
+				}
+			}
+		}
+	};
+}
+export { createSitemapReadStore };