@gscdump/engine 1.4.0 → 1.4.1

package/dist/entities.mjs

@@ -1,4 +1,869 @@
 import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
-import "./adapters/hyparquet.mjs";
-import { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore } from "./_chunks/entities.mjs";
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
+import { readOptional } from "./adapters/read-optional.mjs";
+import { buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
+const ENTITY_IO_CONCURRENCY = 8;
+async function mapEntityIo(items, fn) {
+	if (items.length === 0) return [];
+	const results = Array.from({ length: items.length }, () => void 0);
+	let next = 0;
+	async function worker() {
+		while (true) {
+			const index = next++;
+			if (index >= items.length) return;
+			results[index] = await fn(items[index], index);
+		}
+	}
+	await Promise.all(Array.from({ length: Math.min(ENTITY_IO_CONCURRENCY, items.length) }, worker));
+	return results;
+}
+const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
+const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
+const INSPECTION_PARQUET_COLUMNS = [
+	{
+		name: "urlHash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "inspectedAt",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "indexStatus",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "lastCrawlTime",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "googleCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "userCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "coverageState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "robotsTxtState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "indexingState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "pageFetchState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileUsabilityVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "scheduleNextAt",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "scheduleConsecutiveUnchanged",
+		type: "INTEGER",
+		nullable: true
+	},
+	{
+		name: "schedulePolicyVersion",
+		type: "INTEGER",
+		nullable: true
+	}
+];
+const INSPECTION_EVENT_COLUMNS = [
+	...INSPECTION_PARQUET_COLUMNS,
+	{
+		name: "crawlingUserAgent",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsItems",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "sitemaps",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "referringUrls",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileIssues",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "inspectionResultLink",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "firstCheckedAt",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "checkCount",
+		type: "INTEGER",
+		nullable: true
+	},
+	{
+		name: "nextCheckAfter",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "nextCheckPriority",
+		type: "VARCHAR",
+		nullable: true
+	}
+];
+function createInspectionStore(opts) {
+	const ds = opts.dataSource;
+	function shardFor(record) {
+		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
+		return m ? `${m[1]}-${m[2]}` : "unknown";
+	}
+	function randomBatchId() {
+		return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+	}
+	return {
+		async appendHistory(ctx, records, options) {
+			if (records.length === 0) return;
+			const batchId = options?.batchId ?? randomBatchId();
+			const byMonth = /* @__PURE__ */ new Map();
+			for (const r of records) {
+				const month = shardFor(r);
+				if (!byMonth.has(month)) byMonth.set(month, []);
+				byMonth.get(month).push(r);
+			}
+			await mapEntityIo([...byMonth].map(([yearMonth, batch]) => {
+				const bytes = encodeJsonBigintSafe({
+					version: 1,
+					records: batch
+				});
+				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
+				return {
+					key: inspectionHistoryShardKey(ctx, yearMonth, batchId),
+					bytes
+				};
+			}), (shard) => ds.write(shard.key, shard.bytes));
+		},
+		async loadHistory(ctx, yearMonth) {
+			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));
+			if (keys.length === 0) return void 0;
+			return {
+				version: 1,
+				records: (await mapEntityIo(keys, async (key) => {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) return [];
+					return (await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
+						console.warn("[inspection.loadHistory] failed to decode shard", {
+							key,
+							error: err.message
+						});
+					}))?.records ?? [];
+				})).flat()
+			};
+		},
+		async materialize(ctx, rowIter) {
+			const rows = Array.from(rowIter);
+			rows.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+			const bytes = encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_PARQUET_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			const key = inspectionParquetKey(ctx);
+			await ds.write(key, bytes);
+			return {
+				key,
+				rowCount: rows.length,
+				bytes: bytes.byteLength
+			};
+		},
+		async appendInspectionEvents(ctx, rows, options) {
+			if (rows.length === 0) return {
+				keys: [],
+				rowCount: 0
+			};
+			const batchId = options?.batchId ?? randomBatchId();
+			const byMonth = /* @__PURE__ */ new Map();
+			for (const r of rows) {
+				const m = YEAR_MONTH_RE.exec(r.inspectedAt);
+				const month = m ? `${m[1]}-${m[2]}` : "unknown";
+				const bucket = byMonth.get(month) ?? [];
+				bucket.push(r);
+				byMonth.set(month, bucket);
+			}
+			const files = [...byMonth].map(([month, batch]) => {
+				const bytes = encodeRowsToParquetFlex(batch, {
+					columns: INSPECTION_EVENT_COLUMNS,
+					sortKey: ["urlHash"]
+				});
+				return {
+					key: inspectionEventKey(ctx, month, batchId),
+					bytes
+				};
+			});
+			await mapEntityIo(files, (file) => ds.write(file.key, file.bytes));
+			return {
+				keys: files.map((file) => file.key),
+				rowCount: rows.length
+			};
+		},
+		async compactInspections(ctx) {
+			const eventKeys = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((k) => INSPECTION_EVENT_KEY_RE.test(k));
+			if (eventKeys.length === 0) return {
+				baseRowCount: 0,
+				eventsFolded: 0,
+				eventFilesDeleted: 0
+			};
+			const baseKey = inspectionBaseKey(ctx);
+			const baseBytes = await readOptional(ds, baseKey);
+			const baseRows = baseBytes ? await decodeParquetToRows(baseBytes) : [];
+			const latest = /* @__PURE__ */ new Map();
+			const earliestChecked = /* @__PURE__ */ new Map();
+			const consider = (row) => {
+				const h = String(row.urlHash);
+				const prev = latest.get(h);
+				if (!prev || String(row.inspectedAt ?? "") > String(prev.inspectedAt ?? "")) latest.set(h, row);
+				const fc = row.firstCheckedAt;
+				if (fc != null) {
+					const fcStr = String(fc);
+					const cur = earliestChecked.get(h);
+					if (cur === void 0 || fcStr < cur) earliestChecked.set(h, fcStr);
+				}
+			};
+			for (const row of baseRows) consider(row);
+			let eventsFolded = 0;
+			const consumed = [];
+			const eventFiles = await mapEntityIo(eventKeys.sort(), async (key) => {
+				const bytes = await readOptional(ds, key);
+				if (!bytes) return void 0;
+				return {
+					key,
+					rows: await decodeParquetToRows(bytes)
+				};
+			});
+			for (const file of eventFiles) {
+				if (!file) continue;
+				const { key, rows } = file;
+				consumed.push(key);
+				for (const row of rows) {
+					consider(row);
+					eventsFolded++;
+				}
+			}
+			const merged = [];
+			for (const [h, row] of latest) {
+				const fc = earliestChecked.get(h);
+				if (fc !== void 0) row.firstCheckedAt = fc;
+				merged.push(row);
+			}
+			const bytes = encodeRowsToParquetFlex(merged, {
+				columns: INSPECTION_EVENT_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			await ds.write(baseKey, bytes);
+			if (consumed.length > 0) await ds.delete(consumed);
+			return {
+				baseRowCount: merged.length,
+				eventsFolded,
+				eventFilesDeleted: consumed.length
+			};
+		},
+		parquetUri(ctx) {
+			return ds.uri?.(inspectionParquetKey(ctx));
+		}
+	};
+}
+const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
+const URLS_INDEX_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "first_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "last_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "removed_at",
+		type: "BIGINT",
+		nullable: true
+	}
+];
+const URLS_DELTA_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "op",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "at",
+		type: "BIGINT",
+		nullable: false
+	}
+];
+function rowToUrlRecord(row) {
+	return {
+		feedpath: String(row.feedpath),
+		feedpathHash: String(row.feedpath_hash),
+		urlHash: String(row.url_hash),
+		loc: String(row.loc),
+		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+		firstSeenAt: Number(row.first_seen_at),
+		lastSeenAt: Number(row.last_seen_at),
+		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
+	};
+}
+function urlRecordToRow(r) {
+	return {
+		feedpath: r.feedpath,
+		feedpath_hash: r.feedpathHash,
+		url_hash: r.urlHash,
+		loc: r.loc,
+		lastmod: r.lastmod ?? null,
+		first_seen_at: r.firstSeenAt,
+		last_seen_at: r.lastSeenAt,
+		removed_at: r.removedAt ?? null
+	};
+}
+function isoDate(ms) {
+	return new Date(ms).toISOString().slice(0, 10);
+}
+function createSitemapStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	const now = opts.now ?? (() => Date.now());
+	async function readJson(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return void 0;
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	async function writeJson(key, value) {
+		await ds.write(key, encodeJsonBigintSafe(value));
+	}
+	return {
+		async writeSnapshot(ctx, records) {
+			if (records.length === 0) return;
+			const indexKey = sitemapIndexKey(ctx);
+			const index = await readJson(indexKey) ?? {
+				version: 1,
+				records: {}
+			};
+			const stamp = now();
+			const historyDocs = /* @__PURE__ */ new Map();
+			for (const r of records) {
+				const h = hash(r.path);
+				index.records[h] = r;
+				historyDocs.set(sitemapHistoryKey(ctx, h, stamp), {
+					version: 1,
+					path: r.path,
+					capturedAt: r.capturedAt,
+					record: r
+				});
+			}
+			await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
+			await writeJson(indexKey, index);
+		},
+		async loadIndex(ctx) {
+			return await readJson(sitemapIndexKey(ctx)) ?? {
+				version: 1,
+				records: {}
+			};
+		},
+		async getLatest(ctx, path) {
+			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
+		},
+		async snapshotUrls(ctx, feedpath, urls) {
+			const fpHash = hash(feedpath);
+			const contentHash = hashUrlList(urls);
+			const at = now();
+			const priorByHash = /* @__PURE__ */ new Map();
+			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
+			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
+			if (livePrior.length > 0) {
+				if (hashSortedUrlList(livePrior.map((r) => String(r.loc)).sort()) === contentHash) return {
+					added: 0,
+					removed: 0,
+					kept: livePrior.length,
+					contentHash,
+					unchanged: true
+				};
+			}
+			const incomingByHash = /* @__PURE__ */ new Map();
+			for (const u of urls) incomingByHash.set(hash(u.loc), u);
+			const deltaRows = [];
+			let added = 0;
+			let removed = 0;
+			let kept = 0;
+			const date = isoDate(at);
+			for (const [urlHash, u] of incomingByHash) {
+				const prev = priorByHash.get(urlHash);
+				if (!prev || prev.removedAt != null) {
+					added++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "added",
+						loc: u.loc,
+						lastmod: u.lastmod ?? null,
+						at
+					});
+				} else kept++;
+			}
+			for (const [urlHash, prev] of priorByHash) {
+				if (prev.removedAt != null) continue;
+				if (!incomingByHash.has(urlHash)) {
+					removed++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "removed",
+						loc: prev.loc,
+						lastmod: prev.lastmod ?? null,
+						at
+					});
+				}
+			}
+			if (deltaRows.length > 0) {
+				const bytes = encodeRowsToParquetFlex(deltaRows, {
+					columns: URLS_DELTA_COLUMNS,
+					sortKey: ["url_hash"]
+				});
+				await ds.write(sitemapUrlsDeltaKey(ctx, fpHash, date), bytes);
+			}
+			return {
+				added,
+				removed,
+				kept,
+				contentHash,
+				unchanged: false
+			};
+		},
+		async *loadUrls(ctx, feedpath, opts) {
+			const fpHash = hash(feedpath);
+			const includeRemoved = opts?.includeRemoved ?? false;
+			const [indexRows, listedDeltaKeys] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, fpHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
+			const deltaKeys = listedDeltaKeys.filter((key) => {
+				return SITEMAP_URLS_DELTA_PREFIX_RE.exec(key)?.[2] === fpHash;
+			}).sort();
+			const live = /* @__PURE__ */ new Map();
+			const removedMap = /* @__PURE__ */ new Map();
+			for (const row of indexRows) {
+				const rec = rowToUrlRecord(row);
+				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
+				else live.set(rec.urlHash, rec);
+			}
+			const deltas = await mapEntityIo(deltaKeys, async (key) => {
+				const dBytes = await readOptional(ds, key);
+				if (!dBytes) return [];
+				return decodeParquetToRows(dBytes);
+			});
+			for (const dRows of deltas) for (const r of dRows) {
+				const op = String(r.op);
+				const urlHash = String(r.url_hash);
+				const at = Number(r.at);
+				if (op === "added") {
+					const prev = live.get(urlHash) ?? removedMap.get(urlHash);
+					removedMap.delete(urlHash);
+					live.set(urlHash, {
+						feedpath,
+						feedpathHash: fpHash,
+						urlHash,
+						loc: String(r.loc),
+						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+						firstSeenAt: prev?.firstSeenAt ?? at,
+						lastSeenAt: at
+					});
+				} else if (op === "removed") {
+					const prev = live.get(urlHash);
+					live.delete(urlHash);
+					if (prev) removedMap.set(urlHash, {
+						...prev,
+						removedAt: at
+					});
+				}
+			}
+			for (const rec of live.values()) yield rec;
+			if (includeRemoved) for (const rec of removedMap.values()) yield rec;
+		},
+		async *loadDeltas(ctx, dateRange) {
+			const from = dateRange?.from;
+			const to = dateRange?.to;
+			const keys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			for (const key of keys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const date = m[1];
+				if (!date) continue;
+				if (from && date < from) continue;
+				if (to && date > to) continue;
+				const bytes = await readOptional(ds, key);
+				if (!bytes) continue;
+				const rows = await decodeParquetToRows(bytes);
+				for (const r of rows) {
+					const op = String(r.op);
+					if (op !== "added" && op !== "removed") continue;
+					yield {
+						feedpath: String(r.feedpath),
+						feedpathHash: String(r.feedpath_hash),
+						urlHash: String(r.url_hash),
+						op,
+						loc: String(r.loc),
+						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+						at: Number(r.at)
+					};
+				}
+			}
+		},
+		async compactUrls(ctx, opts = {}) {
+			const startedAt = now();
+			const deadlineMs = opts.deadlineMs ?? Number.POSITIVE_INFINITY;
+			const maxFeedpaths = opts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
+			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
+			const deltasByFeed = /* @__PURE__ */ new Map();
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const feedpathHash = m[2];
+				if (!feedpathHash) continue;
+				const list = deltasByFeed.get(feedpathHash) ?? [];
+				list.push(key);
+				deltasByFeed.set(feedpathHash, list);
+			}
+			const totalFeedpaths = deltasByFeed.size;
+			let compactedFeedpaths = 0;
+			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
+				if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
+				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
+				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo(feedDeltaKeys.sort(), async (key) => {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) return void 0;
+					return {
+						key,
+						rows: await decodeParquetToRows(bytes)
+					};
+				})]);
+				const live = /* @__PURE__ */ new Map();
+				const removed = /* @__PURE__ */ new Map();
+				for (const row of indexRows) {
+					const rec = rowToUrlRecord(row);
+					if (rec.removedAt != null) removed.set(rec.urlHash, rec);
+					else live.set(rec.urlHash, rec);
+				}
+				const consumed = [];
+				for (const file of deltaFiles) {
+					if (!file) continue;
+					const { key, rows } = file;
+					consumed.push(key);
+					for (const r of rows) {
+						const urlHash = String(r.url_hash);
+						const at = Number(r.at);
+						const op = String(r.op);
+						if (op === "added") {
+							const prev = live.get(urlHash) ?? removed.get(urlHash);
+							removed.delete(urlHash);
+							live.set(urlHash, {
+								feedpath: String(r.feedpath),
+								feedpathHash: fpHash,
+								urlHash,
+								loc: String(r.loc),
+								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+								firstSeenAt: prev?.firstSeenAt ?? at,
+								lastSeenAt: at
+							});
+						} else if (op === "removed") {
+							const prev = live.get(urlHash);
+							live.delete(urlHash);
+							if (prev) removed.set(urlHash, {
+								...prev,
+								removedAt: at
+							});
+						}
+					}
+				}
+				const merged = [...live.values(), ...removed.values()];
+				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+					columns: URLS_INDEX_COLUMNS,
+					sortKey: ["feedpath_hash", "url_hash"]
+				});
+				await ds.write(indexKey, bytes);
+				if (consumed.length > 0) await ds.delete(consumed);
+				compactedFeedpaths++;
+			}
+			return {
+				compactedFeedpaths,
+				remainingFeedpaths: totalFeedpaths - compactedFeedpaths
+			};
+		},
+		async reconcile(ctx, { liveFeedpaths, at: atOpt }) {
+			const at = atOpt ?? now();
+			const liveHashes = new Set(liveFeedpaths.map((fp) => hash(fp)));
+			const present = /* @__PURE__ */ new Set();
+			const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
+			for (const key of indexKeys) {
+				const m = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
+				if (m) present.add(m[1]);
+			}
+			const deltasByFeed = /* @__PURE__ */ new Map();
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				present.add(m[2]);
+				const list = deltasByFeed.get(m[2]) ?? [];
+				list.push(key);
+				deltasByFeed.set(m[2], list);
+			}
+			let feedpathsPruned = 0;
+			let urlsRemoved = 0;
+			for (const fpHash of present) {
+				if (liveHashes.has(fpHash)) continue;
+				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
+				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo((deltasByFeed.get(fpHash) ?? []).sort(), async (key) => {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) return void 0;
+					return {
+						key,
+						rows: await decodeParquetToRows(bytes)
+					};
+				})]);
+				const live = /* @__PURE__ */ new Map();
+				const removed = /* @__PURE__ */ new Map();
+				for (const row of indexRows) {
+					const r = rowToUrlRecord(row);
+					if (r.removedAt != null) removed.set(r.urlHash, r);
+					else live.set(r.urlHash, r);
+				}
+				const consumed = [];
+				for (const file of deltaFiles) {
+					if (!file) continue;
+					const { key, rows } = file;
+					consumed.push(key);
+					for (const r of rows) {
+						const urlHash = String(r.url_hash);
+						const dat = Number(r.at);
+						if (String(r.op) === "added") {
+							const prev = live.get(urlHash) ?? removed.get(urlHash);
+							removed.delete(urlHash);
+							live.set(urlHash, {
+								feedpath: String(r.feedpath),
+								feedpathHash: fpHash,
+								urlHash,
+								loc: String(r.loc),
+								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+								firstSeenAt: prev?.firstSeenAt ?? dat,
+								lastSeenAt: dat
+							});
+						} else if (String(r.op) === "removed") {
+							const prev = live.get(urlHash);
+							live.delete(urlHash);
+							if (prev) removed.set(urlHash, {
+								...prev,
+								removedAt: dat
+							});
+						}
+					}
+				}
+				const hadLive = live.size > 0;
+				if (!hadLive && consumed.length === 0) continue;
+				for (const [urlHash, r] of live) {
+					removed.set(urlHash, {
+						...r,
+						removedAt: at
+					});
+					urlsRemoved++;
+				}
+				const merged = [...removed.values()];
+				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+					columns: URLS_INDEX_COLUMNS,
+					sortKey: ["feedpath_hash", "url_hash"]
+				});
+				await ds.write(indexKey, bytes);
+				if (consumed.length > 0) await ds.delete(consumed);
+				if (hadLive) feedpathsPruned++;
+			}
+			return {
+				feedpathsPruned,
+				urlsRemoved
+			};
+		}
+	};
+}
+function createIndexingMetadataStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	async function readIndex(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return {
+			version: 1,
+			records: {}
+		};
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	return {
+		async writeBatch(ctx, records) {
+			if (records.length === 0) return;
+			const key = indexingMetadataIndexKey(ctx);
+			const index = await readIndex(key);
+			for (const r of records) index.records[hash(r.url)] = r;
+			await ds.write(key, encodeJsonBigintSafe(index));
+		},
+		async loadIndex(ctx) {
+			return readIndex(indexingMetadataIndexKey(ctx));
+		},
+		async getLatest(ctx, url) {
+			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
+		}
+	};
+}
+function createEmptyTypesStore(opts) {
+	const ds = opts.dataSource;
+	const now = opts.now ?? (() => Date.now());
+	async function readDoc(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return {
+			version: 1,
+			emptyTypes: [],
+			markedAt: {}
+		};
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	async function writeDoc(key, doc) {
+		await ds.write(key, encodeJsonBigintSafe(doc));
+	}
+	return {
+		async load(ctx) {
+			return readDoc(emptyTypesKey(ctx));
+		},
+		async mark(ctx, types, at) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const stamp = at ?? now();
+			let changed = false;
+			for (const t of types) {
+				if (!doc.emptyTypes.includes(t)) {
+					doc.emptyTypes.push(t);
+					changed = true;
+				}
+				if (doc.markedAt[t] === void 0) {
+					doc.markedAt[t] = stamp;
+					changed = true;
+				}
+			}
+			if (changed) {
+				doc.emptyTypes.sort();
+				await writeDoc(key, doc);
+			}
+			return doc;
+		},
+		async clear(ctx, types) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const drop = new Set(types);
+			const before = doc.emptyTypes.length;
+			doc.emptyTypes = doc.emptyTypes.filter((t) => !drop.has(t));
+			for (const t of drop) delete doc.markedAt[t];
+			if (doc.emptyTypes.length !== before) await writeDoc(key, doc);
+			return doc;
+		}
+	};
+}
 export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };