@gscdump/engine 1.4.0 → 1.4.1

package/dist/_chunks/entities.mjs

@@ -1,967 +0,0 @@
-import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix } from "../entity-keys.mjs";
-import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
-import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
-function isMissingKeyError(e) {
-	if (typeof e !== "object" || e === null) return false;
-	if (e.code === "ENOENT") return true;
-	if (e.name === "NotFoundError") return true;
-	const message = e.message;
-	if (typeof message !== "string") return false;
-	return /\bnot found\b|\bENOENT\b|\bmissing key\b/i.test(message);
-}
-async function readOptional(ds, key, signal) {
-	return await ds.read(key, void 0, signal).catch((e) => {
-		if (isMissingKeyError(e)) return void 0;
-		throw e;
-	});
-}
-const QUERY_DIM_COLUMNS = [
-	{
-		name: "query",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "query_canonical",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "intent_code",
-		type: "INTEGER",
-		nullable: false
-	},
-	{
-		name: "normalizer_version",
-		type: "INTEGER",
-		nullable: false
-	},
-	{
-		name: "intent_version",
-		type: "INTEGER",
-		nullable: false
-	}
-];
-function buildQueryDimRecords(queries, deps) {
-	const seen = /* @__PURE__ */ new Set();
-	const out = [];
-	for (const raw of queries) {
-		const query = String(raw);
-		if (query.trim() === "" || seen.has(query)) continue;
-		seen.add(query);
-		const canonical = deps.normalizeQuery(query);
-		out.push({
-			query,
-			query_canonical: canonical === "" ? query : canonical,
-			intent_code: deps.classifyIntentCode(query),
-			normalizer_version: deps.normalizerVersion,
-			intent_version: deps.intentVersion
-		});
-	}
-	return out;
-}
-function createQueryDimStore({ dataSource }) {
-	return {
-		parquetKey: queryDimParquetKey,
-		async write(ctx, records, builtAt) {
-			const parquetKey = queryDimParquetKey(ctx);
-			const bytes = encodeRowsToParquetFlex(records, {
-				columns: QUERY_DIM_COLUMNS,
-				sortKey: ["query"]
-			});
-			await dataSource.write(parquetKey, bytes);
-			const meta = {
-				version: 1,
-				builtAt,
-				rowCount: records.length,
-				normalizerVersion: records[0]?.normalizer_version ?? 0,
-				intentVersion: records[0]?.intent_version ?? 0
-			};
-			await dataSource.write(queryDimMetaKey(ctx), encodeJsonBigintSafe(meta));
-			return {
-				parquetKey,
-				rowCount: records.length
-			};
-		},
-		async loadMeta(ctx) {
-			const bytes = await readOptional(dataSource, queryDimMetaKey(ctx));
-			if (!bytes) return null;
-			return JSON.parse(new TextDecoder().decode(bytes));
-		},
-		async loadRecords(ctx) {
-			const bytes = await readOptional(dataSource, queryDimParquetKey(ctx));
-			if (!bytes) return [];
-			return (await decodeParquetToRows(bytes)).map((r) => ({
-				query: String(r.query),
-				query_canonical: String(r.query_canonical),
-				intent_code: Number(r.intent_code),
-				normalizer_version: Number(r.normalizer_version),
-				intent_version: Number(r.intent_version)
-			}));
-		}
-	};
-}
-const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
-const ENTITY_IO_CONCURRENCY = 8;
-async function mapEntityIo(items, fn) {
-	if (items.length === 0) return [];
-	const results = Array.from({ length: items.length }, () => void 0);
-	let next = 0;
-	async function worker() {
-		while (true) {
-			const index = next++;
-			if (index >= items.length) return;
-			results[index] = await fn(items[index], index);
-		}
-	}
-	await Promise.all(Array.from({ length: Math.min(ENTITY_IO_CONCURRENCY, items.length) }, worker));
-	return results;
-}
-const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
-const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
-const INSPECTION_PARQUET_COLUMNS = [
-	{
-		name: "urlHash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "inspectedAt",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "indexStatus",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "lastCrawlTime",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "googleCanonical",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "userCanonical",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "coverageState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "robotsTxtState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "indexingState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "pageFetchState",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "mobileUsabilityVerdict",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "richResultsVerdict",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "scheduleNextAt",
-		type: "BIGINT",
-		nullable: true
-	},
-	{
-		name: "scheduleConsecutiveUnchanged",
-		type: "INTEGER",
-		nullable: true
-	},
-	{
-		name: "schedulePolicyVersion",
-		type: "INTEGER",
-		nullable: true
-	}
-];
-const INSPECTION_EVENT_COLUMNS = [
-	...INSPECTION_PARQUET_COLUMNS,
-	{
-		name: "crawlingUserAgent",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "richResultsItems",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "sitemaps",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "referringUrls",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "mobileIssues",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "inspectionResultLink",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "firstCheckedAt",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "checkCount",
-		type: "INTEGER",
-		nullable: true
-	},
-	{
-		name: "nextCheckAfter",
-		type: "BIGINT",
-		nullable: true
-	},
-	{
-		name: "nextCheckPriority",
-		type: "VARCHAR",
-		nullable: true
-	}
-];
-function createInspectionStore(opts) {
-	const ds = opts.dataSource;
-	function shardFor(record) {
-		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
-		return m ? `${m[1]}-${m[2]}` : "unknown";
-	}
-	function randomBatchId() {
-		return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
-	}
-	return {
-		async appendHistory(ctx, records, options) {
-			if (records.length === 0) return;
-			const batchId = options?.batchId ?? randomBatchId();
-			const byMonth = /* @__PURE__ */ new Map();
-			for (const r of records) {
-				const month = shardFor(r);
-				if (!byMonth.has(month)) byMonth.set(month, []);
-				byMonth.get(month).push(r);
-			}
-			await mapEntityIo([...byMonth].map(([yearMonth, batch]) => {
-				const bytes = encodeJsonBigintSafe({
-					version: 1,
-					records: batch
-				});
-				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
-				return {
-					key: inspectionHistoryShardKey(ctx, yearMonth, batchId),
-					bytes
-				};
-			}), (shard) => ds.write(shard.key, shard.bytes));
-		},
-		async loadHistory(ctx, yearMonth) {
-			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));
-			if (keys.length === 0) return void 0;
-			return {
-				version: 1,
-				records: (await mapEntityIo(keys, async (key) => {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) return [];
-					return (await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
-						console.warn("[inspection.loadHistory] failed to decode shard", {
-							key,
-							error: err.message
-						});
-					}))?.records ?? [];
-				})).flat()
-			};
-		},
-		async materialize(ctx, rowIter) {
-			const rows = Array.from(rowIter);
-			rows.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-			const bytes = encodeRowsToParquetFlex(rows, {
-				columns: INSPECTION_PARQUET_COLUMNS,
-				sortKey: ["urlHash"]
-			});
-			const key = inspectionParquetKey(ctx);
-			await ds.write(key, bytes);
-			return {
-				key,
-				rowCount: rows.length,
-				bytes: bytes.byteLength
-			};
-		},
-		async appendInspectionEvents(ctx, rows, options) {
-			if (rows.length === 0) return {
-				keys: [],
-				rowCount: 0
-			};
-			const batchId = options?.batchId ?? randomBatchId();
-			const byMonth = /* @__PURE__ */ new Map();
-			for (const r of rows) {
-				const m = YEAR_MONTH_RE.exec(r.inspectedAt);
-				const month = m ? `${m[1]}-${m[2]}` : "unknown";
-				const bucket = byMonth.get(month) ?? [];
-				bucket.push(r);
-				byMonth.set(month, bucket);
-			}
-			const files = [...byMonth].map(([month, batch]) => {
-				const bytes = encodeRowsToParquetFlex(batch, {
-					columns: INSPECTION_EVENT_COLUMNS,
-					sortKey: ["urlHash"]
-				});
-				return {
-					key: inspectionEventKey(ctx, month, batchId),
-					bytes
-				};
-			});
-			await mapEntityIo(files, (file) => ds.write(file.key, file.bytes));
-			return {
-				keys: files.map((file) => file.key),
-				rowCount: rows.length
-			};
-		},
-		async compactInspections(ctx) {
-			const eventKeys = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((k) => INSPECTION_EVENT_KEY_RE.test(k));
-			if (eventKeys.length === 0) return {
-				baseRowCount: 0,
-				eventsFolded: 0,
-				eventFilesDeleted: 0
-			};
-			const baseKey = inspectionBaseKey(ctx);
-			const baseBytes = await readOptional(ds, baseKey);
-			const baseRows = baseBytes ? await decodeParquetToRows(baseBytes) : [];
-			const latest = /* @__PURE__ */ new Map();
-			const earliestChecked = /* @__PURE__ */ new Map();
-			const consider = (row) => {
-				const h = String(row.urlHash);
-				const prev = latest.get(h);
-				if (!prev || String(row.inspectedAt ?? "") > String(prev.inspectedAt ?? "")) latest.set(h, row);
-				const fc = row.firstCheckedAt;
-				if (fc != null) {
-					const fcStr = String(fc);
-					const cur = earliestChecked.get(h);
-					if (cur === void 0 || fcStr < cur) earliestChecked.set(h, fcStr);
-				}
-			};
-			for (const row of baseRows) consider(row);
-			let eventsFolded = 0;
-			const consumed = [];
-			const eventFiles = await mapEntityIo(eventKeys.sort(), async (key) => {
-				const bytes = await readOptional(ds, key);
-				if (!bytes) return void 0;
-				return {
-					key,
-					rows: await decodeParquetToRows(bytes)
-				};
-			});
-			for (const file of eventFiles) {
-				if (!file) continue;
-				const { key, rows } = file;
-				consumed.push(key);
-				for (const row of rows) {
-					consider(row);
-					eventsFolded++;
-				}
-			}
-			const merged = [];
-			for (const [h, row] of latest) {
-				const fc = earliestChecked.get(h);
-				if (fc !== void 0) row.firstCheckedAt = fc;
-				merged.push(row);
-			}
-			const bytes = encodeRowsToParquetFlex(merged, {
-				columns: INSPECTION_EVENT_COLUMNS,
-				sortKey: ["urlHash"]
-			});
-			await ds.write(baseKey, bytes);
-			if (consumed.length > 0) await ds.delete(consumed);
-			return {
-				baseRowCount: merged.length,
-				eventsFolded,
-				eventFilesDeleted: consumed.length
-			};
-		},
-		parquetUri(ctx) {
-			return ds.uri?.(inspectionParquetKey(ctx));
-		}
-	};
-}
-const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
-const URLS_INDEX_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "first_seen_at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "last_seen_at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "removed_at",
-		type: "BIGINT",
-		nullable: true
-	}
-];
-const URLS_DELTA_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "op",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "at",
-		type: "BIGINT",
-		nullable: false
-	}
-];
-function rowToUrlRecord(row) {
-	return {
-		feedpath: String(row.feedpath),
-		feedpathHash: String(row.feedpath_hash),
-		urlHash: String(row.url_hash),
-		loc: String(row.loc),
-		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-		firstSeenAt: Number(row.first_seen_at),
-		lastSeenAt: Number(row.last_seen_at),
-		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
-	};
-}
-function urlRecordToRow(r) {
-	return {
-		feedpath: r.feedpath,
-		feedpath_hash: r.feedpathHash,
-		url_hash: r.urlHash,
-		loc: r.loc,
-		lastmod: r.lastmod ?? null,
-		first_seen_at: r.firstSeenAt,
-		last_seen_at: r.lastSeenAt,
-		removed_at: r.removedAt ?? null
-	};
-}
-function isoDate(ms) {
-	return new Date(ms).toISOString().slice(0, 10);
-}
-function createSitemapStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	const now = opts.now ?? (() => Date.now());
-	async function readJson(key) {
-		const bytes = await readOptional(ds, key);
-		if (bytes === void 0) return void 0;
-		return JSON.parse(new TextDecoder().decode(bytes));
-	}
-	async function writeJson(key, value) {
-		await ds.write(key, encodeJsonBigintSafe(value));
-	}
-	return {
-		async writeSnapshot(ctx, records) {
-			if (records.length === 0) return;
-			const indexKey = sitemapIndexKey(ctx);
-			const index = await readJson(indexKey) ?? {
-				version: 1,
-				records: {}
-			};
-			const stamp = now();
-			const historyDocs = /* @__PURE__ */ new Map();
-			for (const r of records) {
-				const h = hash(r.path);
-				index.records[h] = r;
-				historyDocs.set(sitemapHistoryKey(ctx, h, stamp), {
-					version: 1,
-					path: r.path,
-					capturedAt: r.capturedAt,
-					record: r
-				});
-			}
-			await mapEntityIo([...historyDocs], ([key, doc]) => writeJson(key, doc));
-			await writeJson(indexKey, index);
-		},
-		async loadIndex(ctx) {
-			return await readJson(sitemapIndexKey(ctx)) ?? {
-				version: 1,
-				records: {}
-			};
-		},
-		async getLatest(ctx, path) {
-			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
-		},
-		async snapshotUrls(ctx, feedpath, urls) {
-			const fpHash = hash(feedpath);
-			const contentHash = hashUrlList(urls);
-			const at = now();
-			const priorByHash = /* @__PURE__ */ new Map();
-			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
-			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
-			if (livePrior.length > 0) {
-				if (hashSortedUrlList(livePrior.map((r) => String(r.loc)).sort()) === contentHash) return {
-					added: 0,
-					removed: 0,
-					kept: livePrior.length,
-					contentHash,
-					unchanged: true
-				};
-			}
-			const incomingByHash = /* @__PURE__ */ new Map();
-			for (const u of urls) incomingByHash.set(hash(u.loc), u);
-			const deltaRows = [];
-			let added = 0;
-			let removed = 0;
-			let kept = 0;
-			const date = isoDate(at);
-			for (const [urlHash, u] of incomingByHash) {
-				const prev = priorByHash.get(urlHash);
-				if (!prev || prev.removedAt != null) {
-					added++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: fpHash,
-						url_hash: urlHash,
-						op: "added",
-						loc: u.loc,
-						lastmod: u.lastmod ?? null,
-						at
-					});
-				} else kept++;
-			}
-			for (const [urlHash, prev] of priorByHash) {
-				if (prev.removedAt != null) continue;
-				if (!incomingByHash.has(urlHash)) {
-					removed++;
-					deltaRows.push({
-						feedpath,
-						feedpath_hash: fpHash,
-						url_hash: urlHash,
-						op: "removed",
-						loc: prev.loc,
-						lastmod: prev.lastmod ?? null,
-						at
-					});
-				}
-			}
-			if (deltaRows.length > 0) {
-				const bytes = encodeRowsToParquetFlex(deltaRows, {
-					columns: URLS_DELTA_COLUMNS,
-					sortKey: ["url_hash"]
-				});
-				await ds.write(sitemapUrlsDeltaKey(ctx, fpHash, date), bytes);
-			}
-			return {
-				added,
-				removed,
-				kept,
-				contentHash,
-				unchanged: false
-			};
-		},
-		async *loadUrls(ctx, feedpath, opts) {
-			const fpHash = hash(feedpath);
-			const includeRemoved = opts?.includeRemoved ?? false;
-			const [indexRows, listedDeltaKeys] = await Promise.all([readOptional(ds, sitemapUrlsIndexKey(ctx, fpHash)).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
-			const deltaKeys = listedDeltaKeys.filter((key) => {
-				return SITEMAP_URLS_DELTA_PREFIX_RE.exec(key)?.[2] === fpHash;
-			}).sort();
-			const live = /* @__PURE__ */ new Map();
-			const removedMap = /* @__PURE__ */ new Map();
-			for (const row of indexRows) {
-				const rec = rowToUrlRecord(row);
-				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
-				else live.set(rec.urlHash, rec);
-			}
-			const deltas = await mapEntityIo(deltaKeys, async (key) => {
-				const dBytes = await readOptional(ds, key);
-				if (!dBytes) return [];
-				return decodeParquetToRows(dBytes);
-			});
-			for (const dRows of deltas) for (const r of dRows) {
-				const op = String(r.op);
-				const urlHash = String(r.url_hash);
-				const at = Number(r.at);
-				if (op === "added") {
-					const prev = live.get(urlHash) ?? removedMap.get(urlHash);
-					removedMap.delete(urlHash);
-					live.set(urlHash, {
-						feedpath,
-						feedpathHash: fpHash,
-						urlHash,
-						loc: String(r.loc),
-						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-						firstSeenAt: prev?.firstSeenAt ?? at,
-						lastSeenAt: at
-					});
-				} else if (op === "removed") {
-					const prev = live.get(urlHash);
-					live.delete(urlHash);
-					if (prev) removedMap.set(urlHash, {
-						...prev,
-						removedAt: at
-					});
-				}
-			}
-			for (const rec of live.values()) yield rec;
-			if (includeRemoved) for (const rec of removedMap.values()) yield rec;
-		},
-		async *loadDeltas(ctx, dateRange) {
-			const from = dateRange?.from;
-			const to = dateRange?.to;
-			const keys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
-			for (const key of keys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				const date = m[1];
-				if (!date) continue;
-				if (from && date < from) continue;
-				if (to && date > to) continue;
-				const bytes = await readOptional(ds, key);
-				if (!bytes) continue;
-				const rows = await decodeParquetToRows(bytes);
-				for (const r of rows) {
-					const op = String(r.op);
-					if (op !== "added" && op !== "removed") continue;
-					yield {
-						feedpath: String(r.feedpath),
-						feedpathHash: String(r.feedpath_hash),
-						urlHash: String(r.url_hash),
-						op,
-						loc: String(r.loc),
-						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-						at: Number(r.at)
-					};
-				}
-			}
-		},
-		async compactUrls(ctx, opts = {}) {
-			const startedAt = now();
-			const deadlineMs = opts.deadlineMs ?? Number.POSITIVE_INFINITY;
-			const maxFeedpaths = opts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
-			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
-			const deltasByFeed = /* @__PURE__ */ new Map();
-			for (const key of deltaKeys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				const feedpathHash = m[2];
-				if (!feedpathHash) continue;
-				const list = deltasByFeed.get(feedpathHash) ?? [];
-				list.push(key);
-				deltasByFeed.set(feedpathHash, list);
-			}
-			const totalFeedpaths = deltasByFeed.size;
-			let compactedFeedpaths = 0;
-			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
-				if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
-				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
-				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo(feedDeltaKeys.sort(), async (key) => {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) return void 0;
-					return {
-						key,
-						rows: await decodeParquetToRows(bytes)
-					};
-				})]);
-				const live = /* @__PURE__ */ new Map();
-				const removed = /* @__PURE__ */ new Map();
-				for (const row of indexRows) {
-					const rec = rowToUrlRecord(row);
-					if (rec.removedAt != null) removed.set(rec.urlHash, rec);
-					else live.set(rec.urlHash, rec);
-				}
-				const consumed = [];
-				for (const file of deltaFiles) {
-					if (!file) continue;
-					const { key, rows } = file;
-					consumed.push(key);
-					for (const r of rows) {
-						const urlHash = String(r.url_hash);
-						const at = Number(r.at);
-						const op = String(r.op);
-						if (op === "added") {
-							const prev = live.get(urlHash) ?? removed.get(urlHash);
-							removed.delete(urlHash);
-							live.set(urlHash, {
-								feedpath: String(r.feedpath),
-								feedpathHash: fpHash,
-								urlHash,
-								loc: String(r.loc),
-								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-								firstSeenAt: prev?.firstSeenAt ?? at,
-								lastSeenAt: at
-							});
-						} else if (op === "removed") {
-							const prev = live.get(urlHash);
-							live.delete(urlHash);
-							if (prev) removed.set(urlHash, {
-								...prev,
-								removedAt: at
-							});
-						}
-					}
-				}
-				const merged = [...live.values(), ...removed.values()];
-				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
-					columns: URLS_INDEX_COLUMNS,
-					sortKey: ["feedpath_hash", "url_hash"]
-				});
-				await ds.write(indexKey, bytes);
-				if (consumed.length > 0) await ds.delete(consumed);
-				compactedFeedpaths++;
-			}
-			return {
-				compactedFeedpaths,
-				remainingFeedpaths: totalFeedpaths - compactedFeedpaths
-			};
-		},
-		async reconcile(ctx, { liveFeedpaths, at: atOpt }) {
-			const at = atOpt ?? now();
-			const liveHashes = new Set(liveFeedpaths.map((fp) => hash(fp)));
-			const present = /* @__PURE__ */ new Set();
-			const [indexKeys, deltaKeys] = await Promise.all([ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`), ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)]);
-			for (const key of indexKeys) {
-				const m = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
-				if (m) present.add(m[1]);
-			}
-			const deltasByFeed = /* @__PURE__ */ new Map();
-			for (const key of deltaKeys) {
-				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
-				if (!m) continue;
-				present.add(m[2]);
-				const list = deltasByFeed.get(m[2]) ?? [];
-				list.push(key);
-				deltasByFeed.set(m[2], list);
-			}
-			let feedpathsPruned = 0;
-			let urlsRemoved = 0;
-			for (const fpHash of present) {
-				if (liveHashes.has(fpHash)) continue;
-				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
-				const [indexRows, deltaFiles] = await Promise.all([readOptional(ds, indexKey).then((bytes) => bytes ? decodeParquetToRows(bytes) : []), mapEntityIo((deltasByFeed.get(fpHash) ?? []).sort(), async (key) => {
-					const bytes = await readOptional(ds, key);
-					if (!bytes) return void 0;
-					return {
-						key,
-						rows: await decodeParquetToRows(bytes)
-					};
-				})]);
-				const live = /* @__PURE__ */ new Map();
-				const removed = /* @__PURE__ */ new Map();
-				for (const row of indexRows) {
-					const r = rowToUrlRecord(row);
-					if (r.removedAt != null) removed.set(r.urlHash, r);
-					else live.set(r.urlHash, r);
-				}
-				const consumed = [];
-				for (const file of deltaFiles) {
-					if (!file) continue;
-					const { key, rows } = file;
-					consumed.push(key);
-					for (const r of rows) {
-						const urlHash = String(r.url_hash);
-						const dat = Number(r.at);
-						if (String(r.op) === "added") {
-							const prev = live.get(urlHash) ?? removed.get(urlHash);
-							removed.delete(urlHash);
-							live.set(urlHash, {
-								feedpath: String(r.feedpath),
-								feedpathHash: fpHash,
-								urlHash,
-								loc: String(r.loc),
-								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-								firstSeenAt: prev?.firstSeenAt ?? dat,
-								lastSeenAt: dat
-							});
-						} else if (String(r.op) === "removed") {
-							const prev = live.get(urlHash);
-							live.delete(urlHash);
-							if (prev) removed.set(urlHash, {
-								...prev,
-								removedAt: dat
-							});
-						}
-					}
-				}
-				const hadLive = live.size > 0;
-				if (!hadLive && consumed.length === 0) continue;
-				for (const [urlHash, r] of live) {
-					removed.set(urlHash, {
-						...r,
-						removedAt: at
-					});
-					urlsRemoved++;
-				}
-				const merged = [...removed.values()];
-				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
-				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
-					columns: URLS_INDEX_COLUMNS,
-					sortKey: ["feedpath_hash", "url_hash"]
-				});
-				await ds.write(indexKey, bytes);
-				if (consumed.length > 0) await ds.delete(consumed);
-				if (hadLive) feedpathsPruned++;
-			}
-			return {
-				feedpathsPruned,
-				urlsRemoved
-			};
-		}
-	};
-}
-function createIndexingMetadataStore(opts) {
-	const ds = opts.dataSource;
-	const hash = opts.hash ?? hashUrl;
-	async function readIndex(key) {
-		const bytes = await readOptional(ds, key);
-		if (bytes === void 0) return {
-			version: 1,
-			records: {}
-		};
-		return JSON.parse(new TextDecoder().decode(bytes));
-	}
-	return {
-		async writeBatch(ctx, records) {
-			if (records.length === 0) return;
-			const key = indexingMetadataIndexKey(ctx);
-			const index = await readIndex(key);
-			for (const r of records) index.records[hash(r.url)] = r;
-			await ds.write(key, encodeJsonBigintSafe(index));
-		},
-		async loadIndex(ctx) {
-			return readIndex(indexingMetadataIndexKey(ctx));
-		},
-		async getLatest(ctx, url) {
-			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
-		}
-	};
-}
-function createEmptyTypesStore(opts) {
-	const ds = opts.dataSource;
-	const now = opts.now ?? (() => Date.now());
-	async function readDoc(key) {
-		const bytes = await readOptional(ds, key);
-		if (bytes === void 0) return {
-			version: 1,
-			emptyTypes: [],
-			markedAt: {}
-		};
-		return JSON.parse(new TextDecoder().decode(bytes));
-	}
-	async function writeDoc(key, doc) {
-		await ds.write(key, encodeJsonBigintSafe(doc));
-	}
-	return {
-		async load(ctx) {
-			return readDoc(emptyTypesKey(ctx));
-		},
-		async mark(ctx, types, at) {
-			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
-			const key = emptyTypesKey(ctx);
-			const doc = await readDoc(key);
-			const stamp = at ?? now();
-			let changed = false;
-			for (const t of types) {
-				if (!doc.emptyTypes.includes(t)) {
-					doc.emptyTypes.push(t);
-					changed = true;
-				}
-				if (doc.markedAt[t] === void 0) {
-					doc.markedAt[t] = stamp;
-					changed = true;
-				}
-			}
-			if (changed) {
-				doc.emptyTypes.sort();
-				await writeDoc(key, doc);
-			}
-			return doc;
-		},
-		async clear(ctx, types) {
-			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
-			const key = emptyTypesKey(ctx);
-			const doc = await readDoc(key);
-			const drop = new Set(types);
-			const before = doc.emptyTypes.length;
-			doc.emptyTypes = doc.emptyTypes.filter((t) => !drop.has(t));
-			for (const t of drop) delete doc.markedAt[t];
-			if (doc.emptyTypes.length !== before) await writeDoc(key, doc);
-			return doc;
-		}
-	};
-}
-export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore };