@gscdump/engine 0.25.14 → 0.26.0

package/dist/_chunks/dispatch.mjs

@@ -1,13 +1,23 @@
+import { engineErrors } from "../errors.mjs";
+import { isQueryError } from "gscdump/query";
 var AnalyzerCapabilityError = class extends Error {
 	tool;
 	missing;
+	engineError;
 	constructor(tool, missing) {
-		super(`analyzer "${tool}" requires capabilities [${missing.join(", ")}] not provided by source`);
+		const engineError = engineErrors.analyzerCapabilityMissing(tool, missing);
+		super(engineError.message);
 		this.tool = tool;
 		this.missing = missing;
 		this.name = "AnalyzerCapabilityError";
+		this.engineError = engineError;
 	}
 };
+function isUnresolvableDatasetError(err) {
+	const queryError = err?.queryError;
+	if (isQueryError(queryError) && queryError.kind === "unresolvable-dataset") return true;
+	return err?.name === "UnresolvableDatasetError";
+}
 function sourceHas(source, cap) {
 	if (cap === "executeSql") return typeof source.executeSql === "function";
 	return source.capabilities[cap] === true;
@@ -28,7 +38,7 @@ async function runAnalyzerFromSource(source, params, registry) {
 	try {
 		plan = analyzer.build(params, buildCtx);
 	} catch (err) {
-		const rowsVariant = err?.name === "UnresolvableDatasetError" ? registry.getAnalyzerVariants(params.type)?.rows : void 0;
+		const rowsVariant = isUnresolvableDatasetError(err) ? registry.getAnalyzerVariants(params.type)?.rows : void 0;
 		if (!rowsVariant) throw err;
 		assertSatisfies(rowsVariant, source);
 		analyzer = rowsVariant;

package/dist/_chunks/entities.mjs

@@ -0,0 +1,664 @@
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
+function isMissingKeyError(e) {
+	if (typeof e !== "object" || e === null) return false;
+	if (e.code === "ENOENT") return true;
+	if (e.name === "NotFoundError") return true;
+	const message = e.message;
+	if (typeof message !== "string") return false;
+	return /\bnot found\b|\bENOENT\b|\bmissing key\b/i.test(message);
+}
+async function readOptional(ds, key, signal) {
+	if (ds.head) {
+		if (await ds.head(key) === void 0) return void 0;
+	}
+	return await ds.read(key, void 0, signal).catch((e) => {
+		if (isMissingKeyError(e)) return void 0;
+		throw e;
+	});
+}
+const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
+function inspectionIndexKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
+}
+function emptyTypesKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/empty-types.json` : `u_${ctx.userId}/entities/empty-types.json`;
+}
+function inspectionParquetKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.parquet` : `u_${ctx.userId}/entities/inspections/index.parquet`;
+}
+function inspectionHistoryPrefix(ctx, yearMonth) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/history/${yearMonth}` : `u_${ctx.userId}/entities/inspections/history/${yearMonth}`;
+}
+function inspectionHistoryShardKey(ctx, yearMonth, batchId) {
+	return `${inspectionHistoryPrefix(ctx, yearMonth)}/${batchId}.json`;
+}
+function hashUrl(url) {
+	let hi = 2166136261;
+	let lo = 3421674724;
+	for (let i = 0; i < url.length; i++) {
+		const c = url.charCodeAt(i);
+		lo ^= c;
+		const loMul = Math.imul(lo, 435) >>> 0;
+		const carry = Math.floor(lo * 435 / 4294967296);
+		const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
+		lo = loMul;
+		hi = hiMul;
+	}
+	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
+}
+const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
+const INSPECTION_PARQUET_COLUMNS = [
+	{
+		name: "urlHash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "inspectedAt",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "indexStatus",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "lastCrawlTime",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "googleCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "userCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "coverageState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "robotsTxtState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "indexingState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "pageFetchState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileUsabilityVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "scheduleNextAt",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "scheduleConsecutiveUnchanged",
+		type: "INTEGER",
+		nullable: true
+	},
+	{
+		name: "schedulePolicyVersion",
+		type: "INTEGER",
+		nullable: true
+	}
+];
+function createInspectionStore(opts) {
+	const ds = opts.dataSource;
+	function shardFor(record) {
+		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
+		return m ? `${m[1]}-${m[2]}` : "unknown";
+	}
+	function randomBatchId() {
+		return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+	}
+	return {
+		async appendHistory(ctx, records, options) {
+			if (records.length === 0) return;
+			const batchId = options?.batchId ?? randomBatchId();
+			const byMonth = /* @__PURE__ */ new Map();
+			for (const r of records) {
+				const month = shardFor(r);
+				if (!byMonth.has(month)) byMonth.set(month, []);
+				byMonth.get(month).push(r);
+			}
+			for (const [yearMonth, batch] of byMonth) {
+				const shard = {
+					version: 1,
+					records: batch
+				};
+				const bytes = new TextEncoder().encode(JSON.stringify(shard));
+				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
+				await ds.write(inspectionHistoryShardKey(ctx, yearMonth, batchId), bytes);
+			}
+		},
+		async loadHistory(ctx, yearMonth) {
+			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));
+			if (keys.length === 0) return void 0;
+			const out = [];
+			for (const key of keys) {
+				const bytes = await readOptional(ds, key);
+				if (!bytes) continue;
+				const shard = await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
+					console.warn("[inspection.loadHistory] failed to decode shard", {
+						key,
+						error: err.message
+					});
+				});
+				if (shard?.records) out.push(...shard.records);
+			}
+			return {
+				version: 1,
+				records: out
+			};
+		},
+		async materialize(ctx, rowIter) {
+			const rows = Array.from(rowIter);
+			rows.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+			const bytes = encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_PARQUET_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			const key = inspectionParquetKey(ctx);
+			await ds.write(key, bytes);
+			return {
+				key,
+				rowCount: rows.length,
+				bytes: bytes.byteLength
+			};
+		},
+		parquetUri(ctx) {
+			return ds.uri?.(inspectionParquetKey(ctx));
+		}
+	};
+}
+function sitemapIndexKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/index.json` : `u_${ctx.userId}/entities/sitemaps/index.json`;
+}
+function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json` : `u_${ctx.userId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
+}
+function sitemapUrlsPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/urls` : `u_${ctx.userId}/entities/sitemaps/urls`;
+}
+function sitemapUrlsIndexPrefix(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/by-feed`;
+}
+function sitemapUrlsIndexKey(ctx, feedpathHash) {
+	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
+}
+function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
+	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
+}
+const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
+const URLS_INDEX_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "first_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "last_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "removed_at",
+		type: "BIGINT",
+		nullable: true
+	}
+];
+const URLS_DELTA_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "op",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "at",
+		type: "BIGINT",
+		nullable: false
+	}
+];
+function rowToUrlRecord(row) {
+	return {
+		feedpath: String(row.feedpath),
+		feedpathHash: String(row.feedpath_hash),
+		urlHash: String(row.url_hash),
+		loc: String(row.loc),
+		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+		firstSeenAt: Number(row.first_seen_at),
+		lastSeenAt: Number(row.last_seen_at),
+		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
+	};
+}
+function urlRecordToRow(r) {
+	return {
+		feedpath: r.feedpath,
+		feedpath_hash: r.feedpathHash,
+		url_hash: r.urlHash,
+		loc: r.loc,
+		lastmod: r.lastmod ?? null,
+		first_seen_at: r.firstSeenAt,
+		last_seen_at: r.lastSeenAt,
+		removed_at: r.removedAt ?? null
+	};
+}
+function isoDate(ms) {
+	return new Date(ms).toISOString().slice(0, 10);
+}
+function hashUrlList(urls) {
+	return hashUrl(urls.map((u) => u.loc).sort().join("\n"));
+}
+function createSitemapStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	const now = opts.now ?? (() => Date.now());
+	async function readJson(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return void 0;
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	async function writeJson(key, value) {
+		await ds.write(key, new TextEncoder().encode(JSON.stringify(value)));
+	}
+	return {
+		async writeSnapshot(ctx, records) {
+			if (records.length === 0) return;
+			const indexKey = sitemapIndexKey(ctx);
+			const index = await readJson(indexKey) ?? {
+				version: 1,
+				records: {}
+			};
+			const stamp = now();
+			for (const r of records) {
+				const h = hash(r.path);
+				index.records[h] = r;
+				await writeJson(sitemapHistoryKey(ctx, h, stamp), {
+					version: 1,
+					path: r.path,
+					capturedAt: r.capturedAt,
+					record: r
+				});
+			}
+			await writeJson(indexKey, index);
+		},
+		async loadIndex(ctx) {
+			return await readJson(sitemapIndexKey(ctx)) ?? {
+				version: 1,
+				records: {}
+			};
+		},
+		async getLatest(ctx, path) {
+			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
+		},
+		async snapshotUrls(ctx, feedpath, urls) {
+			const fpHash = hash(feedpath);
+			const contentHash = hashUrlList(urls);
+			const at = now();
+			const priorByHash = /* @__PURE__ */ new Map();
+			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
+			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
+			if (livePrior.length > 0) {
+				if (hashUrl(livePrior.map((r) => String(r.loc)).sort().join("\n")) === contentHash) return {
+					added: 0,
+					removed: 0,
+					kept: livePrior.length,
+					contentHash,
+					unchanged: true
+				};
+			}
+			const incomingByHash = /* @__PURE__ */ new Map();
+			for (const u of urls) incomingByHash.set(hash(u.loc), u);
+			const deltaRows = [];
+			let added = 0;
+			let removed = 0;
+			let kept = 0;
+			const date = isoDate(at);
+			for (const [urlHash, u] of incomingByHash) {
+				const prev = priorByHash.get(urlHash);
+				if (!prev || prev.removedAt != null) {
+					added++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "added",
+						loc: u.loc,
+						lastmod: u.lastmod ?? null,
+						at
+					});
+				} else kept++;
+			}
+			for (const [urlHash, prev] of priorByHash) {
+				if (prev.removedAt != null) continue;
+				if (!incomingByHash.has(urlHash)) {
+					removed++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "removed",
+						loc: prev.loc,
+						lastmod: prev.lastmod ?? null,
+						at
+					});
+				}
+			}
+			if (deltaRows.length > 0) {
+				const bytes = encodeRowsToParquetFlex(deltaRows, {
+					columns: URLS_DELTA_COLUMNS,
+					sortKey: ["url_hash"]
+				});
+				await ds.write(sitemapUrlsDeltaKey(ctx, fpHash, date), bytes);
+			}
+			return {
+				added,
+				removed,
+				kept,
+				contentHash,
+				unchanged: false
+			};
+		},
+		async *loadUrls(ctx, feedpath, opts) {
+			const fpHash = hash(feedpath);
+			const includeRemoved = opts?.includeRemoved ?? false;
+			const indexBytes = await readOptional(ds, sitemapUrlsIndexKey(ctx, fpHash));
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			const live = /* @__PURE__ */ new Map();
+			const removedMap = /* @__PURE__ */ new Map();
+			for (const row of indexRows) {
+				const rec = rowToUrlRecord(row);
+				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
+				else live.set(rec.urlHash, rec);
+			}
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m || m[2] !== fpHash) continue;
+				const dBytes = await readOptional(ds, key);
+				if (!dBytes) continue;
+				const dRows = await decodeParquetToRows(dBytes);
+				for (const r of dRows) {
+					const op = String(r.op);
+					const urlHash = String(r.url_hash);
+					const at = Number(r.at);
+					if (op === "added") {
+						const prev = live.get(urlHash) ?? removedMap.get(urlHash);
+						removedMap.delete(urlHash);
+						live.set(urlHash, {
+							feedpath,
+							feedpathHash: fpHash,
+							urlHash,
+							loc: String(r.loc),
+							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+							firstSeenAt: prev?.firstSeenAt ?? at,
+							lastSeenAt: at
+						});
+					} else if (op === "removed") {
+						const prev = live.get(urlHash);
+						live.delete(urlHash);
+						if (prev) removedMap.set(urlHash, {
+							...prev,
+							removedAt: at
+						});
+					}
+				}
+			}
+			for (const rec of live.values()) yield rec;
+			if (includeRemoved) for (const rec of removedMap.values()) yield rec;
+		},
+		async *loadDeltas(ctx, dateRange) {
+			const from = dateRange?.from;
+			const to = dateRange?.to;
+			const keys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			for (const key of keys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const date = m[1];
+				if (from && date < from) continue;
+				if (to && date > to) continue;
+				const bytes = await readOptional(ds, key);
+				if (!bytes) continue;
+				const rows = await decodeParquetToRows(bytes);
+				for (const r of rows) {
+					const op = String(r.op);
+					if (op !== "added" && op !== "removed") continue;
+					yield {
+						feedpath: String(r.feedpath),
+						feedpathHash: String(r.feedpath_hash),
+						urlHash: String(r.url_hash),
+						op,
+						loc: String(r.loc),
+						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+						at: Number(r.at)
+					};
+				}
+			}
+		},
+		async compactUrls(ctx) {
+			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
+			const deltasByFeed = /* @__PURE__ */ new Map();
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const list = deltasByFeed.get(m[2]) ?? [];
+				list.push(key);
+				deltasByFeed.set(m[2], list);
+			}
+			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
+				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
+				const indexBytes = await readOptional(ds, indexKey);
+				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+				const live = /* @__PURE__ */ new Map();
+				const removed = /* @__PURE__ */ new Map();
+				for (const row of indexRows) {
+					const rec = rowToUrlRecord(row);
+					if (rec.removedAt != null) removed.set(rec.urlHash, rec);
+					else live.set(rec.urlHash, rec);
+				}
+				const consumed = [];
+				for (const key of feedDeltaKeys.sort()) {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) continue;
+					consumed.push(key);
+					const rows = await decodeParquetToRows(bytes);
+					for (const r of rows) {
+						const urlHash = String(r.url_hash);
+						const at = Number(r.at);
+						const op = String(r.op);
+						if (op === "added") {
+							const prev = live.get(urlHash) ?? removed.get(urlHash);
+							removed.delete(urlHash);
+							live.set(urlHash, {
+								feedpath: String(r.feedpath),
+								feedpathHash: fpHash,
+								urlHash,
+								loc: String(r.loc),
+								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+								firstSeenAt: prev?.firstSeenAt ?? at,
+								lastSeenAt: at
+							});
+						} else if (op === "removed") {
+							const prev = live.get(urlHash);
+							live.delete(urlHash);
+							if (prev) removed.set(urlHash, {
+								...prev,
+								removedAt: at
+							});
+						}
+					}
+				}
+				const merged = [...live.values(), ...removed.values()];
+				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+					columns: URLS_INDEX_COLUMNS,
+					sortKey: ["feedpath_hash", "url_hash"]
+				});
+				await ds.write(indexKey, bytes);
+				if (consumed.length > 0) await ds.delete(consumed);
+			}
+		}
+	};
+}
+function indexingMetadataIndexKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/indexing/index.json` : `u_${ctx.userId}/entities/indexing/index.json`;
+}
+function createIndexingMetadataStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	async function readIndex(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return {
+			version: 1,
+			records: {}
+		};
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	return {
+		async writeBatch(ctx, records) {
+			if (records.length === 0) return;
+			const key = indexingMetadataIndexKey(ctx);
+			const index = await readIndex(key);
+			for (const r of records) index.records[hash(r.url)] = r;
+			await ds.write(key, new TextEncoder().encode(JSON.stringify(index)));
+		},
+		async loadIndex(ctx) {
+			return readIndex(indexingMetadataIndexKey(ctx));
+		},
+		async getLatest(ctx, url) {
+			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
+		}
+	};
+}
+function createEmptyTypesStore(opts) {
+	const ds = opts.dataSource;
+	const now = opts.now ?? (() => Date.now());
+	async function readDoc(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return {
+			version: 1,
+			emptyTypes: [],
+			markedAt: {}
+		};
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	async function writeDoc(key, doc) {
+		await ds.write(key, new TextEncoder().encode(JSON.stringify(doc)));
+	}
+	return {
+		async load(ctx) {
+			return readDoc(emptyTypesKey(ctx));
+		},
+		async mark(ctx, types, at) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const stamp = at ?? now();
+			let changed = false;
+			for (const t of types) {
+				if (!doc.emptyTypes.includes(t)) {
+					doc.emptyTypes.push(t);
+					changed = true;
+				}
+				if (doc.markedAt[t] === void 0) {
+					doc.markedAt[t] = stamp;
+					changed = true;
+				}
+			}
+			if (changed) {
+				doc.emptyTypes.sort();
+				await writeDoc(key, doc);
+			}
+			return doc;
+		},
+		async clear(ctx, types) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const drop = new Set(types);
+			const before = doc.emptyTypes.length;
+			doc.emptyTypes = doc.emptyTypes.filter((t) => !drop.has(t));
+			for (const t of drop) delete doc.markedAt[t];
+			if (doc.emptyTypes.length !== before) await writeDoc(key, doc);
+			return doc;
+		}
+	};
+}
+export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };