@gscdump/engine 0.4.0

package/dist/entities.d.mts

@@ -0,0 +1,238 @@
+import { TenantCtx } from "gscdump/contracts";
+interface DataSource {
+  read: (key: string, range?: {
+    offset: number;
+    length: number;
+  }, signal?: AbortSignal) => Promise<Uint8Array>;
+  write: (key: string, bytes: Uint8Array) => Promise<void>;
+  delete: (keys: string[]) => Promise<void>;
+  /**
+   * One-shot listing under a prefix. Implementations may cap the number of
+   * returned keys (typically 10k) — callers iterating full tenant space
+   * should prefer `streamList` when available or narrow the prefix.
+   */
+  list: (prefix: string) => Promise<string[]>;
+  /**
+   * Per-key URI probe. Returns a URI string DuckDB's `httpfs` (or an
+   * equivalent engine that fetches its own I/O) can read directly, or
+   * `undefined` if the key isn't URI-resolvable on this backend and the
+   * caller must fall back to `read(key)` for the bytes.
+   *
+   * Contracts:
+   * - When defined, the returned URI MUST yield byte-identical content to
+   *   `read(key)`. Callers rely on this for correctness.
+   * - Backends with a native URI for every key (filesystem: absolute path,
+   *   R2 via `httpfs`: signed URL) may always return a string.
+   * - Backends without a native URI shape (in-memory) omit the method or
+   *   return `undefined` per call.
+   * - Mixed-per-query is allowed: some keys in one query may return a URI,
+   *   others may not; the executor branches per key.
+   */
+  uri?: (key: string) => string | undefined;
+  /**
+   * Optional — probe the byte size of a key without reading it. Used by
+   * the engine to fill in `WriteResult.bytes` when a codec reports 0 or
+   * unknown but the file is non-trivial.
+   */
+  head?: (key: string) => Promise<{
+    bytes: number;
+  } | undefined>;
+  /**
+   * Optional streaming variant of `list`. Implementations that page
+   * backing-store results (R2, S3) should implement this and yield keys
+   * lazily. `list` may return up to an adapter-defined cap (typically
+   * 10k keys); callers iterating full tenant space must prefer
+   * `streamList` when available, or chunk by narrower prefixes.
+   */
+  streamList?: (prefix: string) => AsyncIterable<string>;
+}
+/**
+ * GSC URL inspection result fields we persist. Mirrors the
+ *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON
+ *  so storage doesn't depend on the googleapis type tree.
+ */
+interface InspectionRecord {
+  url: string;
+  /** ISO-8601 timestamp of when we ran the inspection. */
+  inspectedAt: string;
+  /** PASS / NEUTRAL / FAIL — the headline verdict from indexStatusResult. */
+  indexStatus?: string;
+  /** Last-crawl timestamp the API reports (ISO-8601). */
+  lastCrawlTime?: string;
+  /** Canonical URL Google selected. */
+  googleCanonical?: string;
+  /** Canonical URL the page declares. */
+  userCanonical?: string;
+  /** Crawl/index/serving disposition strings as the API returns them. */
+  coverageState?: string;
+  robotsTxtState?: string;
+  indexingState?: string;
+  pageFetchState?: string;
+  mobileUsabilityVerdict?: string;
+  richResultsVerdict?: string;
+  /**
+   * Free-form payload for fields we don't promote to first-class columns
+   *  (e.g. `referringUrls`, `crawledAs`). Keeps the wire format forward-compat
+   *  without bumping the schema for every API addition.
+   */
+  raw?: unknown;
+}
+/** Wire shape persisted to disk/R2. */
+interface InspectionIndex {
+  version: 1;
+  /** Map of urlHash → InspectionRecord (latest only). */
+  records: Record<string, InspectionRecord>;
+}
+interface InspectionHistoryShard {
+  version: 1;
+  /** Append-only list of inspection records for the YYYY-MM bucket. */
+  records: InspectionRecord[];
+}
+declare function inspectionIndexKey(ctx: TenantCtx): string;
+declare function emptyTypesKey(ctx: TenantCtx): string;
+declare function inspectionHistoryKey(ctx: TenantCtx, yearMonth: string): string;
+/**
+ * Stable URL hash used as the index key. Short, URL-safe, deterministic.
+ * Uses a 64-bit FNV-1a; collisions vanishingly unlikely at the scales we
+ * care about (≤100k URLs/site).
+ */
+declare function hashUrl(url: string): string;
+interface InspectionStore {
+  /**
+   * Persist a batch of fresh inspection results. Updates the index +
+   *  appends to the per-month history shard.
+   */
+  writeBatch: (ctx: TenantCtx, records: readonly InspectionRecord[]) => Promise<void>;
+  /** Fetch the latest inspection record for a URL, or undefined. */
+  getLatest: (ctx: TenantCtx, url: string) => Promise<InspectionRecord | undefined>;
+  /**
+   * Read the full index for a site (latest record per URL). Cheap on
+   *  Workers; on big tenants the dashboard reads this once per page load.
+   */
+  loadIndex: (ctx: TenantCtx) => Promise<InspectionIndex>;
+  /** Read the per-month history shard if it exists. */
+  loadHistory: (ctx: TenantCtx, yearMonth: string) => Promise<InspectionHistoryShard | undefined>;
+}
+interface CreateInspectionStoreOptions {
+  dataSource: DataSource;
+  /**
+   * Override the FNV hash with a callable (test seam, or to swap in
+   *  SHA-256 if hash collisions become a concern at extreme scale).
+   */
+  hash?: (url: string) => string;
+  now?: () => number;
+}
+declare function createInspectionStore(opts: CreateInspectionStoreOptions): InspectionStore;
+interface InspectionSqlDriver {
+  exec: (sql: string) => void | Promise<void>;
+  run: (sql: string, params: unknown[]) => void | Promise<void>;
+  all: (sql: string, params: unknown[]) => unknown[] | Promise<unknown[]>;
+  serialize: () => Uint8Array | Promise<Uint8Array>;
+  close: () => void | Promise<void>;
+}
+interface CreateInspectionStoreSqliteOptions {
+  dataSource: DataSource;
+  openDriver: (bytes: Uint8Array | undefined) => InspectionSqlDriver | Promise<InspectionSqlDriver>;
+  hash?: (url: string) => string;
+}
+declare function inspectionSqliteKey(ctx: TenantCtx): string;
+declare function createInspectionStoreSqlite(opts: CreateInspectionStoreSqliteOptions): InspectionStore;
+/** GSC sitemap record we persist. Matches `Schema$WmxSitemap` but as plain JSON. */
+interface SitemapRecord {
+  /** The sitemap URL (feedpath) as returned by GSC. */
+  path: string;
+  /** ISO-8601 timestamp of the snapshot run that captured this record. */
+  capturedAt: string;
+  /** Last time Google downloaded this sitemap (RFC 3339, from the API). */
+  lastDownloaded?: string;
+  /** Last time the sitemap was submitted. */
+  lastSubmitted?: string;
+  type?: string;
+  isPending?: boolean;
+  isSitemapsIndex?: boolean;
+  errors?: string;
+  warnings?: string;
+  /** Per-content-type counts (web, image, video, news). */
+  contents?: Array<{
+    type?: string;
+    submitted?: string;
+    indexed?: string;
+  }>;
+  /** Raw payload for fields we don't promote to first-class columns. */
+  raw?: unknown;
+}
+interface SitemapIndex {
+  version: 1;
+  /** Map of feedpathHash → latest SitemapRecord. */
+  records: Record<string, SitemapRecord>;
+}
+interface SitemapHistoryDoc {
+  version: 1;
+  path: string;
+  capturedAt: string;
+  record: SitemapRecord;
+}
+declare function sitemapIndexKey(ctx: TenantCtx): string;
+declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
+interface SitemapStore {
+  /**
+   * Persist a snapshot run. Updates the index + writes one immutable
+   * history doc per record under `history/<feedpathHash>__<capturedAtMs>.json`.
+   */
+  writeSnapshot: (ctx: TenantCtx, records: readonly SitemapRecord[]) => Promise<void>;
+  /** Load the full site index (latest record per feedpath). */
+  loadIndex: (ctx: TenantCtx) => Promise<SitemapIndex>;
+  /** Fetch the latest snapshot for a feedpath, or undefined. */
+  getLatest: (ctx: TenantCtx, path: string) => Promise<SitemapRecord | undefined>;
+}
+interface CreateSitemapStoreOptions {
+  dataSource: DataSource;
+  /** Override the feedpath hash (test seam). */
+  hash?: (path: string) => string;
+  now?: () => number;
+}
+declare function createSitemapStore(opts: CreateSitemapStoreOptions): SitemapStore;
+interface IndexingMetadataRecord {
+  url: string;
+  capturedAt: string;
+  /** ISO-8601 notifyTime of the latest `URL_UPDATED` notification we've seen. */
+  latestUpdateAt?: string;
+  /** ISO-8601 notifyTime of the latest `URL_REMOVED` notification we've seen. */
+  latestRemoveAt?: string;
+  raw?: unknown;
+}
+interface IndexingMetadataIndex {
+  version: 1;
+  records: Record<string, IndexingMetadataRecord>;
+}
+declare function indexingMetadataIndexKey(ctx: TenantCtx): string;
+interface IndexingMetadataStore {
+  writeBatch: (ctx: TenantCtx, records: readonly IndexingMetadataRecord[]) => Promise<void>;
+  loadIndex: (ctx: TenantCtx) => Promise<IndexingMetadataIndex>;
+  getLatest: (ctx: TenantCtx, url: string) => Promise<IndexingMetadataRecord | undefined>;
+}
+interface CreateIndexingMetadataStoreOptions {
+  dataSource: DataSource;
+  hash?: (url: string) => string;
+}
+declare function createIndexingMetadataStore(opts: CreateIndexingMetadataStoreOptions): IndexingMetadataStore;
+interface EmptyTypesDoc {
+  version: 1;
+  /** SearchType strings detected as empty for this (user, site). */
+  emptyTypes: string[];
+  /** When each type was last marked empty (unix ms). Helps debug stale skips. */
+  markedAt: Record<string, number>;
+}
+interface EmptyTypesStore {
+  load: (ctx: TenantCtx) => Promise<EmptyTypesDoc>;
+  /** Add types to the empty set, preserving existing markers. No-op if all already present. */
+  mark: (ctx: TenantCtx, types: readonly string[], now?: number) => Promise<EmptyTypesDoc>;
+  /** Remove types from the empty set. Returns the updated doc. */
+  clear: (ctx: TenantCtx, types: readonly string[]) => Promise<EmptyTypesDoc>;
+}
+interface CreateEmptyTypesStoreOptions {
+  dataSource: DataSource;
+  now?: () => number;
+}
+declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateInspectionStoreSqliteOptions, CreateSitemapStoreOptions, EmptyTypesDoc, EmptyTypesStore, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionIndex, InspectionRecord, InspectionSqlDriver, InspectionStore, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createInspectionStoreSqlite, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionSqliteKey, sitemapHistoryKey, sitemapIndexKey };

package/dist/entities.mjs

@@ -0,0 +1,359 @@
+const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
+function inspectionIndexKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
+}
+function emptyTypesKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/empty-types.json` : `u_${ctx.userId}/entities/empty-types.json`;
+}
+function inspectionHistoryKey(ctx, yearMonth) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/history/${yearMonth}.json` : `u_${ctx.userId}/entities/inspections/history/${yearMonth}.json`;
+}
+function hashUrl(url) {
+	let hi = 2166136261;
+	let lo = 3421674724;
+	for (let i = 0; i < url.length; i++) {
+		const c = url.charCodeAt(i);
+		lo ^= c;
+		const loMul = Math.imul(lo, 435) >>> 0;
+		const carry = Math.floor(lo * 435 / 4294967296);
+		const hiMul = Math.imul(hi, 435) + Math.imul(lo, 1) + carry >>> 0;
+		lo = loMul;
+		hi = hiMul;
+	}
+	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
+}
+function createInspectionStore(opts) {
+	const hash = opts.hash ?? hashUrl;
+	const ds = opts.dataSource;
+	async function readJson(key) {
+		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => void 0);
+	}
+	async function writeJson(key, value) {
+		await ds.write(key, new TextEncoder().encode(JSON.stringify(value)));
+	}
+	function emptyIndex() {
+		return {
+			version: 1,
+			records: {}
+		};
+	}
+	function emptyShard() {
+		return {
+			version: 1,
+			records: []
+		};
+	}
+	function shardFor(record) {
+		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
+		return m ? `${m[1]}-${m[2]}` : "unknown";
+	}
+	return {
+		async writeBatch(ctx, records) {
+			if (records.length === 0) return;
+			const indexKey = inspectionIndexKey(ctx);
+			const index = await readJson(indexKey) ?? emptyIndex();
+			const byShard = /* @__PURE__ */ new Map();
+			for (const r of records) {
+				index.records[hash(r.url)] = r;
+				const shardKey = shardFor(r);
+				if (!byShard.has(shardKey)) byShard.set(shardKey, []);
+				byShard.get(shardKey).push(r);
+			}
+			await writeJson(indexKey, index);
+			for (const [yearMonth, batch] of byShard) {
+				const histKey = inspectionHistoryKey(ctx, yearMonth);
+				const existing = await readJson(histKey) ?? emptyShard();
+				existing.records.push(...batch);
+				await writeJson(histKey, existing);
+			}
+		},
+		async getLatest(ctx, url) {
+			return (await readJson(inspectionIndexKey(ctx)))?.records[hash(url)];
+		},
+		async loadIndex(ctx) {
+			return await readJson(inspectionIndexKey(ctx)) ?? emptyIndex();
+		},
+		async loadHistory(ctx, yearMonth) {
+			return await readJson(inspectionHistoryKey(ctx, yearMonth));
+		}
+	};
+}
+function inspectionSqliteKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/inspections.db` : `u_${ctx.userId}/entities/inspections/inspections.db`;
+}
+const INSPECTION_SCHEMA_SQL = `
+CREATE TABLE IF NOT EXISTS inspections (
+  url_hash TEXT PRIMARY KEY,
+  url TEXT NOT NULL,
+  inspected_at TEXT NOT NULL,
+  index_status TEXT,
+  last_crawl_time TEXT,
+  google_canonical TEXT,
+  user_canonical TEXT,
+  coverage_state TEXT,
+  robots_txt_state TEXT,
+  indexing_state TEXT,
+  page_fetch_state TEXT,
+  mobile_usability_verdict TEXT,
+  rich_results_verdict TEXT,
+  raw TEXT
+);
+CREATE TABLE IF NOT EXISTS inspection_history (
+  year_month TEXT NOT NULL,
+  url_hash TEXT NOT NULL,
+  url TEXT NOT NULL,
+  inspected_at TEXT NOT NULL,
+  payload TEXT NOT NULL,
+  PRIMARY KEY (year_month, url_hash, inspected_at)
+);
+CREATE INDEX IF NOT EXISTS inspection_history_by_month ON inspection_history(year_month);
+`;
+function rowToRecord(r) {
+	const out = {
+		url: r.url,
+		inspectedAt: r.inspected_at
+	};
+	if (r.index_status != null) out.indexStatus = r.index_status;
+	if (r.last_crawl_time != null) out.lastCrawlTime = r.last_crawl_time;
+	if (r.google_canonical != null) out.googleCanonical = r.google_canonical;
+	if (r.user_canonical != null) out.userCanonical = r.user_canonical;
+	if (r.coverage_state != null) out.coverageState = r.coverage_state;
+	if (r.robots_txt_state != null) out.robotsTxtState = r.robots_txt_state;
+	if (r.indexing_state != null) out.indexingState = r.indexing_state;
+	if (r.page_fetch_state != null) out.pageFetchState = r.page_fetch_state;
+	if (r.mobile_usability_verdict != null) out.mobileUsabilityVerdict = r.mobile_usability_verdict;
+	if (r.rich_results_verdict != null) out.richResultsVerdict = r.rich_results_verdict;
+	if (r.raw != null) out.raw = JSON.parse(r.raw);
+	return out;
+}
+function shardForRecord(record) {
+	const m = YEAR_MONTH_RE.exec(record.inspectedAt);
+	return m ? `${m[1]}-${m[2]}` : "unknown";
+}
+function createInspectionStoreSqlite(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	async function withDriver(ctx, fn, persist) {
+		const key = inspectionSqliteKey(ctx);
+		const bytes = await ds.read(key).catch(() => void 0);
+		const driver = await opts.openDriver(bytes);
+		await driver.exec(INSPECTION_SCHEMA_SQL);
+		const result = await fn(driver);
+		if (persist) {
+			const out = await driver.serialize();
+			await ds.write(key, out);
+		}
+		await driver.close();
+		return result;
+	}
+	return {
+		async writeBatch(ctx, records) {
+			if (records.length === 0) return;
+			await withDriver(ctx, async (driver) => {
+				for (const r of records) {
+					const h = hash(r.url);
+					await driver.run(`INSERT INTO inspections (
+              url_hash, url, inspected_at, index_status, last_crawl_time,
+              google_canonical, user_canonical, coverage_state, robots_txt_state,
+              indexing_state, page_fetch_state, mobile_usability_verdict,
+              rich_results_verdict, raw
+            ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+            ON CONFLICT(url_hash) DO UPDATE SET
+              url = excluded.url,
+              inspected_at = excluded.inspected_at,
+              index_status = excluded.index_status,
+              last_crawl_time = excluded.last_crawl_time,
+              google_canonical = excluded.google_canonical,
+              user_canonical = excluded.user_canonical,
+              coverage_state = excluded.coverage_state,
+              robots_txt_state = excluded.robots_txt_state,
+              indexing_state = excluded.indexing_state,
+              page_fetch_state = excluded.page_fetch_state,
+              mobile_usability_verdict = excluded.mobile_usability_verdict,
+              rich_results_verdict = excluded.rich_results_verdict,
+              raw = excluded.raw`, [
+						h,
+						r.url,
+						r.inspectedAt,
+						r.indexStatus ?? null,
+						r.lastCrawlTime ?? null,
+						r.googleCanonical ?? null,
+						r.userCanonical ?? null,
+						r.coverageState ?? null,
+						r.robotsTxtState ?? null,
+						r.indexingState ?? null,
+						r.pageFetchState ?? null,
+						r.mobileUsabilityVerdict ?? null,
+						r.richResultsVerdict ?? null,
+						r.raw === void 0 ? null : JSON.stringify(r.raw)
+					]);
+					await driver.run(`INSERT OR REPLACE INTO inspection_history
+               (year_month, url_hash, url, inspected_at, payload)
+               VALUES (?,?,?,?,?)`, [
+						shardForRecord(r),
+						h,
+						r.url,
+						r.inspectedAt,
+						JSON.stringify(r)
+					]);
+				}
+			}, true);
+		},
+		async getLatest(ctx, url) {
+			return await withDriver(ctx, async (driver) => {
+				const rows = await driver.all("SELECT * FROM inspections WHERE url_hash = ? LIMIT 1", [hash(url)]);
+				return rows.length === 0 ? void 0 : rowToRecord(rows[0]);
+			}, false);
+		},
+		async loadIndex(ctx) {
+			return await withDriver(ctx, async (driver) => {
+				const rows = await driver.all("SELECT * FROM inspections", []);
+				const records = {};
+				for (const r of rows) records[r.url_hash] = rowToRecord(r);
+				return {
+					version: 1,
+					records
+				};
+			}, false);
+		},
+		async loadHistory(ctx, yearMonth) {
+			return await withDriver(ctx, async (driver) => {
+				const rows = await driver.all("SELECT * FROM inspection_history WHERE year_month = ? ORDER BY inspected_at ASC", [yearMonth]);
+				if (rows.length === 0) return void 0;
+				return {
+					version: 1,
+					records: rows.map((r) => JSON.parse(r.payload))
+				};
+			}, false);
+		}
+	};
+}
+function sitemapIndexKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/index.json` : `u_${ctx.userId}/entities/sitemaps/index.json`;
+}
+function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json` : `u_${ctx.userId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
+}
+function createSitemapStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	const now = opts.now ?? (() => Date.now());
+	async function readJson(key) {
+		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => void 0);
+	}
+	async function writeJson(key, value) {
+		await ds.write(key, new TextEncoder().encode(JSON.stringify(value)));
+	}
+	return {
+		async writeSnapshot(ctx, records) {
+			if (records.length === 0) return;
+			const indexKey = sitemapIndexKey(ctx);
+			const index = await readJson(indexKey) ?? {
+				version: 1,
+				records: {}
+			};
+			const stamp = now();
+			for (const r of records) {
+				const h = hash(r.path);
+				index.records[h] = r;
+				await writeJson(sitemapHistoryKey(ctx, h, stamp), {
+					version: 1,
+					path: r.path,
+					capturedAt: r.capturedAt,
+					record: r
+				});
+			}
+			await writeJson(indexKey, index);
+		},
+		async loadIndex(ctx) {
+			return await readJson(sitemapIndexKey(ctx)) ?? {
+				version: 1,
+				records: {}
+			};
+		},
+		async getLatest(ctx, path) {
+			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
+		}
+	};
+}
+function indexingMetadataIndexKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/indexing/index.json` : `u_${ctx.userId}/entities/indexing/index.json`;
+}
+function createIndexingMetadataStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	async function readIndex(key) {
+		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => ({
+			version: 1,
+			records: {}
+		}));
+	}
+	return {
+		async writeBatch(ctx, records) {
+			if (records.length === 0) return;
+			const key = indexingMetadataIndexKey(ctx);
+			const index = await readIndex(key);
+			for (const r of records) index.records[hash(r.url)] = r;
+			await ds.write(key, new TextEncoder().encode(JSON.stringify(index)));
+		},
+		async loadIndex(ctx) {
+			return readIndex(indexingMetadataIndexKey(ctx));
+		},
+		async getLatest(ctx, url) {
+			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
+		}
+	};
+}
+function createEmptyTypesStore(opts) {
+	const ds = opts.dataSource;
+	const now = opts.now ?? (() => Date.now());
+	async function readDoc(key) {
+		return await ds.read(key).then((bytes) => JSON.parse(new TextDecoder().decode(bytes)), () => ({
+			version: 1,
+			emptyTypes: [],
+			markedAt: {}
+		}));
+	}
+	async function writeDoc(key, doc) {
+		await ds.write(key, new TextEncoder().encode(JSON.stringify(doc)));
+	}
+	return {
+		async load(ctx) {
+			return readDoc(emptyTypesKey(ctx));
+		},
+		async mark(ctx, types, at) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const stamp = at ?? now();
+			let changed = false;
+			for (const t of types) {
+				if (!doc.emptyTypes.includes(t)) {
+					doc.emptyTypes.push(t);
+					changed = true;
+				}
+				if (doc.markedAt[t] === void 0) {
+					doc.markedAt[t] = stamp;
+					changed = true;
+				}
+			}
+			if (changed) {
+				doc.emptyTypes.sort();
+				await writeDoc(key, doc);
+			}
+			return doc;
+		},
+		async clear(ctx, types) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const drop = new Set(types);
+			const before = doc.emptyTypes.length;
+			doc.emptyTypes = doc.emptyTypes.filter((t) => !drop.has(t));
+			for (const t of drop) delete doc.markedAt[t];
+			if (doc.emptyTypes.length !== before) await writeDoc(key, doc);
+			return doc;
+		}
+	};
+}
+export { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createInspectionStoreSqlite, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionSqliteKey, sitemapHistoryKey, sitemapIndexKey };