@gscdump/engine 1.4.10 → 1.5.0

package/dist/entities/inspection.mjs

@@ -0,0 +1,443 @@
+import { inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionParquetKey, inspectionTransitionsMonthKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
+import { mapEntityIo } from "./io.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+import { GSCDUMP_INDEXING_TRANSITION_FIELDS } from "@gscdump/contracts";
+import { classifyCanonicalDifference } from "gscdump";
+const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
+const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
+const INSPECTION_HISTORY_MAX_BYTES = 5 * 1024 * 1024;
+const INSPECTION_PARQUET_COLUMNS = [
+	{
+		name: "urlHash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "inspectedAt",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "indexStatus",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "lastCrawlTime",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "googleCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "userCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "coverageState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "robotsTxtState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "indexingState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "pageFetchState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileUsabilityVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "scheduleNextAt",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "scheduleConsecutiveUnchanged",
+		type: "INTEGER",
+		nullable: true
+	},
+	{
+		name: "schedulePolicyVersion",
+		type: "INTEGER",
+		nullable: true
+	}
+];
+const INSPECTION_EVENT_COLUMNS = [
+	...INSPECTION_PARQUET_COLUMNS,
+	{
+		name: "canonicalMismatchKind",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "crawlingUserAgent",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsItems",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "sitemaps",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "referringUrls",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileIssues",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "inspectionResultLink",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "firstCheckedAt",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "checkCount",
+		type: "INTEGER",
+		nullable: true
+	},
+	{
+		name: "nextCheckAfter",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "nextCheckPriority",
+		type: "VARCHAR",
+		nullable: true
+	}
+];
+const CANONICAL_DIFFERENCE_KINDS = /* @__PURE__ */ new Set([
+	"none",
+	"formatting",
+	"path",
+	"cross_domain"
+]);
+function populateCanonicalMismatchKind(row) {
+	const current = row.canonicalMismatchKind;
+	const expected = classifyCanonicalDifference(typeof row.userCanonical === "string" ? row.userCanonical : null, typeof row.googleCanonical === "string" ? row.googleCanonical : null);
+	const missingOrInvalid = typeof current !== "string" || !CANONICAL_DIFFERENCE_KINDS.has(current);
+	row.canonicalMismatchKind = expected;
+	return missingOrInvalid || current !== expected;
+}
+const INSPECTION_TRANSITION_COLUMNS = [
+	{
+		name: "urlHash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "changedAfter",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "changedBefore",
+		type: "VARCHAR",
+		nullable: false
+	},
+	...GSCDUMP_INDEXING_TRANSITION_FIELDS.flatMap((field) => {
+		const capped = field[0].toUpperCase() + field.slice(1);
+		return [{
+			name: `from${capped}`,
+			type: "VARCHAR",
+			nullable: true
+		}, {
+			name: `to${capped}`,
+			type: "VARCHAR",
+			nullable: true
+		}];
+	})
+];
+function isStateTransition(before, after) {
+	return GSCDUMP_INDEXING_TRANSITION_FIELDS.some((field) => (before[field] ?? null) !== (after[field] ?? null));
+}
+function buildTransitionRow(before, after) {
+	const row = {
+		urlHash: String(after.urlHash),
+		url: String(after.url ?? before.url ?? ""),
+		changedAfter: String(before.inspectedAt ?? ""),
+		changedBefore: String(after.inspectedAt ?? "")
+	};
+	for (const field of GSCDUMP_INDEXING_TRANSITION_FIELDS) {
+		const capped = field[0].toUpperCase() + field.slice(1);
+		row[`from${capped}`] = before[field] ?? null;
+		row[`to${capped}`] = after[field] ?? null;
+	}
+	return row;
+}
+async function appendTransitions(ds, ctx, rows) {
+	if (rows.length === 0) return 0;
+	const byMonth = /* @__PURE__ */ new Map();
+	for (const row of rows) {
+		const month = transitionMonth(row);
+		const bucket = byMonth.get(month);
+		if (bucket) bucket.push(row);
+		else byMonth.set(month, [row]);
+	}
+	let written = 0;
+	for (const [month, monthRows] of byMonth) {
+		const key = inspectionTransitionsMonthKey(ctx, month);
+		const existingBytes = await readOptional(ds, key);
+		const existing = existingBytes ? await decodeParquetToRows(existingBytes) : [];
+		const seen = new Set(existing.map((r) => `${String(r.urlHash)}\u0000${String(r.changedBefore)}`));
+		const fresh = monthRows.filter((r) => {
+			const id = `${String(r.urlHash)}\u0000${String(r.changedBefore)}`;
+			if (seen.has(id)) return false;
+			seen.add(id);
+			return true;
+		});
+		if (fresh.length === 0) continue;
+		const merged = [...existing, ...fresh];
+		await ds.write(key, encodeRowsToParquetFlex(merged, {
+			columns: INSPECTION_TRANSITION_COLUMNS,
+			sortKey: ["urlHash"]
+		}));
+		written += fresh.length;
+	}
+	return written;
+}
+function transitionMonth(row) {
+	return String(row.changedBefore ?? "").slice(0, 7) || "unknown";
+}
+function createInspectionStore(opts) {
+	const ds = opts.dataSource;
+	function shardFor(record) {
+		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
+		return m ? `${m[1]}-${m[2]}` : "unknown";
+	}
+	function randomBatchId() {
+		return typeof crypto !== "undefined" && "randomUUID" in crypto ? crypto.randomUUID() : `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+	}
+	return {
+		async appendHistory(ctx, records, options) {
+			if (records.length === 0) return;
+			const batchId = options?.batchId ?? randomBatchId();
+			const byMonth = /* @__PURE__ */ new Map();
+			for (const r of records) {
+				const month = shardFor(r);
+				if (!byMonth.has(month)) byMonth.set(month, []);
+				byMonth.get(month).push(r);
+			}
+			await mapEntityIo([...byMonth].map(([yearMonth, batch]) => {
+				const bytes = encodeJsonBigintSafe({
+					version: 1,
+					records: batch
+				});
+				if (bytes.byteLength > 5242880) throw new Error(`inspection history shard exceeds ${INSPECTION_HISTORY_MAX_BYTES} bytes (got ${bytes.byteLength}); split the batch`);
+				return {
+					key: inspectionHistoryShardKey(ctx, yearMonth, batchId),
+					bytes
+				};
+			}), (shard) => ds.write(shard.key, shard.bytes));
+		},
+		async loadHistory(ctx, yearMonth) {
+			const keys = await ds.list(inspectionHistoryPrefix(ctx, yearMonth));
+			if (keys.length === 0) return void 0;
+			return {
+				version: 1,
+				records: (await mapEntityIo(keys, async (key) => {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) return [];
+					return (await Promise.resolve().then(() => JSON.parse(new TextDecoder().decode(bytes))).catch((err) => {
+						console.warn("[inspection.loadHistory] failed to decode shard", {
+							key,
+							error: err.message
+						});
+					}))?.records ?? [];
+				})).flat()
+			};
+		},
+		async materialize(ctx, rowIter) {
+			const rows = Array.from(rowIter);
+			rows.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+			const bytes = encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_PARQUET_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			const key = inspectionParquetKey(ctx);
+			await ds.write(key, bytes);
+			return {
+				key,
+				rowCount: rows.length,
+				bytes: bytes.byteLength
+			};
+		},
+		async appendInspectionEvents(ctx, rows, options) {
+			if (rows.length === 0) return {
+				keys: [],
+				rowCount: 0
+			};
+			const batchId = options?.batchId ?? randomBatchId();
+			const byMonth = /* @__PURE__ */ new Map();
+			for (const r of rows) {
+				const m = YEAR_MONTH_RE.exec(r.inspectedAt);
+				const month = m ? `${m[1]}-${m[2]}` : "unknown";
+				const bucket = byMonth.get(month) ?? [];
+				bucket.push(r);
+				byMonth.set(month, bucket);
+			}
+			const files = [...byMonth].map(([month, batch]) => {
+				const bytes = encodeRowsToParquetFlex(batch, {
+					columns: INSPECTION_EVENT_COLUMNS,
+					sortKey: ["urlHash"]
+				});
+				return {
+					key: inspectionEventKey(ctx, month, batchId),
+					bytes
+				};
+			});
+			await mapEntityIo(files, (file) => ds.write(file.key, file.bytes));
+			return {
+				keys: files.map((file) => file.key),
+				rowCount: rows.length
+			};
+		},
+		async compactInspections(ctx, opts) {
+			const eventKeys = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((k) => INSPECTION_EVENT_KEY_RE.test(k));
+			if (eventKeys.length === 0) return {
+				baseRowCount: 0,
+				eventsFolded: 0,
+				eventFilesDeleted: 0,
+				transitionsWritten: 0
+			};
+			const baseKey = inspectionBaseKey(ctx);
+			const baseBytes = await readOptional(ds, baseKey);
+			const baseRows = baseBytes ? await decodeParquetToRows(baseBytes) : [];
+			const latest = /* @__PURE__ */ new Map();
+			const earliestChecked = /* @__PURE__ */ new Map();
+			const transitions = [];
+			const consider = (row) => {
+				const h = String(row.urlHash);
+				const prev = latest.get(h);
+				if (!prev || String(row.inspectedAt ?? "") > String(prev.inspectedAt ?? "")) {
+					if (opts?.transitions && prev && isStateTransition(prev, row)) transitions.push(buildTransitionRow(prev, row));
+					latest.set(h, row);
+				}
+				const fc = row.firstCheckedAt;
+				if (fc != null) {
+					const fcStr = String(fc);
+					const cur = earliestChecked.get(h);
+					if (cur === void 0 || fcStr < cur) earliestChecked.set(h, fcStr);
+				}
+			};
+			for (const row of baseRows) consider(row);
+			let eventsFolded = 0;
+			const consumed = [];
+			const eventRows = [];
+			const eventFiles = await mapEntityIo(eventKeys.sort(), async (key) => {
+				const bytes = await readOptional(ds, key);
+				if (!bytes) return void 0;
+				return {
+					key,
+					rows: await decodeParquetToRows(bytes)
+				};
+			});
+			for (const file of eventFiles) {
+				if (!file) continue;
+				const { key, rows } = file;
+				consumed.push(key);
+				eventRows.push(...rows);
+				eventsFolded += rows.length;
+			}
+			eventRows.sort((a, b) => String(a.urlHash).localeCompare(String(b.urlHash)) || String(a.inspectedAt ?? "").localeCompare(String(b.inspectedAt ?? "")));
+			for (const row of eventRows) consider(row);
+			const merged = [];
+			for (const [h, row] of latest) {
+				const fc = earliestChecked.get(h);
+				if (fc !== void 0) row.firstCheckedAt = fc;
+				populateCanonicalMismatchKind(row);
+				merged.push(row);
+			}
+			const bytes = encodeRowsToParquetFlex(merged, {
+				columns: INSPECTION_EVENT_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			const transitionsWritten = opts?.transitions ? await appendTransitions(ds, ctx, transitions) : 0;
+			await ds.write(baseKey, bytes);
+			if (consumed.length > 0) await ds.delete(consumed);
+			return {
+				baseRowCount: merged.length,
+				eventsFolded,
+				eventFilesDeleted: consumed.length,
+				transitionsWritten
+			};
+		},
+		async backfillCanonicalMismatchKinds(ctx) {
+			const baseKey = inspectionBaseKey(ctx);
+			const baseBytes = await readOptional(ds, baseKey);
+			if (!baseBytes) return {
+				baseRowCount: 0,
+				rowsBackfilled: 0,
+				rewritten: false
+			};
+			const rows = await decodeParquetToRows(baseBytes);
+			let rowsBackfilled = 0;
+			for (const row of rows) if (populateCanonicalMismatchKind(row)) rowsBackfilled++;
+			if (rowsBackfilled === 0) return {
+				baseRowCount: rows.length,
+				rowsBackfilled: 0,
+				rewritten: false
+			};
+			await ds.write(baseKey, encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_EVENT_COLUMNS,
+				sortKey: ["urlHash"]
+			}));
+			return {
+				baseRowCount: rows.length,
+				rowsBackfilled,
+				rewritten: true
+			};
+		},
+		parquetUri(ctx) {
+			return ds.uri?.(inspectionParquetKey(ctx));
+		}
+	};
+}
+export { INSPECTION_HISTORY_MAX_BYTES, createInspectionStore };

package/dist/entities/io.mjs

@@ -0,0 +1,16 @@
+const ENTITY_IO_CONCURRENCY = 8;
+async function mapEntityIo(items, fn) {
+	if (items.length === 0) return [];
+	const results = Array.from({ length: items.length }, () => void 0);
+	let next = 0;
+	async function worker() {
+		while (true) {
+			const index = next++;
+			if (index >= items.length) return;
+			results[index] = await fn(items[index], index);
+		}
+	}
+	await Promise.all(Array.from({ length: Math.min(ENTITY_IO_CONCURRENCY, items.length) }, worker));
+	return results;
+}
+export { mapEntityIo };

package/dist/{query-dim.d.mts → entities/query-dim.d.mts}

@@ -1,5 +1,5 @@
-import { DataSource } from "./storage.mjs";
-import { queryDimMetaKey, queryDimParquetKey } from "./entity-keys.mjs";
+import { DataSource } from "../storage.mjs";
+import { queryDimMetaKey, queryDimParquetKey } from "../entity-keys.mjs";
 import { TenantCtx } from "@gscdump/contracts";
 interface QueryDimRecord {
   query: string;
@@ -51,4 +51,4 @@ interface QueryDimStore {
 declare function createQueryDimStore({ dataSource }: {
   dataSource: DataSource;
 }): QueryDimStore;
-export { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore };
+export { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore, queryDimMetaKey, queryDimParquetKey };

package/dist/{query-dim.mjs → entities/query-dim.mjs}

@@ -1,6 +1,6 @@
-import { queryDimMetaKey, queryDimParquetKey } from "./entity-keys.mjs";
-import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { readOptional } from "./adapters/read-optional.mjs";
+import { queryDimMetaKey, queryDimParquetKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 const QUERY_DIM_COLUMNS = [
 	{
@@ -88,4 +88,4 @@ function createQueryDimStore({ dataSource }) {
 		}
 	};
 }
-export { buildQueryDimRecords, createQueryDimStore };
+export { buildQueryDimRecords, createQueryDimStore, queryDimMetaKey, queryDimParquetKey };

package/dist/{sitemap-projection.mjs → entities/sitemap-projection.mjs}

@@ -1,4 +1,4 @@
-import { parseSitemapUrlsDeltaKey } from "./entity-keys.mjs";
+import { parseSitemapUrlsDeltaKey } from "../entity-keys.mjs";
 const SITEMAP_PROJECTION_GRACE_MS = 900 * 1e3;
 function emptySitemapProjectionManifest() {
 	return {

package/dist/entities/sitemap-shared.d.mts

@@ -0,0 +1,199 @@
+import { DataSource } from "../storage.mjs";
+import { ScheduleState } from "../schedule.mjs";
+import { Row, TenantCtx } from "@gscdump/contracts";
+/** GSC sitemap record we persist. Matches `Schema$WmxSitemap` but as plain JSON. */
+interface SitemapRecord {
+  /** The sitemap URL (feedpath) as returned by GSC. */
+  path: string;
+  /** ISO-8601 timestamp of the snapshot run that captured this record. */
+  capturedAt: string;
+  /** Last time Google downloaded this sitemap (RFC 3339, from the API). */
+  lastDownloaded?: string;
+  /** Last time the sitemap was submitted. */
+  lastSubmitted?: string;
+  type?: string;
+  isPending?: boolean;
+  isSitemapsIndex?: boolean;
+  errors?: string;
+  warnings?: string;
+  /** Per-content-type counts (web, image, video, news). */
+  contents?: Array<{
+    type?: string;
+    submitted?: string;
+    indexed?: string;
+  }>;
+  /** Raw payload for fields we don't promote to first-class columns. */
+  raw?: unknown;
+  /** Number of URLs observed in this feedpath at last snapshot. */
+  urlCount?: number;
+  /** Stable hash of the sorted normalized loc list at last snapshot. */
+  contentHash?: string;
+  /** Adaptive cadence state owned by `sitemapPolicy`. */
+  schedule?: ScheduleState;
+}
+interface SitemapIndex {
+  version: 1;
+  /** Map of feedpathHash → latest SitemapRecord. */
+  records: Record<string, SitemapRecord>;
+}
+interface SitemapHistoryDoc {
+  version: 1;
+  path: string;
+  capturedAt: string;
+  record: SitemapRecord;
+}
+/** Parsed URL entry from a sitemap XML. */
+interface ParsedUrl {
+  loc: string;
+  /** ISO-8601 lastmod from the sitemap, if present. */
+  lastmod?: string;
+}
+/** A single URL row in the urls/index.parquet partition. */
+interface SitemapUrlRecord {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  loc: string;
+  lastmod?: string;
+  firstSeenAt: number;
+  lastSeenAt: number;
+  /** Set when the URL has been removed. Null/undefined = currently live. */
+  removedAt?: number;
+}
+interface SnapshotUrlsResult {
+  added: number;
+  removed: number;
+  kept: number;
+  contentHash: string;
+  /** True when current membership did not change. Initial history seeding may write. */
+  unchanged: boolean;
+}
+interface CompleteSitemapGeneration {
+  _tag: 'complete';
+  id: string;
+  /** Unix epoch milliseconds. */
+  observedAt: number;
+}
+interface ReconcileResult {
+  /** Feedpaths that were absent from the live set and had their live URLs pruned. */
+  feedpathsPruned: number;
+  /** Total URL rows transitioned live → removed across pruned feedpaths. */
+  urlsRemoved: number;
+}
+/**
+ * Bounds on a single `compactUrls` call. `compactUrls` is memory-bounded (one
+ * feedpath at a time) but was previously unbounded in TIME: a tenant with a
+ * large accumulated delta backlog could run past a caller's execution budget.
+ * Both bounds are checked BETWEEN feedpaths and only after at least one has
+ * been compacted, so a call always makes forward progress.
+ */
+interface CompactUrlsOptions {
+  /** Stop before starting another feedpath once this many ms have elapsed. */
+  deadlineMs?: number;
+  /** Stop after compacting this many feedpaths. */
+  maxFeedpaths?: number;
+}
+interface CompactUrlsResult {
+  /** Feedpaths whose active deltas were folded and retired by this call. */
+  compactedFeedpaths: number;
+  /**
+   * Feedpaths that still hold outstanding deltas. Call `compactUrls` again to
+   * continue. No cursor is needed because the projection watermark excludes a
+   * compacted feedpath's grace-retained deltas from the next call.
+   */
+  remainingFeedpaths: number;
+}
+interface DeltaEntry {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  op: 'added' | 'removed';
+  loc: string;
+  lastmod?: string;
+  at: number;
+}
+interface SitemapMembershipEvent {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  op: 'added' | 'removed';
+  loc: string;
+  lastmod?: string;
+  generationId: string;
+  /** Unix epoch milliseconds. */
+  observedAt: number;
+  /** Stable ordering within one generation. */
+  sequence: number;
+  /** Whether this row also mutates the disposable current-state projection. */
+  projectsState: boolean;
+}
+interface DateRange {
+  /** YYYY-MM-DD inclusive. */
+  from?: string;
+  /** YYYY-MM-DD inclusive. */
+  to?: string;
+}
+interface LoadUrlsOptions {
+  includeRemoved?: boolean;
+}
+interface SitemapReadStore {
+  /** Load the full site index (latest record per feedpath). */
+  loadIndex: (ctx: TenantCtx) => Promise<SitemapIndex>;
+  /** Fetch the latest snapshot for a feedpath, or undefined. */
+  getLatest: (ctx: TenantCtx, path: string) => Promise<SitemapRecord | undefined>;
+  /** Stream live (and optionally removed) URL rows for a feedpath. */
+  loadUrls: (ctx: TenantCtx, feedpath: string, opts?: LoadUrlsOptions) => AsyncIterable<SitemapUrlRecord>;
+  /** Stream all delta entries within `[from, to]` (YYYY-MM-DD inclusive). */
+  loadDeltas: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<DeltaEntry>;
+  /** Stream immutable membership events within `[from, to]`. */
+  loadEvents: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<SitemapMembershipEvent>;
+}
+interface SitemapStore extends SitemapReadStore {
+  /**
+   * Persist a snapshot run. Updates the index + writes one immutable
+   * history doc per record under `history/<feedpathHash>__<capturedAtMs>.json`.
+   */
+  writeSnapshot: (ctx: TenantCtx, records: readonly SitemapRecord[]) => Promise<void>;
+  /**
+   * Diff a complete sitemap generation against current state. The immutable
+   * membership event lands before its disposable state delta.
+   */
+  snapshotUrls: (ctx: TenantCtx, generation: CompleteSitemapGeneration, feedpath: string, urls: readonly ParsedUrl[]) => Promise<SnapshotUrlsResult>;
+  /**
+   * Fold accumulated deltas into the prior index, one feedpath at a time:
+   * rewrites each touched feedpath's `by-feed/<hash>/index.parquet` and deletes
+   * the consumed delta files. Bounded per feedpath, so it stays within memory
+   * regardless of total site URL count.
+   *
+   * Optionally bounded in TIME too (`opts.deadlineMs` / `opts.maxFeedpaths`).
+   * A bounded call is safe to stop mid-way: each rewritten feedpath advances
+   * the projection watermark, so the remainder is simply what the next call
+   * finds. Callers drive this from `remainingFeedpaths`, not a cursor.
+   */
+  compactUrls: (ctx: TenantCtx, opts?: CompactUrlsOptions) => Promise<CompactUrlsResult>;
+  /**
+   * Site-wide convergence: mark every still-live URL whose owning feedpath is
+   * absent from `liveFeedpaths` as removed. `compactUrls`/`snapshotUrls` only
+   * prune URLs *inside* a feedpath that was re-observed; a whole feed dropped
+   * from the sitemap list (no `snapshotUrls` call) leaves its URLs frozen-live
+   * forever. This is the sidecar mirror of the D1 generation sweep: it rewrites
+   * each dropped feedpath's `by-feed/<hash>/index.parquet` with `removedAt` set,
+   * advances its projection watermark, and grace-retires outstanding deltas.
+   * Bounded per feedpath, so memory stays flat regardless of site size. Live
+   * feedpaths are never touched.
+   */
+  reconcile: (ctx: TenantCtx, generation: CompleteSitemapGeneration, opts: {
+    liveFeedpaths: readonly string[];
+  }) => Promise<ReconcileResult>;
+}
+interface CreateSitemapReadStoreOptions {
+  dataSource: DataSource;
+  /** Override the feedpath hash (test seam). */
+  hash?: (path: string) => string;
+}
+type SitemapMutation = <T>(ctx: TenantCtx, fn: () => Promise<T>) => Promise<T>;
+interface CreateSitemapStoreOptions extends CreateSitemapReadStoreOptions {
+  withMutation: SitemapMutation;
+  now?: () => number;
+}
+export { CompactUrlsOptions, CompactUrlsResult, CompleteSitemapGeneration, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, LoadUrlsOptions, ParsedUrl, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapMembershipEvent, SitemapMutation, SitemapReadStore, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult };