@gscdump/engine 0.9.2 → 0.11.0

package/dist/entities.d.mts

@@ -1,4 +1,5 @@
 import { a as DataSource } from "./_chunks/storage.mjs";
+import { ScheduleState } from "./schedule.mjs";
 import { TenantCtx } from "gscdump/contracts";
 /**
  * GSC URL inspection result fields we persist. Mirrors the
@@ -28,8 +29,17 @@ interface InspectionRecord {
    * Free-form payload for fields we don't promote to first-class columns
    *  (e.g. `referringUrls`, `crawledAs`). Keeps the wire format forward-compat
    *  without bumping the schema for every API addition.
+   *
+   *  Recognised keys:
+   *  - `schedule`: optional `ScheduleState` from {@link inspectionPolicy}
+   *    governing when this URL is next due for re-inspection. Undefined on
+   *    pre-§0 records — readers must tolerate the missing field and fall
+   *    back to default policy on first observe.
    */
-  raw?: unknown;
+  raw?: {
+    schedule?: ScheduleState;
+    [key: string]: unknown;
+  };
 }
 /** Wire shape persisted to disk/R2. */
 interface InspectionIndex {
@@ -44,6 +54,7 @@ interface InspectionHistoryShard {
 }
 declare function inspectionIndexKey(ctx: TenantCtx): string;
 declare function emptyTypesKey(ctx: TenantCtx): string;
+declare function inspectionParquetKey(ctx: TenantCtx): string;
 declare function inspectionHistoryKey(ctx: TenantCtx, yearMonth: string): string;
 /**
  * Stable URL hash used as the index key. Short, URL-safe, deterministic.
@@ -66,6 +77,34 @@ interface InspectionStore {
   loadIndex: (ctx: TenantCtx) => Promise<InspectionIndex>;
   /** Read the per-month history shard if it exists. */
   loadHistory: (ctx: TenantCtx, yearMonth: string) => Promise<InspectionHistoryShard | undefined>;
+  /**
+   * Snapshot the current JSON index to a parquet sidecar at
+   * `entities/inspections/index.parquet`. One PUT. Sorted by `urlHash` so
+   * DuckDB row-group stats can prune URL-keyed JOINs efficiently.
+   *
+   * Internal seam: callers don't choose JSON-vs-parquet — the store materialises
+   * the parquet at end-of-batch (e.g. after `indexing/complete`) and readers
+   * pick the format that matches their access pattern (parquet for JOINs,
+   * JSON for full-index scans / point lookups).
+   *
+   * Returns the parquet object key (matches {@link parquetUri} after write).
+   */
+  materialize: (ctx: TenantCtx) => Promise<{
+    key: string;
+    rowCount: number;
+    bytes: number;
+  }>;
+  /**
+   * DuckDB-resolvable URI for the materialised parquet sidecar, or
+   * `undefined` if the underlying `DataSource` has no native URI shape
+   * (in-memory tests). When defined, read paths can `read_parquet(<uri>)`
+   * directly without staging bytes through JS.
+   *
+   * Does not check existence — caller is responsible for ensuring
+   * `materialize` has run at least once. Returning a URI for a missing key
+   * is safe; DuckDB will surface a 404 / not-found at query time.
+   */
+  parquetUri: (ctx: TenantCtx) => string | undefined;
 }
 interface CreateInspectionStoreOptions {
   dataSource: DataSource;
@@ -100,6 +139,12 @@ interface SitemapRecord {
   }>;
   /** Raw payload for fields we don't promote to first-class columns. */
   raw?: unknown;
+  /** Number of URLs observed in this feedpath at last snapshot. */
+  urlCount?: number;
+  /** Stable hash of the sorted normalized loc list at last snapshot. */
+  contentHash?: string;
+  /** Adaptive cadence state owned by `sitemapPolicy`. */
+  schedule?: ScheduleState;
 }
 interface SitemapIndex {
   version: 1;
@@ -114,6 +159,57 @@ interface SitemapHistoryDoc {
 }
 declare function sitemapIndexKey(ctx: TenantCtx): string;
 declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
+declare function sitemapUrlsIndexKey(ctx: TenantCtx): string;
+declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, date: string): string;
+/** Parsed URL entry from a sitemap XML. */
+interface ParsedUrl {
+  loc: string;
+  /** ISO-8601 lastmod from the sitemap, if present. */
+  lastmod?: string;
+}
+/** A single URL row in the urls/index.parquet partition. */
+interface SitemapUrlRecord {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  loc: string;
+  lastmod?: string;
+  firstSeenAt: number;
+  lastSeenAt: number;
+  /** Set when the URL has been removed. Null/undefined = currently live. */
+  removedAt?: number;
+}
+interface SnapshotUrlsResult {
+  added: number;
+  removed: number;
+  kept: number;
+  contentHash: string;
+  /** True when contentHash matched prior; the call performed zero writes. */
+  unchanged: boolean;
+}
+interface DeltaEntry {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  op: 'added' | 'removed';
+  loc: string;
+  lastmod?: string;
+  at: number;
+}
+interface DateRange {
+  /** YYYY-MM-DD inclusive. */
+  from?: string;
+  /** YYYY-MM-DD inclusive. */
+  to?: string;
+}
+interface LoadUrlsOptions {
+  includeRemoved?: boolean;
+}
+/**
+ * Hash a URL list for change detection. Sorts then folds via FNV-1a so it's
+ * deterministic, locale-free, and cheap on Workers.
+ */
+declare function hashUrlList(urls: readonly ParsedUrl[]): string;
 interface SitemapStore {
   /**
    * Persist a snapshot run. Updates the index + writes one immutable
@@ -124,6 +220,24 @@ interface SitemapStore {
   loadIndex: (ctx: TenantCtx) => Promise<SitemapIndex>;
   /** Fetch the latest snapshot for a feedpath, or undefined. */
   getLatest: (ctx: TenantCtx, path: string) => Promise<SitemapRecord | undefined>;
+  /**
+   * Diff incoming URLs against the prior `urls/index.parquet` partition for
+   * `feedpath`; on change, writes a single delta parquet under
+   * `urls/deltas/YYYY-MM-DD__{feedpathHash}.parquet`. Skipped (0 PUTs) when
+   * `contentHash` matches prior.
+   */
+  snapshotUrls: (ctx: TenantCtx, feedpath: string, urls: readonly ParsedUrl[]) => Promise<SnapshotUrlsResult>;
+  /** Stream live (and optionally removed) URL rows for a feedpath. */
+  loadUrls: (ctx: TenantCtx, feedpath: string, opts?: LoadUrlsOptions) => AsyncIterable<SitemapUrlRecord>;
+  /** Stream all delta entries within `[from, to]` (YYYY-MM-DD inclusive). */
+  loadDeltas: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<DeltaEntry>;
+  /**
+   * Fold every accumulated delta into the prior index; writes a fresh
+   * `urls/index.parquet` and deletes the consumed delta files.
+   */
+  compactUrls: (ctx: TenantCtx) => Promise<void>;
+  /** DuckDB-resolvable URI for the URLs index; `undefined` if backend lacks one. */
+  urlsParquetUri: (ctx: TenantCtx) => string | undefined;
 }
 interface CreateSitemapStoreOptions {
   dataSource: DataSource;
@@ -175,4 +289,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, EmptyTypesDoc, EmptyTypesStore, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionIndex, InspectionRecord, InspectionStore, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, sitemapHistoryKey, sitemapIndexKey };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionIndex, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };

package/dist/entities.mjs

@@ -1,3 +1,4 @@
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 function inspectionIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
@@ -5,6 +6,9 @@ function inspectionIndexKey(ctx) {
 function emptyTypesKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/empty-types.json` : `u_${ctx.userId}/entities/empty-types.json`;
 }
+function inspectionParquetKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.parquet` : `u_${ctx.userId}/entities/inspections/index.parquet`;
+}
 function inspectionHistoryKey(ctx, yearMonth) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/history/${yearMonth}.json` : `u_${ctx.userId}/entities/inspections/history/${yearMonth}.json`;
 }
@@ -75,15 +79,251 @@ function createInspectionStore(opts) {
 		},
 		async loadHistory(ctx, yearMonth) {
 			return await readJson(inspectionHistoryKey(ctx, yearMonth));
+		},
+		async materialize(ctx) {
+			const index = await readJson(inspectionIndexKey(ctx)) ?? emptyIndex();
+			const rows = Object.entries(index.records).map(([urlHash, r]) => ({
+				urlHash,
+				url: r.url,
+				inspectedAt: r.inspectedAt,
+				indexStatus: r.indexStatus ?? null,
+				lastCrawlTime: r.lastCrawlTime ?? null,
+				googleCanonical: r.googleCanonical ?? null,
+				userCanonical: r.userCanonical ?? null,
+				coverageState: r.coverageState ?? null,
+				robotsTxtState: r.robotsTxtState ?? null,
+				indexingState: r.indexingState ?? null,
+				pageFetchState: r.pageFetchState ?? null,
+				mobileUsabilityVerdict: r.mobileUsabilityVerdict ?? null,
+				richResultsVerdict: r.richResultsVerdict ?? null,
+				scheduleNextAt: r.raw?.schedule?.nextAt ?? null,
+				scheduleConsecutiveUnchanged: r.raw?.schedule?.consecutiveUnchanged ?? null,
+				schedulePolicyVersion: r.raw?.schedule?.policyVersion ?? null
+			}));
+			const bytes = encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_PARQUET_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			const key = inspectionParquetKey(ctx);
+			await ds.write(key, bytes);
+			return {
+				key,
+				rowCount: rows.length,
+				bytes: bytes.byteLength
+			};
+		},
+		parquetUri(ctx) {
+			return ds.uri?.(inspectionParquetKey(ctx));
 		}
 	};
 }
+const INSPECTION_PARQUET_COLUMNS = [
+	{
+		name: "urlHash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "inspectedAt",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "indexStatus",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "lastCrawlTime",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "googleCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "userCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "coverageState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "robotsTxtState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "indexingState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "pageFetchState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileUsabilityVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "scheduleNextAt",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "scheduleConsecutiveUnchanged",
+		type: "INTEGER",
+		nullable: true
+	},
+	{
+		name: "schedulePolicyVersion",
+		type: "INTEGER",
+		nullable: true
+	}
+];
 function sitemapIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/index.json` : `u_${ctx.userId}/entities/sitemaps/index.json`;
 }
 function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json` : `u_${ctx.userId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
 }
+function sitemapUrlsPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/urls` : `u_${ctx.userId}/entities/sitemaps/urls`;
+}
+function sitemapUrlsIndexKey(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/index.parquet`;
+}
+function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
+	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
+}
+const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
+const URLS_INDEX_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "first_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "last_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "removed_at",
+		type: "BIGINT",
+		nullable: true
+	}
+];
+const URLS_DELTA_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "op",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "at",
+		type: "BIGINT",
+		nullable: false
+	}
+];
+function rowToUrlRecord(row) {
+	return {
+		feedpath: String(row.feedpath),
+		feedpathHash: String(row.feedpath_hash),
+		urlHash: String(row.url_hash),
+		loc: String(row.loc),
+		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+		firstSeenAt: Number(row.first_seen_at),
+		lastSeenAt: Number(row.last_seen_at),
+		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
+	};
+}
+function urlRecordToRow(r) {
+	return {
+		feedpath: r.feedpath,
+		feedpath_hash: r.feedpathHash,
+		url_hash: r.urlHash,
+		loc: r.loc,
+		lastmod: r.lastmod ?? null,
+		first_seen_at: r.firstSeenAt,
+		last_seen_at: r.lastSeenAt,
+		removed_at: r.removedAt ?? null
+	};
+}
+function isoDate(ms) {
+	return new Date(ms).toISOString().slice(0, 10);
+}
+function hashUrlList(urls) {
+	return hashUrl(urls.map((u) => u.loc).sort().join("\n"));
+}
 function createSitemapStore(opts) {
 	const ds = opts.dataSource;
 	const hash = opts.hash ?? hashUrl;
@@ -123,6 +363,218 @@ function createSitemapStore(opts) {
 		},
 		async getLatest(ctx, path) {
 			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
+		},
+		async snapshotUrls(ctx, feedpath, urls) {
+			const fpHash = hash(feedpath);
+			const contentHash = hashUrlList(urls);
+			const at = now();
+			const priorByHash = /* @__PURE__ */ new Map();
+			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
+			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
+			if (livePrior.length > 0) {
+				if (hashUrl(livePrior.map((r) => String(r.loc)).sort().join("\n")) === contentHash) return {
+					added: 0,
+					removed: 0,
+					kept: livePrior.length,
+					contentHash,
+					unchanged: true
+				};
+			}
+			const incomingByHash = /* @__PURE__ */ new Map();
+			for (const u of urls) incomingByHash.set(hash(u.loc), u);
+			const deltaRows = [];
+			let added = 0;
+			let removed = 0;
+			let kept = 0;
+			const date = isoDate(at);
+			for (const [urlHash, u] of incomingByHash) {
+				const prev = priorByHash.get(urlHash);
+				if (!prev || prev.removedAt != null) {
+					added++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "added",
+						loc: u.loc,
+						lastmod: u.lastmod ?? null,
+						at
+					});
+				} else kept++;
+			}
+			for (const [urlHash, prev] of priorByHash) {
+				if (prev.removedAt != null) continue;
+				if (!incomingByHash.has(urlHash)) {
+					removed++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "removed",
+						loc: prev.loc,
+						lastmod: prev.lastmod ?? null,
+						at
+					});
+				}
+			}
+			if (deltaRows.length > 0) {
+				const bytes = encodeRowsToParquetFlex(deltaRows, {
+					columns: URLS_DELTA_COLUMNS,
+					sortKey: ["url_hash"]
+				});
+				await ds.write(sitemapUrlsDeltaKey(ctx, fpHash, date), bytes);
+			}
+			return {
+				added,
+				removed,
+				kept,
+				contentHash,
+				unchanged: false
+			};
+		},
+		async *loadUrls(ctx, feedpath, opts) {
+			const fpHash = hash(feedpath);
+			const includeRemoved = opts?.includeRemoved ?? false;
+			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx)).catch(() => void 0);
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			const live = /* @__PURE__ */ new Map();
+			const removedMap = /* @__PURE__ */ new Map();
+			for (const row of indexRows) {
+				if (row.feedpath_hash !== fpHash) continue;
+				const rec = rowToUrlRecord(row);
+				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
+				else live.set(rec.urlHash, rec);
+			}
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m || m[2] !== fpHash) continue;
+				const dBytes = await ds.read(key).catch(() => void 0);
+				if (!dBytes) continue;
+				const dRows = await decodeParquetToRows(dBytes);
+				for (const r of dRows) {
+					const op = String(r.op);
+					const urlHash = String(r.url_hash);
+					const at = Number(r.at);
+					if (op === "added") {
+						const prev = live.get(urlHash) ?? removedMap.get(urlHash);
+						removedMap.delete(urlHash);
+						live.set(urlHash, {
+							feedpath,
+							feedpathHash: fpHash,
+							urlHash,
+							loc: String(r.loc),
+							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+							firstSeenAt: prev?.firstSeenAt ?? at,
+							lastSeenAt: at
+						});
+					} else if (op === "removed") {
+						const prev = live.get(urlHash);
+						live.delete(urlHash);
+						if (prev) removedMap.set(urlHash, {
+							...prev,
+							removedAt: at
+						});
+					}
+				}
+			}
+			for (const rec of live.values()) yield rec;
+			if (includeRemoved) for (const rec of removedMap.values()) yield rec;
+		},
+		async *loadDeltas(ctx, dateRange) {
+			const from = dateRange?.from;
+			const to = dateRange?.to;
+			const keys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			for (const key of keys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const date = m[1];
+				if (from && date < from) continue;
+				if (to && date > to) continue;
+				const bytes = await ds.read(key).catch(() => void 0);
+				if (!bytes) continue;
+				const rows = await decodeParquetToRows(bytes);
+				for (const r of rows) {
+					const op = String(r.op);
+					if (op !== "added" && op !== "removed") continue;
+					yield {
+						feedpath: String(r.feedpath),
+						feedpathHash: String(r.feedpath_hash),
+						urlHash: String(r.url_hash),
+						op,
+						loc: String(r.loc),
+						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+						at: Number(r.at)
+					};
+				}
+			}
+		},
+		async compactUrls(ctx) {
+			const indexKey = sitemapUrlsIndexKey(ctx);
+			const indexBytes = await ds.read(indexKey).catch(() => void 0);
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+			const stateKey = (fp, u) => `${fp}::${u}`;
+			const live = /* @__PURE__ */ new Map();
+			const removed = /* @__PURE__ */ new Map();
+			for (const row of indexRows) {
+				const rec = rowToUrlRecord(row);
+				const k = stateKey(rec.feedpathHash, rec.urlHash);
+				if (rec.removedAt != null) removed.set(k, rec);
+				else live.set(k, rec);
+			}
+			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			const consumed = [];
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const fpHash = m[2];
+				const bytes = await ds.read(key).catch(() => void 0);
+				if (!bytes) continue;
+				consumed.push(key);
+				const rows = await decodeParquetToRows(bytes);
+				for (const r of rows) {
+					const urlHash = String(r.url_hash);
+					const at = Number(r.at);
+					const k = stateKey(fpHash, urlHash);
+					const op = String(r.op);
+					if (op === "added") {
+						const prev = live.get(k) ?? removed.get(k);
+						removed.delete(k);
+						live.set(k, {
+							feedpath: String(r.feedpath),
+							feedpathHash: fpHash,
+							urlHash,
+							loc: String(r.loc),
+							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+							firstSeenAt: prev?.firstSeenAt ?? at,
+							lastSeenAt: at
+						});
+					} else if (op === "removed") {
+						const prev = live.get(k);
+						live.delete(k);
+						if (prev) removed.set(k, {
+							...prev,
+							removedAt: at
+						});
+					}
+				}
+			}
+			const merged = [...live.values(), ...removed.values()];
+			merged.sort((a, b) => {
+				if (a.feedpathHash !== b.feedpathHash) return a.feedpathHash < b.feedpathHash ? -1 : 1;
+				if (a.urlHash !== b.urlHash) return a.urlHash < b.urlHash ? -1 : 1;
+				return 0;
+			});
+			const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+				columns: URLS_INDEX_COLUMNS,
+				sortKey: ["feedpath_hash", "url_hash"]
+			});
+			await ds.write(indexKey, bytes);
+			if (consumed.length > 0) await ds.delete(consumed);
+		},
+		urlsParquetUri(ctx) {
+			const key = sitemapUrlsIndexKey(ctx);
+			return ds.uri ? ds.uri(key) : void 0;
 		}
 	};
 }
@@ -206,4 +658,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, sitemapHistoryKey, sitemapIndexKey };
+export { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };

package/dist/index.d.mts

@@ -1,6 +1,7 @@
-import { A as SyncStateDetail, B as WriteResult, C as QueryExecutor, D as SearchType, E as RunSQLOptions, F as TenantCtx, G as monthPartition, H as inferLegacyTier, I as Watermark, J as quarterPartition, K as objectKey, L as WatermarkFilter, M as SyncStateKind, N as SyncStateScope, O as StorageEngine, P as TableName, R as WatermarkScope, S as QueryExecuteResult, T as Row, U as inferSearchType, V as dayPartition, W as mondayOfWeek, X as CompactionThresholds, Y as weekPartition, Z as enumeratePartitions, _ as PurgeFilter, a as DataSource, b as QueryCtx, c as FileSetRef, d as LockScope, f as ManifestEntry, g as ParquetCodec, h as OptimizedQueryResult, i as DEFAULT_SEARCH_TYPE, j as SyncStateFilter, k as SyncState, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, q as quarterOfMonth, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeResult, w as QueryResult, x as QueryExecuteOptions, y as PurgeUrlsResult, z as WriteCtx } from "./_chunks/storage.mjs";
+import { A as SyncStateDetail, B as WriteResult, C as QueryExecutor, D as SearchType, E as RunSQLOptions, F as TenantCtx, G as CompactionThresholds, H as inferLegacyTier, I as Watermark, K as enumeratePartitions, L as WatermarkFilter, M as SyncStateKind, N as SyncStateScope, O as StorageEngine, P as TableName, R as WatermarkScope, S as QueryExecuteResult, T as Row, U as inferSearchType, V as dayPartition, W as objectKey, _ as PurgeFilter, a as DataSource, b as QueryCtx, c as FileSetRef, d as LockScope, f as ManifestEntry, g as ParquetCodec, h as OptimizedQueryResult, i as DEFAULT_SEARCH_TYPE, j as SyncStateFilter, k as SyncState, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeResult, w as QueryResult, x as QueryExecuteOptions, y as PurgeUrlsResult, z as WriteCtx } from "./_chunks/storage.mjs";
 import { a as createDuckDBExecutor, i as createDuckDBCodec, n as DuckDBHandle, r as canonicalEmptyParquetSchema, t as DuckDBFactory } from "./_chunks/duckdb.mjs";
 import { _ as pages, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as page_keywords, h as keywords, i as TableSchema, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema } from "./_chunks/schema.mjs";
+import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
@@ -9,4 +10,4 @@ declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type OptimizedQueryResult, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type OptimizedQueryResult, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, resolveToSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow };