@gscdump/engine 0.10.0 → 0.11.1

package/dist/_chunks/engine.mjs

@@ -318,6 +318,7 @@ function createStorageEngine(opts) {
 		opts.signal?.throwIfAborted();
 		const entries = Object.entries(opts.fileSets);
 		const perSet = await Promise.all(entries.map(async ([name, ref]) => {
+			if (ref.keys !== void 0) return [name, ref.keys];
 			return [name, (await manifestStore.listLive({
 				userId: opts.ctx.userId,
 				siteId: opts.ctx.siteId,

package/dist/_chunks/storage.d.mts

@@ -375,6 +375,15 @@ interface QueryExecutor {
 interface FileSetRef {
   table: TableName;
   partitions?: string[];
+  /**
+   * Pre-resolved object keys, bypassing the manifest lookup. When provided,
+   * runSQL skips `manifestStore.listLive` for this entry and uses these keys
+   * directly. Use for entity-store sidecars (`entities/inspections/index.parquet`,
+   * `entities/sitemaps/urls/index.parquet`) which aren't registered in the
+   * analytics manifest. `table` is still required as the schema sentinel for
+   * the empty-fallback rewrite, but isn't consulted when `keys` is non-empty.
+   */
+  keys?: string[];
 }
 interface RunSQLOptions {
   ctx: TenantCtx;

package/dist/entities.d.mts

@@ -1,4 +1,5 @@
 import { a as DataSource } from "./_chunks/storage.mjs";
+import { ScheduleState } from "./schedule.mjs";
 import { TenantCtx } from "gscdump/contracts";
 /**
  * GSC URL inspection result fields we persist. Mirrors the
@@ -28,8 +29,17 @@ interface InspectionRecord {
    * Free-form payload for fields we don't promote to first-class columns
    *  (e.g. `referringUrls`, `crawledAs`). Keeps the wire format forward-compat
    *  without bumping the schema for every API addition.
+   *
+   *  Recognised keys:
+   *  - `schedule`: optional `ScheduleState` from {@link inspectionPolicy}
+   *    governing when this URL is next due for re-inspection. Undefined on
+   *    pre-§0 records — readers must tolerate the missing field and fall
+   *    back to default policy on first observe.
    */
-  raw?: unknown;
+  raw?: {
+    schedule?: ScheduleState;
+    [key: string]: unknown;
+  };
 }
 /** Wire shape persisted to disk/R2. */
 interface InspectionIndex {
@@ -44,6 +54,7 @@ interface InspectionHistoryShard {
 }
 declare function inspectionIndexKey(ctx: TenantCtx): string;
 declare function emptyTypesKey(ctx: TenantCtx): string;
+declare function inspectionParquetKey(ctx: TenantCtx): string;
 declare function inspectionHistoryKey(ctx: TenantCtx, yearMonth: string): string;
 /**
  * Stable URL hash used as the index key. Short, URL-safe, deterministic.
@@ -66,6 +77,34 @@ interface InspectionStore {
   loadIndex: (ctx: TenantCtx) => Promise<InspectionIndex>;
   /** Read the per-month history shard if it exists. */
   loadHistory: (ctx: TenantCtx, yearMonth: string) => Promise<InspectionHistoryShard | undefined>;
+  /**
+   * Snapshot the current JSON index to a parquet sidecar at
+   * `entities/inspections/index.parquet`. One PUT. Sorted by `urlHash` so
+   * DuckDB row-group stats can prune URL-keyed JOINs efficiently.
+   *
+   * Internal seam: callers don't choose JSON-vs-parquet — the store materialises
+   * the parquet at end-of-batch (e.g. after `indexing/complete`) and readers
+   * pick the format that matches their access pattern (parquet for JOINs,
+   * JSON for full-index scans / point lookups).
+   *
+   * Returns the parquet object key (matches {@link parquetUri} after write).
+   */
+  materialize: (ctx: TenantCtx) => Promise<{
+    key: string;
+    rowCount: number;
+    bytes: number;
+  }>;
+  /**
+   * DuckDB-resolvable URI for the materialised parquet sidecar, or
+   * `undefined` if the underlying `DataSource` has no native URI shape
+   * (in-memory tests). When defined, read paths can `read_parquet(<uri>)`
+   * directly without staging bytes through JS.
+   *
+   * Does not check existence — caller is responsible for ensuring
+   * `materialize` has run at least once. Returning a URI for a missing key
+   * is safe; DuckDB will surface a 404 / not-found at query time.
+   */
+  parquetUri: (ctx: TenantCtx) => string | undefined;
 }
 interface CreateInspectionStoreOptions {
   dataSource: DataSource;
@@ -100,6 +139,12 @@ interface SitemapRecord {
   }>;
   /** Raw payload for fields we don't promote to first-class columns. */
   raw?: unknown;
+  /** Number of URLs observed in this feedpath at last snapshot. */
+  urlCount?: number;
+  /** Stable hash of the sorted normalized loc list at last snapshot. */
+  contentHash?: string;
+  /** Adaptive cadence state owned by `sitemapPolicy`. */
+  schedule?: ScheduleState;
 }
 interface SitemapIndex {
   version: 1;
@@ -114,6 +159,57 @@ interface SitemapHistoryDoc {
 }
 declare function sitemapIndexKey(ctx: TenantCtx): string;
 declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
+declare function sitemapUrlsIndexKey(ctx: TenantCtx): string;
+declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, date: string): string;
+/** Parsed URL entry from a sitemap XML. */
+interface ParsedUrl {
+  loc: string;
+  /** ISO-8601 lastmod from the sitemap, if present. */
+  lastmod?: string;
+}
+/** A single URL row in the urls/index.parquet partition. */
+interface SitemapUrlRecord {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  loc: string;
+  lastmod?: string;
+  firstSeenAt: number;
+  lastSeenAt: number;
+  /** Set when the URL has been removed. Null/undefined = currently live. */
+  removedAt?: number;
+}
+interface SnapshotUrlsResult {
+  added: number;
+  removed: number;
+  kept: number;
+  contentHash: string;
+  /** True when contentHash matched prior; the call performed zero writes. */
+  unchanged: boolean;
+}
+interface DeltaEntry {
+  feedpath: string;
+  feedpathHash: string;
+  urlHash: string;
+  op: 'added' | 'removed';
+  loc: string;
+  lastmod?: string;
+  at: number;
+}
+interface DateRange {
+  /** YYYY-MM-DD inclusive. */
+  from?: string;
+  /** YYYY-MM-DD inclusive. */
+  to?: string;
+}
+interface LoadUrlsOptions {
+  includeRemoved?: boolean;
+}
+/**
+ * Hash a URL list for change detection. Sorts then folds via FNV-1a so it's
+ * deterministic, locale-free, and cheap on Workers.
+ */
+declare function hashUrlList(urls: readonly ParsedUrl[]): string;
 interface SitemapStore {
   /**
    * Persist a snapshot run. Updates the index + writes one immutable
@@ -124,6 +220,24 @@ interface SitemapStore {
   loadIndex: (ctx: TenantCtx) => Promise<SitemapIndex>;
   /** Fetch the latest snapshot for a feedpath, or undefined. */
   getLatest: (ctx: TenantCtx, path: string) => Promise<SitemapRecord | undefined>;
+  /**
+   * Diff incoming URLs against the prior `urls/index.parquet` partition for
+   * `feedpath`; on change, writes a single delta parquet under
+   * `urls/deltas/YYYY-MM-DD__{feedpathHash}.parquet`. Skipped (0 PUTs) when
+   * `contentHash` matches prior.
+   */
+  snapshotUrls: (ctx: TenantCtx, feedpath: string, urls: readonly ParsedUrl[]) => Promise<SnapshotUrlsResult>;
+  /** Stream live (and optionally removed) URL rows for a feedpath. */
+  loadUrls: (ctx: TenantCtx, feedpath: string, opts?: LoadUrlsOptions) => AsyncIterable<SitemapUrlRecord>;
+  /** Stream all delta entries within `[from, to]` (YYYY-MM-DD inclusive). */
+  loadDeltas: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<DeltaEntry>;
+  /**
+   * Fold every accumulated delta into the prior index; writes a fresh
+   * `urls/index.parquet` and deletes the consumed delta files.
+   */
+  compactUrls: (ctx: TenantCtx) => Promise<void>;
+  /** DuckDB-resolvable URI for the URLs index; `undefined` if backend lacks one. */
+  urlsParquetUri: (ctx: TenantCtx) => string | undefined;
 }
 interface CreateSitemapStoreOptions {
   dataSource: DataSource;
@@ -175,4 +289,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, EmptyTypesDoc, EmptyTypesStore, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionIndex, InspectionRecord, InspectionStore, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, sitemapHistoryKey, sitemapIndexKey };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionIndex, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };

package/dist/entities.mjs

@@ -1,3 +1,4 @@
+import { decodeParquetToRows, encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 function inspectionIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.json` : `u_${ctx.userId}/entities/inspections/index.json`;
@@ -5,6 +6,9 @@ function inspectionIndexKey(ctx) {
 function emptyTypesKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/empty-types.json` : `u_${ctx.userId}/entities/empty-types.json`;
 }
+function inspectionParquetKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.parquet` : `u_${ctx.userId}/entities/inspections/index.parquet`;
+}
 function inspectionHistoryKey(ctx, yearMonth) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/history/${yearMonth}.json` : `u_${ctx.userId}/entities/inspections/history/${yearMonth}.json`;
 }
@@ -75,15 +79,251 @@ function createInspectionStore(opts) {
 		},
 		async loadHistory(ctx, yearMonth) {
 			return await readJson(inspectionHistoryKey(ctx, yearMonth));
+		},
+		async materialize(ctx) {
+			const index = await readJson(inspectionIndexKey(ctx)) ?? emptyIndex();
+			const rows = Object.entries(index.records).map(([urlHash, r]) => ({
+				urlHash,
+				url: r.url,
+				inspectedAt: r.inspectedAt,
+				indexStatus: r.indexStatus ?? null,
+				lastCrawlTime: r.lastCrawlTime ?? null,
+				googleCanonical: r.googleCanonical ?? null,
+				userCanonical: r.userCanonical ?? null,
+				coverageState: r.coverageState ?? null,
+				robotsTxtState: r.robotsTxtState ?? null,
+				indexingState: r.indexingState ?? null,
+				pageFetchState: r.pageFetchState ?? null,
+				mobileUsabilityVerdict: r.mobileUsabilityVerdict ?? null,
+				richResultsVerdict: r.richResultsVerdict ?? null,
+				scheduleNextAt: r.raw?.schedule?.nextAt ?? null,
+				scheduleConsecutiveUnchanged: r.raw?.schedule?.consecutiveUnchanged ?? null,
+				schedulePolicyVersion: r.raw?.schedule?.policyVersion ?? null
+			}));
+			const bytes = encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_PARQUET_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			const key = inspectionParquetKey(ctx);
+			await ds.write(key, bytes);
+			return {
+				key,
+				rowCount: rows.length,
+				bytes: bytes.byteLength
+			};
+		},
+		parquetUri(ctx) {
+			return ds.uri?.(inspectionParquetKey(ctx));
 		}
 	};
 }
+const INSPECTION_PARQUET_COLUMNS = [
+	{
+		name: "urlHash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "inspectedAt",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "indexStatus",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "lastCrawlTime",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "googleCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "userCanonical",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "coverageState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "robotsTxtState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "indexingState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "pageFetchState",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileUsabilityVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsVerdict",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "scheduleNextAt",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "scheduleConsecutiveUnchanged",
+		type: "INTEGER",
+		nullable: true
+	},
+	{
+		name: "schedulePolicyVersion",
+		type: "INTEGER",
+		nullable: true
+	}
+];
 function sitemapIndexKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/index.json` : `u_${ctx.userId}/entities/sitemaps/index.json`;
 }
 function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json` : `u_${ctx.userId}/entities/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
 }
+function sitemapUrlsPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/urls` : `u_${ctx.userId}/entities/sitemaps/urls`;
+}
+function sitemapUrlsIndexKey(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/index.parquet`;
+}
+function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
+	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
+}
+const SITEMAP_URLS_DELTA_PREFIX_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)\.parquet$/;
+const URLS_INDEX_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "first_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "last_seen_at",
+		type: "BIGINT",
+		nullable: false
+	},
+	{
+		name: "removed_at",
+		type: "BIGINT",
+		nullable: true
+	}
+];
+const URLS_DELTA_COLUMNS = [
+	{
+		name: "feedpath",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "feedpath_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "url_hash",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "op",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "loc",
+		type: "VARCHAR",
+		nullable: false
+	},
+	{
+		name: "lastmod",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "at",
+		type: "BIGINT",
+		nullable: false
+	}
+];
+function rowToUrlRecord(row) {
+	return {
+		feedpath: String(row.feedpath),
+		feedpathHash: String(row.feedpath_hash),
+		urlHash: String(row.url_hash),
+		loc: String(row.loc),
+		lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
+		firstSeenAt: Number(row.first_seen_at),
+		lastSeenAt: Number(row.last_seen_at),
+		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
+	};
+}
+function urlRecordToRow(r) {
+	return {
+		feedpath: r.feedpath,
+		feedpath_hash: r.feedpathHash,
+		url_hash: r.urlHash,
+		loc: r.loc,
+		lastmod: r.lastmod ?? null,
+		first_seen_at: r.firstSeenAt,
+		last_seen_at: r.lastSeenAt,
+		removed_at: r.removedAt ?? null
+	};
+}
+function isoDate(ms) {
+	return new Date(ms).toISOString().slice(0, 10);
+}
+function hashUrlList(urls) {
+	return hashUrl(urls.map((u) => u.loc).sort().join("\n"));
+}
 function createSitemapStore(opts) {
 	const ds = opts.dataSource;
 	const hash = opts.hash ?? hashUrl;
@@ -123,6 +363,218 @@ function createSitemapStore(opts) {
 		},
 		async getLatest(ctx, path) {
 			return (await readJson(sitemapIndexKey(ctx)))?.records[hash(path)];
+		},
+		async snapshotUrls(ctx, feedpath, urls) {
+			const fpHash = hash(feedpath);
+			const contentHash = hashUrlList(urls);
+			const at = now();
+			const priorByHash = /* @__PURE__ */ new Map();
+			for await (const rec of this.loadUrls(ctx, feedpath, { includeRemoved: true })) priorByHash.set(rec.urlHash, rec);
+			const livePrior = Array.from(priorByHash.values()).filter((r) => r.removedAt == null);
+			if (livePrior.length > 0) {
+				if (hashUrl(livePrior.map((r) => String(r.loc)).sort().join("\n")) === contentHash) return {
+					added: 0,
+					removed: 0,
+					kept: livePrior.length,
+					contentHash,
+					unchanged: true
+				};
+			}
+			const incomingByHash = /* @__PURE__ */ new Map();
+			for (const u of urls) incomingByHash.set(hash(u.loc), u);
+			const deltaRows = [];
+			let added = 0;
+			let removed = 0;
+			let kept = 0;
+			const date = isoDate(at);
+			for (const [urlHash, u] of incomingByHash) {
+				const prev = priorByHash.get(urlHash);
+				if (!prev || prev.removedAt != null) {
+					added++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "added",
+						loc: u.loc,
+						lastmod: u.lastmod ?? null,
+						at
+					});
+				} else kept++;
+			}
+			for (const [urlHash, prev] of priorByHash) {
+				if (prev.removedAt != null) continue;
+				if (!incomingByHash.has(urlHash)) {
+					removed++;
+					deltaRows.push({
+						feedpath,
+						feedpath_hash: fpHash,
+						url_hash: urlHash,
+						op: "removed",
+						loc: prev.loc,
+						lastmod: prev.lastmod ?? null,
+						at
+					});
+				}
+			}
+			if (deltaRows.length > 0) {
+				const bytes = encodeRowsToParquetFlex(deltaRows, {
+					columns: URLS_DELTA_COLUMNS,
+					sortKey: ["url_hash"]
+				});
+				await ds.write(sitemapUrlsDeltaKey(ctx, fpHash, date), bytes);
+			}
+			return {
+				added,
+				removed,
+				kept,
+				contentHash,
+				unchanged: false
+			};
+		},
+		async *loadUrls(ctx, feedpath, opts) {
+			const fpHash = hash(feedpath);
+			const includeRemoved = opts?.includeRemoved ?? false;
+			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx)).catch(() => void 0);
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			const live = /* @__PURE__ */ new Map();
+			const removedMap = /* @__PURE__ */ new Map();
+			for (const row of indexRows) {
+				if (row.feedpath_hash !== fpHash) continue;
+				const rec = rowToUrlRecord(row);
+				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
+				else live.set(rec.urlHash, rec);
+			}
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m || m[2] !== fpHash) continue;
+				const dBytes = await ds.read(key).catch(() => void 0);
+				if (!dBytes) continue;
+				const dRows = await decodeParquetToRows(dBytes);
+				for (const r of dRows) {
+					const op = String(r.op);
+					const urlHash = String(r.url_hash);
+					const at = Number(r.at);
+					if (op === "added") {
+						const prev = live.get(urlHash) ?? removedMap.get(urlHash);
+						removedMap.delete(urlHash);
+						live.set(urlHash, {
+							feedpath,
+							feedpathHash: fpHash,
+							urlHash,
+							loc: String(r.loc),
+							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+							firstSeenAt: prev?.firstSeenAt ?? at,
+							lastSeenAt: at
+						});
+					} else if (op === "removed") {
+						const prev = live.get(urlHash);
+						live.delete(urlHash);
+						if (prev) removedMap.set(urlHash, {
+							...prev,
+							removedAt: at
+						});
+					}
+				}
+			}
+			for (const rec of live.values()) yield rec;
+			if (includeRemoved) for (const rec of removedMap.values()) yield rec;
+		},
+		async *loadDeltas(ctx, dateRange) {
+			const from = dateRange?.from;
+			const to = dateRange?.to;
+			const keys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			for (const key of keys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const date = m[1];
+				if (from && date < from) continue;
+				if (to && date > to) continue;
+				const bytes = await ds.read(key).catch(() => void 0);
+				if (!bytes) continue;
+				const rows = await decodeParquetToRows(bytes);
+				for (const r of rows) {
+					const op = String(r.op);
+					if (op !== "added" && op !== "removed") continue;
+					yield {
+						feedpath: String(r.feedpath),
+						feedpathHash: String(r.feedpath_hash),
+						urlHash: String(r.url_hash),
+						op,
+						loc: String(r.loc),
+						lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+						at: Number(r.at)
+					};
+				}
+			}
+		},
+		async compactUrls(ctx) {
+			const indexKey = sitemapUrlsIndexKey(ctx);
+			const indexBytes = await ds.read(indexKey).catch(() => void 0);
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+			const stateKey = (fp, u) => `${fp}::${u}`;
+			const live = /* @__PURE__ */ new Map();
+			const removed = /* @__PURE__ */ new Map();
+			for (const row of indexRows) {
+				const rec = rowToUrlRecord(row);
+				const k = stateKey(rec.feedpathHash, rec.urlHash);
+				if (rec.removedAt != null) removed.set(k, rec);
+				else live.set(k, rec);
+			}
+			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
+			const consumed = [];
+			for (const key of deltaKeys) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				const fpHash = m[2];
+				const bytes = await ds.read(key).catch(() => void 0);
+				if (!bytes) continue;
+				consumed.push(key);
+				const rows = await decodeParquetToRows(bytes);
+				for (const r of rows) {
+					const urlHash = String(r.url_hash);
+					const at = Number(r.at);
+					const k = stateKey(fpHash, urlHash);
+					const op = String(r.op);
+					if (op === "added") {
+						const prev = live.get(k) ?? removed.get(k);
+						removed.delete(k);
+						live.set(k, {
+							feedpath: String(r.feedpath),
+							feedpathHash: fpHash,
+							urlHash,
+							loc: String(r.loc),
+							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+							firstSeenAt: prev?.firstSeenAt ?? at,
+							lastSeenAt: at
+						});
+					} else if (op === "removed") {
+						const prev = live.get(k);
+						live.delete(k);
+						if (prev) removed.set(k, {
+							...prev,
+							removedAt: at
+						});
+					}
+				}
+			}
+			const merged = [...live.values(), ...removed.values()];
+			merged.sort((a, b) => {
+				if (a.feedpathHash !== b.feedpathHash) return a.feedpathHash < b.feedpathHash ? -1 : 1;
+				if (a.urlHash !== b.urlHash) return a.urlHash < b.urlHash ? -1 : 1;
+				return 0;
+			});
+			const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+				columns: URLS_INDEX_COLUMNS,
+				sortKey: ["feedpath_hash", "url_hash"]
+			});
+			await ds.write(indexKey, bytes);
+			if (consumed.length > 0) await ds.delete(consumed);
+		},
+		urlsParquetUri(ctx) {
+			const key = sitemapUrlsIndexKey(ctx);
+			return ds.uri ? ds.uri(key) : void 0;
 		}
 	};
 }
@@ -206,4 +658,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, sitemapHistoryKey, sitemapIndexKey };
+export { createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };

package/dist/index.d.mts

@@ -1,6 +1,7 @@
 import { A as SyncStateDetail, B as WriteResult, C as QueryExecutor, D as SearchType, E as RunSQLOptions, F as TenantCtx, G as CompactionThresholds, H as inferLegacyTier, I as Watermark, K as enumeratePartitions, L as WatermarkFilter, M as SyncStateKind, N as SyncStateScope, O as StorageEngine, P as TableName, R as WatermarkScope, S as QueryExecuteResult, T as Row, U as inferSearchType, V as dayPartition, W as objectKey, _ as PurgeFilter, a as DataSource, b as QueryCtx, c as FileSetRef, d as LockScope, f as ManifestEntry, g as ParquetCodec, h as OptimizedQueryResult, i as DEFAULT_SEARCH_TYPE, j as SyncStateFilter, k as SyncState, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeResult, w as QueryResult, x as QueryExecuteOptions, y as PurgeUrlsResult, z as WriteCtx } from "./_chunks/storage.mjs";
 import { a as createDuckDBExecutor, i as createDuckDBCodec, n as DuckDBHandle, r as canonicalEmptyParquetSchema, t as DuckDBFactory } from "./_chunks/duckdb.mjs";
 import { _ as pages, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as page_keywords, h as keywords, i as TableSchema, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema } from "./_chunks/schema.mjs";
+import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
@@ -9,4 +10,4 @@ declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type OptimizedQueryResult, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, objectKey, page_keywords, pages, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type OptimizedQueryResult, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, resolveToSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow };

package/dist/index.mjs

@@ -5,6 +5,7 @@ import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
 import { a as createDuckDBExecutor, i as createDuckDBCodec, n as createStorageEngine, r as canonicalEmptyParquetSchema, t as MAX_DAY_BYTES } from "./_chunks/engine.mjs";
 import { createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import "./planner.mjs";
+import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 function coerceRow(row) {
 	let mutated = null;
 	for (const [k, v] of Object.entries(row)) if (typeof v === "bigint") {
@@ -18,4 +19,4 @@ function coerceRows(rows) {
 	for (let i = 0; i < rows.length; i++) out[i] = coerceRow(rows[i]);
 	return out;
 }
-export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, SCHEMAS, TABLE_METADATA, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, objectKey, page_keywords, pages, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow };
+export { DEFAULT_SEARCH_TYPE, FILES_PLACEHOLDER, MAX_DAY_BYTES, SCHEMAS, TABLE_METADATA, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, fixedPolicy, formatLiteral, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, keywords, objectKey, page_keywords, pages, resolveToSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow };

package/dist/rollups.d.mts

@@ -159,5 +159,38 @@ declare const topKeywords28dParquetRollup: RollupDef;
  * so downstream readers don't have to special-case first-run sites.
  */
 declare const indexingMetadataRollup: RollupDef;
+/**
+ * Indexing-API health by day: per `inspectedAt` date, counts of indexed,
+ * soft-404, redirect, not-found, mobile passes, rich-results passes, and
+ * canonical mismatches. Sourced from the inspections parquet sidecar
+ * (`InspectionStore.parquetUri`), which holds the latest record per URL.
+ *
+ * Empty-payload no-op when the sidecar URI is unavailable (in-memory
+ * `DataSource`, or before `materialize` has run).
+ */
+declare const indexingHealthRollup: RollupDef;
+/**
+ * Per-day index-percent: ratio of (sitemap URLs that received GSC clicks on
+ * that date) / (total live sitemap URLs). Uses a DuckDB JOIN between the
+ * sitemap urls parquet (`SitemapStore.urlsParquetUri`) and the `pages` fact
+ * parquet. Total denominator is the count of live URLs in the urls index;
+ * numerator is per-day distinct loc count where pages.clicks > 0.
+ */
+declare const indexPercentRollup: RollupDef;
+/**
+ * Sitemap-health per-day series materialized from the sitemap-store JSON
+ * index. Each `SitemapRecord` carries `urlCount`, `errors`, `warnings`,
+ * `contentHash`, and `lastDownloaded`. We bucket records by the day of their
+ * `capturedAt` (or `lastDownloaded` fallback) and emit per-day aggregates plus
+ * a snapshot of per-feed stats at the most recent capture.
+ */
+declare const sitemapHealthRollup: RollupDef;
+/**
+ * Trailing-28-day sitemap URL changes: per-day per-feedpath {added, removed}
+ * counts plus rolling top-200 added and removed URLs. Streams from
+ * `SitemapStore.loadDeltas()` so it scales independently of how many feeds
+ * exist on the site.
+ */
+declare const sitemapChanges28dRollup: RollupDef;
 declare const DEFAULT_ROLLUPS: readonly RollupDef[];
-export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildRollupResult, RebuildRollupsOptions, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, dailyTotalsRollup, indexingMetadataRollup, rebuildRollups, rollupKey, rollupParquetKey, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildRollupResult, RebuildRollupsOptions, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -1,5 +1,5 @@
-import { createIndexingMetadataStore } from "./entities.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
+import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexKey } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
@@ -335,12 +335,220 @@ const indexingMetadataRollup = {
 		};
 	}
 };
+const indexingHealthRollup = {
+	id: "indexing_health",
+	windowDays: 90,
+	async build({ engine, ctx, dataSource, builtAt }) {
+		const key = inspectionParquetKey(ctx);
+		if (!await dataSource.head?.(key)) return { days: [] };
+		const sql = `
+      SELECT
+        substr(inspectedAt, 1, 10) AS date,
+        COUNT(*)::BIGINT AS total_urls,
+        SUM(CASE WHEN indexStatus = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS indexed_count,
+        SUM(CASE WHEN pageFetchState = 'SOFT_404' THEN 1 ELSE 0 END)::BIGINT AS soft_404,
+        SUM(CASE WHEN pageFetchState = 'REDIRECT_ERROR' THEN 1 ELSE 0 END)::BIGINT AS redirect,
+        SUM(CASE WHEN pageFetchState = 'NOT_FOUND' THEN 1 ELSE 0 END)::BIGINT AS not_found,
+        SUM(CASE WHEN mobileUsabilityVerdict = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS mobile_passes,
+        SUM(CASE WHEN richResultsVerdict = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
+        SUM(CASE WHEN userCanonical IS NOT NULL AND googleCanonical IS NOT NULL AND userCanonical <> googleCanonical THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
+      FROM read_parquet({{INSPECTIONS}}, union_by_name = true)
+      WHERE substr(inspectedAt, 1, 10) >= '${utcDateMinusDays(builtAt, 90)}'
+      GROUP BY 1
+      ORDER BY 1
+    `;
+		return { days: (await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: { INSPECTIONS: {
+				table: "pages",
+				keys: [key]
+			} },
+			sql
+		})).rows.map((r) => ({
+			date: String(r.date),
+			total_urls: Number(r.total_urls),
+			indexed_count: Number(r.indexed_count),
+			soft_404: Number(r.soft_404),
+			redirect: Number(r.redirect),
+			not_found: Number(r.not_found),
+			mobile_passes: Number(r.mobile_passes),
+			rich_results_passes: Number(r.rich_results_passes),
+			canonical_mismatches: Number(r.canonical_mismatches)
+		})) };
+	}
+};
+const indexPercentRollup = {
+	id: "index_percent",
+	windowDays: 90,
+	async build({ engine, ctx, dataSource, builtAt }) {
+		const urlsKey = sitemapUrlsIndexKey(ctx);
+		if (!await dataSource.head?.(urlsKey)) return {
+			totalSitemapUrls: 0,
+			days: []
+		};
+		const cutoff = utcDateMinusDays(builtAt, 90);
+		const numerator = await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: {
+				PAGES: { table: "pages" },
+				URLS: {
+					table: "pages",
+					keys: [urlsKey]
+				}
+			},
+			sql: `
+        SELECT
+          p.date AS date,
+          COUNT(DISTINCT p.url)::BIGINT AS clicked_urls
+        FROM read_parquet({{PAGES}}, union_by_name = true) p
+        INNER JOIN read_parquet({{URLS}}, union_by_name = true) s
+          ON s.loc = p.url AND s.removed_at IS NULL
+        WHERE p.clicks > 0 AND p.date >= '${cutoff}'
+        GROUP BY p.date
+        ORDER BY p.date
+      `
+		});
+		const denom = await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: { URLS: {
+				table: "pages",
+				keys: [urlsKey]
+			} },
+			sql: `
+        SELECT COUNT(*)::BIGINT AS total
+        FROM read_parquet({{URLS}}, union_by_name = true)
+        WHERE removed_at IS NULL
+      `
+		});
+		const total = Number(denom.rows[0]?.total ?? 0);
+		return {
+			totalSitemapUrls: total,
+			days: numerator.rows.map((r) => {
+				const clicked = Number(r.clicked_urls);
+				return {
+					date: String(r.date),
+					clicked_urls: clicked,
+					total_sitemap_urls: total,
+					ratio: total === 0 ? 0 : clicked / total
+				};
+			})
+		};
+	}
+};
+const sitemapHealthRollup = {
+	id: "sitemap_health",
+	windowDays: 90,
+	async build({ dataSource, ctx, builtAt }) {
+		const index = await createSitemapStore({ dataSource }).loadIndex(ctx);
+		const records = Object.values(index.records);
+		const cutoff = utcDateMinusDays(builtAt, 90);
+		const byDay = /* @__PURE__ */ new Map();
+		const feeds = [];
+		for (const r of records) {
+			const day = (r.capturedAt ?? r.lastDownloaded ?? "").slice(0, 10);
+			if (!day || day < cutoff) continue;
+			const errors = Number(r.errors ?? 0);
+			const warnings = Number(r.warnings ?? 0);
+			const urlCount = Number(r.urlCount ?? 0);
+			const bucket = byDay.get(day) ?? {
+				day,
+				feeds: 0,
+				total_urls: 0,
+				errors: 0,
+				warnings: 0
+			};
+			bucket.feeds += 1;
+			bucket.total_urls += urlCount;
+			bucket.errors += errors;
+			bucket.warnings += warnings;
+			byDay.set(day, bucket);
+			feeds.push({
+				path: r.path,
+				urlCount,
+				errors,
+				warnings,
+				contentHash: r.contentHash ?? null,
+				lastDownloaded: r.lastDownloaded ?? null,
+				capturedAt: r.capturedAt
+			});
+		}
+		return {
+			days: Array.from(byDay.values()).sort((a, b) => a.day < b.day ? -1 : 1),
+			feeds
+		};
+	}
+};
+const sitemapChanges28dRollup = {
+	id: "sitemap_changes_28d",
+	windowDays: 28,
+	async build({ dataSource, ctx, builtAt }) {
+		const store = createSitemapStore({ dataSource });
+		const from = utcDateMinusDays(builtAt, 28);
+		const to = utcDateMinusDays(builtAt, 0);
+		const counts = /* @__PURE__ */ new Map();
+		const addedTop = [];
+		const removedTop = [];
+		function key(k) {
+			return `${k.day}\x00${k.feedpath}`;
+		}
+		for await (const d of store.loadDeltas(ctx, {
+			from,
+			to
+		})) {
+			const day = new Date(d.at).toISOString().slice(0, 10);
+			const k = key({
+				day,
+				feedpath: d.feedpath
+			});
+			const cur = counts.get(k) ?? {
+				day,
+				feedpath: d.feedpath,
+				added: 0,
+				removed: 0
+			};
+			if (d.op === "added") {
+				cur.added += 1;
+				addedTop.push({
+					loc: d.loc,
+					feedpath: d.feedpath,
+					at: d.at
+				});
+			} else {
+				cur.removed += 1;
+				removedTop.push({
+					loc: d.loc,
+					feedpath: d.feedpath,
+					at: d.at
+				});
+			}
+			counts.set(k, cur);
+		}
+		const days = Array.from(counts.values()).sort((a, b) => {
+			if (a.day !== b.day) return a.day < b.day ? -1 : 1;
+			return a.feedpath < b.feedpath ? -1 : 1;
+		});
+		addedTop.sort((a, b) => b.at - a.at);
+		removedTop.sort((a, b) => b.at - a.at);
+		return {
+			days,
+			topAdded: addedTop.slice(0, 200),
+			topRemoved: removedTop.slice(0, 200)
+		};
+	}
+};
 const DEFAULT_ROLLUPS = [
 	dailyTotalsRollup,
 	weeklyTotalsRollup,
 	topPages28dRollup,
 	topKeywords28dRollup,
 	topCountries28dRollup,
-	indexingMetadataRollup
+	indexingMetadataRollup,
+	indexingHealthRollup,
+	indexPercentRollup,
+	sitemapHealthRollup,
+	sitemapChanges28dRollup
 ];
-export { DEFAULT_ROLLUPS, dailyTotalsRollup, indexingMetadataRollup, rebuildRollups, rollupKey, rollupParquetKey, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { DEFAULT_ROLLUPS, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/schedule.d.mts

@@ -0,0 +1,19 @@
+interface ScheduleState {
+  nextAt: number;
+  consecutiveUnchanged: number;
+  policyVersion: number;
+}
+interface SchedulePolicy {
+  readonly version: number;
+  initial: (now: number) => ScheduleState;
+  observe: (prev: ScheduleState, evt: {
+    changed: boolean;
+    at: number;
+  }) => ScheduleState;
+  isDue: (state: ScheduleState, now: number) => boolean;
+}
+declare const sitemapPolicy: SchedulePolicy;
+type InspectionVerdict = 'PASS' | 'FAIL' | 'NEUTRAL';
+declare function inspectionPolicy(verdict: InspectionVerdict): SchedulePolicy;
+declare function fixedPolicy(intervalMs: number): SchedulePolicy;
+export { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspectionPolicy, sitemapPolicy };

package/dist/schedule.mjs

@@ -0,0 +1,100 @@
+const DAY = 24 * (3600 * 1e3);
+function isDue(state, now) {
+	return now >= state.nextAt;
+}
+function sitemapCadenceMs(consecutiveUnchanged) {
+	if (consecutiveUnchanged >= 7) return 30 * DAY;
+	if (consecutiveUnchanged >= 3) return 7 * DAY;
+	return DAY;
+}
+const SITEMAP_VERSION = 1;
+const sitemapPolicy = {
+	version: SITEMAP_VERSION,
+	initial(now) {
+		return {
+			nextAt: now + DAY,
+			consecutiveUnchanged: 0,
+			policyVersion: SITEMAP_VERSION
+		};
+	},
+	observe(prev, evt) {
+		if (prev.policyVersion !== SITEMAP_VERSION) return {
+			nextAt: evt.at + sitemapCadenceMs(0),
+			consecutiveUnchanged: 0,
+			policyVersion: SITEMAP_VERSION
+		};
+		if (evt.changed) return {
+			nextAt: evt.at + DAY,
+			consecutiveUnchanged: 0,
+			policyVersion: SITEMAP_VERSION
+		};
+		const next = prev.consecutiveUnchanged + 1;
+		return {
+			nextAt: evt.at + sitemapCadenceMs(next),
+			consecutiveUnchanged: next,
+			policyVersion: SITEMAP_VERSION
+		};
+	},
+	isDue
+};
+const INSPECTION_VERSION = 1;
+function inspectionCadenceMs(verdict) {
+	if (verdict === "PASS") return 30 * DAY;
+	if (verdict === "FAIL") return 7 * DAY;
+	return 14 * DAY;
+}
+function inspectionPolicy(verdict) {
+	const cadence = inspectionCadenceMs(verdict);
+	return {
+		version: INSPECTION_VERSION,
+		initial(now) {
+			return {
+				nextAt: now + cadence,
+				consecutiveUnchanged: 0,
+				policyVersion: INSPECTION_VERSION
+			};
+		},
+		observe(prev, evt) {
+			if (prev.policyVersion !== INSPECTION_VERSION) return {
+				nextAt: evt.at + cadence,
+				consecutiveUnchanged: 0,
+				policyVersion: INSPECTION_VERSION
+			};
+			const next = evt.changed ? 0 : prev.consecutiveUnchanged + 1;
+			return {
+				nextAt: evt.at + cadence,
+				consecutiveUnchanged: next,
+				policyVersion: INSPECTION_VERSION
+			};
+		},
+		isDue
+	};
+}
+const FIXED_VERSION = 1;
+function fixedPolicy(intervalMs) {
+	return {
+		version: FIXED_VERSION,
+		initial(now) {
+			return {
+				nextAt: now + intervalMs,
+				consecutiveUnchanged: 0,
+				policyVersion: FIXED_VERSION
+			};
+		},
+		observe(prev, evt) {
+			if (prev.policyVersion !== FIXED_VERSION) return {
+				nextAt: evt.at + intervalMs,
+				consecutiveUnchanged: 0,
+				policyVersion: FIXED_VERSION
+			};
+			const next = evt.changed ? 0 : prev.consecutiveUnchanged + 1;
+			return {
+				nextAt: evt.at + intervalMs,
+				consecutiveUnchanged: next,
+				policyVersion: FIXED_VERSION
+			};
+		},
+		isDue
+	};
+}
+export { fixedPolicy, inspectionPolicy, sitemapPolicy };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.10.0",
+  "version": "0.11.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -61,6 +61,11 @@
       "import": "./dist/sql-fragments.mjs",
       "default": "./dist/sql-fragments.mjs"
     },
+    "./schedule": {
+      "types": "./dist/schedule.d.mts",
+      "import": "./dist/schedule.mjs",
+      "default": "./dist/schedule.mjs"
+    },
     "./entities": {
       "types": "./dist/entities.d.mts",
       "import": "./dist/entities.mjs",
@@ -164,7 +169,7 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.10.0"
+    "gscdump": "0.11.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",