@gscdump/engine 1.5.3 → 2.0.0

package/dist/entities/sitemap-shared.d.mts

@@ -1,54 +1,9 @@
 import { DataSource } from "../storage.mjs";
-import { ScheduleState } from "../schedule.mjs";
 import { Row, TenantCtx } from "@gscdump/contracts";
-/** GSC sitemap record we persist. Matches `Schema$WmxSitemap` but as plain JSON. */
-interface SitemapRecord {
-  /** The sitemap URL (feedpath) as returned by GSC. */
-  path: string;
-  /** ISO-8601 timestamp of the snapshot run that captured this record. */
-  capturedAt: string;
-  /** Last time Google downloaded this sitemap (RFC 3339, from the API). */
-  lastDownloaded?: string;
-  /** Last time the sitemap was submitted. */
-  lastSubmitted?: string;
-  type?: string;
-  isPending?: boolean;
-  isSitemapsIndex?: boolean;
-  errors?: string;
-  warnings?: string;
-  /** Per-content-type counts (web, image, video, news). */
-  contents?: Array<{
-    type?: string;
-    submitted?: string;
-    indexed?: string;
-  }>;
-  /** Raw payload for fields we don't promote to first-class columns. */
-  raw?: unknown;
-  /** Number of URLs observed in this feedpath at last snapshot. */
-  urlCount?: number;
-  /** Stable hash of the sorted normalized loc list at last snapshot. */
-  contentHash?: string;
-  /** Adaptive cadence state owned by `sitemapPolicy`. */
-  schedule?: ScheduleState;
-}
-interface SitemapIndex {
-  version: 1;
-  /** Map of feedpathHash → latest SitemapRecord. */
-  records: Record<string, SitemapRecord>;
-}
-interface SitemapHistoryDoc {
-  version: 1;
-  path: string;
-  capturedAt: string;
-  record: SitemapRecord;
-}
-/** Parsed URL entry from a sitemap XML. */
 interface ParsedUrl {
   loc: string;
-  /** ISO-8601 lastmod from the sitemap, if present. */
   lastmod?: string;
 }
-/** A single URL row in the urls/index.parquet partition. */
 interface SitemapUrlRecord {
   feedpath: string;
   feedpathHash: string;
@@ -57,143 +12,231 @@ interface SitemapUrlRecord {
   lastmod?: string;
   firstSeenAt: number;
   lastSeenAt: number;
-  /** Set when the URL has been removed. Null/undefined = currently live. */
   removedAt?: number;
 }
-interface SnapshotUrlsResult {
+interface SitemapGenerationFeedManifest {
+  feedpath: string;
+  feedpathHash: string;
+  baseKey: string;
+  eventKey: string | null;
+  membershipHash: string;
+  payloadHash: string;
+  urlCount: number;
+  observedAt: number;
+}
+type SitemapLegacyImportEvidence = {
+  _tag: 'none';
+} | {
+  _tag: 'metadata_only';
+  importedAt: number;
+  recordCount: number;
+  source: 'gsc_sitemaps';
+};
+interface SitemapSiteGenerationManifest {
+  version: 1;
+  generationId: string;
+  observedAt: number;
+  publishedAt: number;
+  completeness: {
+    _tag: 'complete';
+  };
+  membershipHistoryAvailableFrom: number | null;
+  legacyImport: SitemapLegacyImportEvidence;
+  previousGenerationId: string | null;
+  previousManifestKey: string | null;
+  eventKeys: string[];
+  feeds: Record<string, SitemapGenerationFeedManifest>;
+}
+interface SitemapStagedFeed {
+  version: 1;
+  generationId: string;
+  observedAt: number;
+  feed: SitemapGenerationFeedManifest;
+}
+type StageSitemapGenerationFeedResult = {
+  _tag: 'staged';
+  generationId: string;
+  feedpath: string;
+  membershipChanged: boolean;
+  payloadChanged: boolean;
   added: number;
   removed: number;
-  kept: number;
-  contentHash: string;
-  /** True when current membership did not change. Initial history seeding may write. */
-  unchanged: boolean;
+  updated: number;
+} | {
+  _tag: 'invalid_feed';
+  reason: 'credentials' | 'empty' | 'invalid_url' | 'unsupported_protocol';
+} | {
+  _tag: 'invalid_generation';
+  reason: 'empty_id' | 'invalid_observed_at';
+} | {
+  _tag: 'conflict';
+  reason: 'stale_generation' | 'generation_reused';
+};
+interface CompleteSitemapTraversal {
+  _tag: 'complete';
+  expectedFeedpaths: readonly string[];
 }
+type FinalizeSitemapGenerationResult = {
+  _tag: 'published';
+  generationId: string;
+  observedAt: number;
+  feedCount: number;
+} | {
+  _tag: 'unchanged';
+  generationId: string;
+  observedAt: number;
+  feedCount: number;
+} | {
+  _tag: 'incomplete';
+  missingFeedpaths: string[];
+  invalidFeedpaths: string[];
+} | {
+  _tag: 'conflict';
+  reason: 'stale_generation' | 'generation_reused' | 'publication_changed';
+} | {
+  _tag: 'invalid_generation';
+  reason: 'empty_id' | 'invalid_observed_at';
+};
+type SitemapGenerationReadResult = {
+  _tag: 'available';
+  manifest: SitemapSiteGenerationManifest;
+} | {
+  _tag: 'unavailable';
+  reason: 'no_generation' | 'generation_not_found';
+};
+interface IterateSitemapGenerationUrlsOptions {
+  generationId?: string;
+  feedpath?: string;
+}
+type IterateSitemapGenerationUrlsResult = {
+  _tag: 'iterator';
+  generationId: string;
+  observedAt: number;
+  items: AsyncIterable<SitemapUrlRecord>;
+  completeness: {
+    _tag: 'complete';
+  };
+  membershipHistoryAvailableFrom: number | null;
+} | {
+  _tag: 'invalid_request';
+  reason: 'invalid_feed';
+} | {
+  _tag: 'unavailable';
+  reason: 'no_generation' | 'generation_not_found';
+};
+interface ListSitemapGenerationUrlsOptions {
+  generationId?: string;
+  feedpath?: string;
+  cursor?: string;
+  limit?: number;
+}
+type ListSitemapGenerationUrlsResult = {
+  _tag: 'page';
+  generationId: string;
+  observedAt: number;
+  items: SitemapUrlRecord[];
+  nextCursor: string | null;
+  completeness: {
+    _tag: 'complete';
+  };
+  membershipHistoryAvailableFrom: number | null;
+} | {
+  _tag: 'invalid_request';
+  reason: 'invalid_limit' | 'invalid_cursor' | 'invalid_feed';
+} | {
+  _tag: 'unavailable';
+  reason: 'no_generation' | 'generation_not_found';
+};
+type SitemapMembershipEvidence = {
+  _tag: 'present';
+  url: string;
+  feedpath: string;
+  lastmod: string | null;
+  firstSeenAt: number;
+  lastSeenAt: number;
+} | {
+  _tag: 'absent';
+  url: string;
+  observedAt: number;
+} | {
+  _tag: 'unknown';
+  url: string;
+  reason: 'no_generation' | 'generation_not_found';
+};
+type QuerySitemapGenerationMembershipResult = {
+  _tag: 'result';
+  generationId: string;
+  observedAt: number;
+  evidence: SitemapMembershipEvidence[];
+  completeness: {
+    _tag: 'complete';
+  };
+  membershipHistoryAvailableFrom: number | null;
+} | {
+  _tag: 'unavailable';
+  reason: 'no_generation' | 'generation_not_found';
+};
+type RecordSitemapLegacyImportResult = {
+  _tag: 'recorded';
+  evidence: Extract<SitemapLegacyImportEvidence, {
+    _tag: 'metadata_only';
+  }>;
+} | {
+  _tag: 'existing';
+  evidence: Extract<SitemapLegacyImportEvidence, {
+    _tag: 'metadata_only';
+  }>;
+} | {
+  _tag: 'invalid';
+  reason: 'invalid_imported_at' | 'invalid_record_count';
+};
 interface CompleteSitemapGeneration {
   _tag: 'complete';
   id: string;
-  /** Unix epoch milliseconds. */
   observedAt: number;
 }
-interface ReconcileResult {
-  /** Feedpaths that were absent from the live set and had their live URLs pruned. */
-  feedpathsPruned: number;
-  /** Total URL rows transitioned live → removed across pruned feedpaths. */
-  urlsRemoved: number;
-}
-/**
- * Bounds on a single `compactUrls` call. `compactUrls` is memory-bounded (one
- * feedpath at a time) but was previously unbounded in TIME: a tenant with a
- * large accumulated delta backlog could run past a caller's execution budget.
- * Both bounds are checked BETWEEN feedpaths and only after at least one has
- * been compacted, so a call always makes forward progress.
- */
-interface CompactUrlsOptions {
-  /** Stop before starting another feedpath once this many ms have elapsed. */
-  deadlineMs?: number;
-  /** Stop after compacting this many feedpaths. */
-  maxFeedpaths?: number;
-}
-interface CompactUrlsResult {
-  /** Feedpaths whose active deltas were folded and retired by this call. */
-  compactedFeedpaths: number;
-  /**
-   * Feedpaths that still hold outstanding deltas. Call `compactUrls` again to
-   * continue. No cursor is needed because the projection watermark excludes a
-   * compacted feedpath's grace-retained deltas from the next call.
-   */
-  remainingFeedpaths: number;
-}
-interface DeltaEntry {
-  feedpath: string;
-  feedpathHash: string;
-  urlHash: string;
-  op: 'added' | 'removed';
-  loc: string;
-  lastmod?: string;
-  at: number;
-}
 interface SitemapMembershipEvent {
   feedpath: string;
   feedpathHash: string;
   urlHash: string;
-  op: 'added' | 'removed';
+  op: 'added' | 'removed' | 'updated';
   loc: string;
   lastmod?: string;
+  previousLastmod?: string;
   generationId: string;
-  /** Unix epoch milliseconds. */
   observedAt: number;
-  /** Stable ordering within one generation. */
   sequence: number;
-  /** Whether this row also mutates the disposable current-state projection. */
   projectsState: boolean;
 }
 interface DateRange {
-  /** YYYY-MM-DD inclusive. */
   from?: string;
-  /** YYYY-MM-DD inclusive. */
   to?: string;
 }
-interface LoadUrlsOptions {
-  includeRemoved?: boolean;
-}
 interface SitemapReadStore {
-  /** Load the full site index (latest record per feedpath). */
-  loadIndex: (ctx: TenantCtx) => Promise<SitemapIndex>;
-  /** Fetch the latest snapshot for a feedpath, or undefined. */
-  getLatest: (ctx: TenantCtx, path: string) => Promise<SitemapRecord | undefined>;
-  /** Stream live (and optionally removed) URL rows for a feedpath. */
-  loadUrls: (ctx: TenantCtx, feedpath: string, opts?: LoadUrlsOptions) => AsyncIterable<SitemapUrlRecord>;
-  /** Stream all delta entries within `[from, to]` (YYYY-MM-DD inclusive). */
-  loadDeltas: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<DeltaEntry>;
-  /** Stream immutable membership events within `[from, to]`. */
   loadEvents: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<SitemapMembershipEvent>;
+  getSitemapGeneration: (ctx: TenantCtx, generationId?: string) => Promise<SitemapGenerationReadResult>;
+  iterateSitemapGenerationUrls: (ctx: TenantCtx, opts?: IterateSitemapGenerationUrlsOptions) => Promise<IterateSitemapGenerationUrlsResult>;
+  listSitemapGenerationUrls: (ctx: TenantCtx, opts?: ListSitemapGenerationUrlsOptions) => Promise<ListSitemapGenerationUrlsResult>;
+  querySitemapGenerationMembership: (ctx: TenantCtx, opts: {
+    generationId?: string;
+    urls: readonly string[];
+  }) => Promise<QuerySitemapGenerationMembershipResult>;
 }
 interface SitemapStore extends SitemapReadStore {
-  /**
-   * Persist a snapshot run. Updates the index + writes one immutable
-   * history doc per record under `history/<feedpathHash>__<capturedAtMs>.json`.
-   */
-  writeSnapshot: (ctx: TenantCtx, records: readonly SitemapRecord[]) => Promise<void>;
-  /**
-   * Diff a complete sitemap generation against current state. The immutable
-   * membership event lands before its disposable state delta.
-   */
-  snapshotUrls: (ctx: TenantCtx, generation: CompleteSitemapGeneration, feedpath: string, urls: readonly ParsedUrl[]) => Promise<SnapshotUrlsResult>;
-  /**
-   * Fold accumulated deltas into the prior index, one feedpath at a time:
-   * rewrites each touched feedpath's `by-feed/<hash>/index.parquet` and deletes
-   * the consumed delta files. Bounded per feedpath, so it stays within memory
-   * regardless of total site URL count.
-   *
-   * Optionally bounded in TIME too (`opts.deadlineMs` / `opts.maxFeedpaths`).
-   * A bounded call is safe to stop mid-way: each rewritten feedpath advances
-   * the projection watermark, so the remainder is simply what the next call
-   * finds. Callers drive this from `remainingFeedpaths`, not a cursor.
-   */
-  compactUrls: (ctx: TenantCtx, opts?: CompactUrlsOptions) => Promise<CompactUrlsResult>;
-  /**
-   * Site-wide convergence: mark every still-live URL whose owning feedpath is
-   * absent from `liveFeedpaths` as removed. `compactUrls`/`snapshotUrls` only
-   * prune URLs *inside* a feedpath that was re-observed; a whole feed dropped
-   * from the sitemap list (no `snapshotUrls` call) leaves its URLs frozen-live
-   * forever. This is the sidecar mirror of the D1 generation sweep: it rewrites
-   * each dropped feedpath's `by-feed/<hash>/index.parquet` with `removedAt` set,
-   * advances its projection watermark, and grace-retires outstanding deltas.
-   * Bounded per feedpath, so memory stays flat regardless of site size. Live
-   * feedpaths are never touched.
-   */
-  reconcile: (ctx: TenantCtx, generation: CompleteSitemapGeneration, opts: {
-    liveFeedpaths: readonly string[];
-  }) => Promise<ReconcileResult>;
+  stageSitemapGenerationFeed: (ctx: TenantCtx, generation: CompleteSitemapGeneration, feedpath: string, urls: readonly ParsedUrl[]) => Promise<StageSitemapGenerationFeedResult>;
+  finalizeSitemapGeneration: (ctx: TenantCtx, generation: CompleteSitemapGeneration, traversal: CompleteSitemapTraversal) => Promise<FinalizeSitemapGenerationResult>;
+  abortSitemapGeneration: (ctx: TenantCtx, generation: CompleteSitemapGeneration) => Promise<void>;
+  recordSitemapLegacyImport: (ctx: TenantCtx, input: {
+    importedAt: number;
+    recordCount: number;
+  }) => Promise<RecordSitemapLegacyImportResult>;
 }
 interface CreateSitemapReadStoreOptions {
   dataSource: DataSource;
-  /** Override the feedpath hash (test seam). */
-  hash?: (path: string) => string;
 }
-type SitemapMutation = <T>(ctx: TenantCtx, fn: () => Promise<T>) => Promise<T>;
+type SitemapMutation = <T>(ctx: TenantCtx, effect: () => Promise<T>) => Promise<T>;
 interface CreateSitemapStoreOptions extends CreateSitemapReadStoreOptions {
   withMutation: SitemapMutation;
   now?: () => number;
 }
-export { CompactUrlsOptions, CompactUrlsResult, CompleteSitemapGeneration, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, LoadUrlsOptions, ParsedUrl, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapMembershipEvent, SitemapMutation, SitemapReadStore, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult };
+export { CompleteSitemapGeneration, CompleteSitemapTraversal, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, FinalizeSitemapGenerationResult, IterateSitemapGenerationUrlsOptions, IterateSitemapGenerationUrlsResult, ListSitemapGenerationUrlsOptions, ListSitemapGenerationUrlsResult, ParsedUrl, QuerySitemapGenerationMembershipResult, RecordSitemapLegacyImportResult, SitemapGenerationFeedManifest, SitemapGenerationReadResult, SitemapLegacyImportEvidence, SitemapMembershipEvent, SitemapMembershipEvidence, SitemapMutation, SitemapReadStore, SitemapSiteGenerationManifest, SitemapStagedFeed, SitemapStore, SitemapUrlRecord, StageSitemapGenerationFeedResult };

package/dist/entities/sitemap-shared.mjs

@@ -1,8 +1,3 @@
-import { readOptional } from "../adapters/read-optional.mjs";
-import { decodeParquetToRows } from "../adapters/hyparquet.mjs";
-import { mapEntityIo } from "./io.mjs";
-const SITEMAP_URLS_EVENT_PREFIX_RE = /\/urls\/events\/(\d{4}-\d{2}-\d{2})__[0-9a-f]+__\d+__[0-9a-f]+\.parquet$/;
-const SITEMAP_URLS_PENDING_GENERATION_RE = /\/urls\/generations\/pending\/([0-9a-f]+)\.json$/;
 const URLS_INDEX_COLUMNS = [
 	{
 		name: "feedpath",
@@ -45,7 +40,7 @@ const URLS_INDEX_COLUMNS = [
 		nullable: true
 	}
 ];
-const URLS_DELTA_COLUMNS = [
+const URLS_EVENT_COLUMNS = [
 	{
 		name: "feedpath",
 		type: "VARCHAR",
@@ -77,44 +72,7 @@ const URLS_DELTA_COLUMNS = [
 		nullable: true
 	},
 	{
-		name: "at",
-		type: "BIGINT",
-		nullable: false
-	},
-	{
-		name: "generation_id",
-		type: "VARCHAR",
-		nullable: true
-	}
-];
-const URLS_EVENT_COLUMNS = [
-	{
-		name: "feedpath",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "feedpath_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "url_hash",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "op",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "loc",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "lastmod",
+		name: "previous_lastmod",
 		type: "VARCHAR",
 		nullable: true
 	},
@@ -137,26 +95,6 @@ const URLS_EVENT_COLUMNS = [
 		name: "projects_state",
 		type: "INTEGER",
 		nullable: false
-	},
-	{
-		name: "seed_generation",
-		type: "INTEGER",
-		nullable: false
-	},
-	{
-		name: "generation_kind",
-		type: "VARCHAR",
-		nullable: false
-	},
-	{
-		name: "content_hash",
-		type: "VARCHAR",
-		nullable: true
-	},
-	{
-		name: "input_count",
-		type: "INTEGER",
-		nullable: true
 	}
 ];
 function rowToUrlRecord(row) {
@@ -171,73 +109,16 @@ function rowToUrlRecord(row) {
 		removedAt: row.removed_at == null ? void 0 : Number(row.removed_at)
 	};
 }
-function urlRecordToRow(r) {
+function urlRecordToRow(record) {
 	return {
-		feedpath: r.feedpath,
-		feedpath_hash: r.feedpathHash,
-		url_hash: r.urlHash,
-		loc: r.loc,
-		lastmod: r.lastmod ?? null,
-		first_seen_at: r.firstSeenAt,
-		last_seen_at: r.lastSeenAt,
-		removed_at: r.removedAt ?? null
-	};
-}
-function createSitemapUrlState(indexRows) {
-	const state = {
-		live: /* @__PURE__ */ new Map(),
-		removed: /* @__PURE__ */ new Map()
+		feedpath: record.feedpath,
+		feedpath_hash: record.feedpathHash,
+		url_hash: record.urlHash,
+		loc: record.loc,
+		lastmod: record.lastmod ?? null,
+		first_seen_at: record.firstSeenAt,
+		last_seen_at: record.lastSeenAt,
+		removed_at: record.removedAt ?? null
 	};
-	for (const row of indexRows) {
-		const record = rowToUrlRecord(row);
-		if (record.removedAt == null) state.live.set(record.urlHash, record);
-		else state.removed.set(record.urlHash, record);
-	}
-	return state;
-}
-function applySitemapDeltaRows(state, rows) {
-	for (const row of rows) {
-		const urlHash = String(row.url_hash);
-		const observedAt = Number(row.at);
-		const op = String(row.op);
-		if (op === "added") {
-			const previous = state.live.get(urlHash) ?? state.removed.get(urlHash);
-			state.removed.delete(urlHash);
-			state.live.set(urlHash, {
-				feedpath: String(row.feedpath),
-				feedpathHash: String(row.feedpath_hash),
-				urlHash,
-				loc: String(row.loc),
-				lastmod: row.lastmod == null ? void 0 : String(row.lastmod),
-				firstSeenAt: previous?.firstSeenAt ?? observedAt,
-				lastSeenAt: observedAt
-			});
-		} else if (op === "removed") {
-			const previous = state.live.get(urlHash);
-			state.live.delete(urlHash);
-			if (previous) state.removed.set(urlHash, {
-				...previous,
-				removedAt: observedAt
-			});
-		}
-	}
-}
-function applySitemapDeltaFiles(state, files) {
-	for (const file of files) applySitemapDeltaRows(state, file.rows);
-}
-function sortSitemapDeltaFiles(files) {
-	return files.sort((a, b) => a.key.localeCompare(b.key));
-}
-async function readSitemapDeltaFiles(ds, keys) {
-	return sortSitemapDeltaFiles((await mapEntityIo(keys, async (key) => {
-		const bytes = await readOptional(ds, key);
-		return bytes ? {
-			key,
-			rows: await decodeParquetToRows(bytes)
-		} : void 0;
-	})).filter((file) => file !== void 0));
-}
-function dateInRange(date, range) {
-	return (!range?.from || date >= range.from) && (!range?.to || date <= range.to);
 }
-export { SITEMAP_URLS_EVENT_PREFIX_RE, SITEMAP_URLS_PENDING_GENERATION_RE, URLS_DELTA_COLUMNS, URLS_EVENT_COLUMNS, URLS_INDEX_COLUMNS, applySitemapDeltaFiles, applySitemapDeltaRows, createSitemapUrlState, dateInRange, readSitemapDeltaFiles, rowToUrlRecord, sortSitemapDeltaFiles, urlRecordToRow };
+export { URLS_EVENT_COLUMNS, URLS_INDEX_COLUMNS, rowToUrlRecord, urlRecordToRow };