@gscdump/engine 1.4.11 → 1.5.0

package/dist/entities/empty-types.d.mts

@@ -0,0 +1,22 @@
+import { DataSource } from "../storage.mjs";
+import { TenantCtx } from "@gscdump/contracts";
+interface EmptyTypesDoc {
+  version: 1;
+  /** SearchType strings detected as empty for this (user, site). */
+  emptyTypes: string[];
+  /** When each type was last marked empty (unix ms). Helps debug stale skips. */
+  markedAt: Record<string, number>;
+}
+interface EmptyTypesStore {
+  load: (ctx: TenantCtx) => Promise<EmptyTypesDoc>;
+  /** Add types to the empty set, preserving existing markers. No-op if all already present. */
+  mark: (ctx: TenantCtx, types: readonly string[], now?: number) => Promise<EmptyTypesDoc>;
+  /** Remove types from the empty set. Returns the updated doc. */
+  clear: (ctx: TenantCtx, types: readonly string[]) => Promise<EmptyTypesDoc>;
+}
+interface CreateEmptyTypesStoreOptions {
+  dataSource: DataSource;
+  now?: () => number;
+}
+declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
+export { CreateEmptyTypesStoreOptions, EmptyTypesDoc, EmptyTypesStore, createEmptyTypesStore };

package/dist/entities/empty-types.mjs

@@ -0,0 +1,58 @@
+import { emptyTypesKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+function createEmptyTypesStore(opts) {
+	const ds = opts.dataSource;
+	const now = opts.now ?? (() => Date.now());
+	async function readDoc(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return {
+			version: 1,
+			emptyTypes: [],
+			markedAt: {}
+		};
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	async function writeDoc(key, doc) {
+		await ds.write(key, encodeJsonBigintSafe(doc));
+	}
+	return {
+		async load(ctx) {
+			return readDoc(emptyTypesKey(ctx));
+		},
+		async mark(ctx, types, at) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const stamp = at ?? now();
+			let changed = false;
+			for (const t of types) {
+				if (!doc.emptyTypes.includes(t)) {
+					doc.emptyTypes.push(t);
+					changed = true;
+				}
+				if (doc.markedAt[t] === void 0) {
+					doc.markedAt[t] = stamp;
+					changed = true;
+				}
+			}
+			if (changed) {
+				doc.emptyTypes.sort();
+				await writeDoc(key, doc);
+			}
+			return doc;
+		},
+		async clear(ctx, types) {
+			if (types.length === 0) return readDoc(emptyTypesKey(ctx));
+			const key = emptyTypesKey(ctx);
+			const doc = await readDoc(key);
+			const drop = new Set(types);
+			const before = doc.emptyTypes.length;
+			doc.emptyTypes = doc.emptyTypes.filter((t) => !drop.has(t));
+			for (const t of drop) delete doc.markedAt[t];
+			if (doc.emptyTypes.length !== before) await writeDoc(key, doc);
+			return doc;
+		}
+	};
+}
+export { createEmptyTypesStore };

package/dist/entities/indexing-metadata.d.mts

@@ -0,0 +1,26 @@
+import { DataSource } from "../storage.mjs";
+import { TenantCtx } from "@gscdump/contracts";
+interface IndexingMetadataRecord {
+  url: string;
+  capturedAt: string;
+  /** ISO-8601 notifyTime of the latest `URL_UPDATED` notification we've seen. */
+  latestUpdateAt?: string;
+  /** ISO-8601 notifyTime of the latest `URL_REMOVED` notification we've seen. */
+  latestRemoveAt?: string;
+  raw?: unknown;
+}
+interface IndexingMetadataIndex {
+  version: 1;
+  records: Record<string, IndexingMetadataRecord>;
+}
+interface IndexingMetadataStore {
+  writeBatch: (ctx: TenantCtx, records: readonly IndexingMetadataRecord[]) => Promise<void>;
+  loadIndex: (ctx: TenantCtx) => Promise<IndexingMetadataIndex>;
+  getLatest: (ctx: TenantCtx, url: string) => Promise<IndexingMetadataRecord | undefined>;
+}
+interface CreateIndexingMetadataStoreOptions {
+  dataSource: DataSource;
+  hash?: (url: string) => string;
+}
+declare function createIndexingMetadataStore(opts: CreateIndexingMetadataStoreOptions): IndexingMetadataStore;
+export { CreateIndexingMetadataStoreOptions, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, createIndexingMetadataStore };

package/dist/entities/indexing-metadata.mjs

@@ -0,0 +1,31 @@
+import { hashUrl, indexingMetadataIndexKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+function createIndexingMetadataStore(opts) {
+	const ds = opts.dataSource;
+	const hash = opts.hash ?? hashUrl;
+	async function readIndex(key) {
+		const bytes = await readOptional(ds, key);
+		if (bytes === void 0) return {
+			version: 1,
+			records: {}
+		};
+		return JSON.parse(new TextDecoder().decode(bytes));
+	}
+	return {
+		async writeBatch(ctx, records) {
+			if (records.length === 0) return;
+			const key = indexingMetadataIndexKey(ctx);
+			const index = await readIndex(key);
+			for (const r of records) index.records[hash(r.url)] = r;
+			await ds.write(key, encodeJsonBigintSafe(index));
+		},
+		async loadIndex(ctx) {
+			return readIndex(indexingMetadataIndexKey(ctx));
+		},
+		async getLatest(ctx, url) {
+			return (await readIndex(indexingMetadataIndexKey(ctx))).records[hash(url)];
+		}
+	};
+}
+export { createIndexingMetadataStore };

package/dist/entities/inspection.d.mts

@@ -0,0 +1,240 @@
+import { DataSource } from "../storage.mjs";
+import { ScheduleState } from "../schedule.mjs";
+import { TenantCtx } from "@gscdump/contracts";
+import { CanonicalDifferenceKind } from "gscdump";
+/**
+ * GSC URL inspection result fields we persist. Mirrors the
+ *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON
+ *  so storage doesn't depend on the googleapis type tree.
+ */
+interface InspectionRecord {
+  url: string;
+  /** ISO-8601 timestamp of when we ran the inspection. */
+  inspectedAt: string;
+  /** PASS / NEUTRAL / FAIL — the headline verdict from indexStatusResult. */
+  indexStatus?: string;
+  /** Last-crawl timestamp the API reports (ISO-8601). */
+  lastCrawlTime?: string;
+  /** Canonical URL Google selected. */
+  googleCanonical?: string;
+  /** Canonical URL the page declares. */
+  userCanonical?: string;
+  /** Crawl/index/serving disposition strings as the API returns them. */
+  coverageState?: string;
+  robotsTxtState?: string;
+  indexingState?: string;
+  pageFetchState?: string;
+  mobileUsabilityVerdict?: string;
+  richResultsVerdict?: string;
+  /**
+   * Free-form payload for fields we don't promote to first-class columns
+   *  (e.g. `referringUrls`, `crawledAs`). Keeps the wire format forward-compat
+   *  without bumping the schema for every API addition.
+   *
+   *  Recognised keys:
+   *  - `schedule`: optional `ScheduleState` from {@link inspectionPolicy}
+   *    governing when this URL is next due for re-inspection. Undefined on
+   *    pre-§0 records — readers must tolerate the missing field and fall
+   *    back to default policy on first observe.
+   */
+  raw?: {
+    schedule?: ScheduleState;
+    [key: string]: unknown;
+  };
+}
+/** Wire shape persisted to disk/R2. */
+interface InspectionIndex {
+  version: 1;
+  /** Map of urlHash → InspectionRecord (latest only). */
+  records: Record<string, InspectionRecord>;
+}
+/**
+ * Append-only history shard, one blob per `appendHistory` call.
+ * Keyed by UUID under the month directory — retries write a new blob,
+ * never RMW an existing one. Idempotent under job retries.
+ */
+interface InspectionHistoryShard {
+  version: 1;
+  /** Records persisted in this batch. */
+  records: InspectionRecord[];
+}
+/**
+ * Directory prefix for a month's history shards. Each shard is a UUID-keyed
+ * blob under this prefix; `appendHistory` writes one per call, `loadHistory`
+ * lists + concatenates.
+ */
+/**
+ * Row shape for the inspections parquet sidecar. Caller-side schema for
+ * `materialize` — D1 is the source of truth in the 2026-05-19 redesign, so
+ * consumers stream rows from `url_indexing_status` and pass them in. The
+ * parquet sidecar exists for DuckDB JOIN seams; readers go through
+ * `parquetUri`.
+ */
+interface InspectionParquetRow {
+  [column: string]: string | number | null;
+  urlHash: string;
+  url: string;
+  inspectedAt: string;
+  indexStatus: string | null;
+  lastCrawlTime: string | null;
+  googleCanonical: string | null;
+  userCanonical: string | null;
+  coverageState: string | null;
+  robotsTxtState: string | null;
+  indexingState: string | null;
+  pageFetchState: string | null;
+  mobileUsabilityVerdict: string | null;
+  richResultsVerdict: string | null;
+  scheduleNextAt: number | null;
+  scheduleConsecutiveUnchanged: number | null;
+  schedulePolicyVersion: number | null;
+}
+/**
+ * Row shape for the append-only inspection-event store. Superset of
+ * {@link InspectionParquetRow}: carries the full-fidelity columns the lossy
+ * `materialize` parquet dropped (`crawlingUserAgent`, `richResultsItems`,
+ * `sitemaps`, `referringUrls`, `mobileIssues`, `inspectionResultLink`,
+ * `firstCheckedAt`, `checkCount`). Object/array fields are persisted as JSON
+ * strings — read paths unpack them with DuckDB's JSON functions.
+ *
+ * `firstCheckedAt` / `checkCount` are caller-managed: the writer carries the
+ * earliest-seen timestamp + running observation count forward. Compaction
+ * preserves the EARLIEST `firstCheckedAt` per url (mirrors the sitemap store's
+ * `firstSeenAt` preservation); every other column is taken from the
+ * newest-by-`inspectedAt` event.
+ */
+interface InspectionEventRow extends InspectionParquetRow {
+  /** Declared-vs-selected canonical classification, computed at inspection ingest. */
+  canonicalMismatchKind: CanonicalDifferenceKind;
+  crawlingUserAgent: string | null;
+  /** JSON-encoded `RichResultsItem[]`. */
+  richResultsItems: string | null;
+  /** JSON-encoded list of sitemap URLs referencing this page. */
+  sitemaps: string | null;
+  /** JSON-encoded list of referring URLs. */
+  referringUrls: string | null;
+  /** JSON-encoded mobile-usability issues. */
+  mobileIssues: string | null;
+  inspectionResultLink: string | null;
+  /** ISO-8601 timestamp of the first inspection we ever recorded for this url. */
+  firstCheckedAt: string | null;
+  /** Total number of inspections recorded for this url. */
+  checkCount: number | null;
+  /**
+   * Stored next-recheck unix-seconds + priority as computed AT INSPECT TIME.
+   * Carried verbatim (NOT recomputed at read) because the scheduling policy can
+   * change over time — `__gsc/inspections` must replay the historical value to
+   * keep its frozen wire shape byte-stable.
+   */
+  nextCheckAfter: number | null;
+  nextCheckPriority: string | null;
+}
+/**
+ * Hard cap on a single `appendHistory` shard payload. Encoded bytes >
+ * this threshold throws — the caller logs and moves on (D1 is
+ * authoritative, R2 history is a sidecar). At `URLS_PER_JOB=3` a real
+ * batch encodes to ~10 KB so the cap is purely defensive against future
+ * batch-size bumps.
+ */
+declare const INSPECTION_HISTORY_MAX_BYTES: number;
+interface InspectionStore {
+  /**
+   * Append a batch of fresh inspection results as an immutable per-batch
+   * shard under `history/<YYYY-MM>/<batchId>.json`. Idempotent under job
+   * retry (caller-supplied UUID per logical batch), no read-before-write,
+   * one PUT per month-group within the batch.
+   *
+   * Throws if the encoded payload exceeds {@link INSPECTION_HISTORY_MAX_BYTES}.
+   */
+  appendHistory: (ctx: TenantCtx, records: readonly InspectionRecord[], opts?: {
+    batchId?: string;
+  }) => Promise<void>;
+  /**
+   * Read every shard in a month directory and concatenate. Best-effort:
+   * shards that fail to decode are skipped (logged via console). Returns
+   * `undefined` if the month has no shards.
+   */
+  loadHistory: (ctx: TenantCtx, yearMonth: string) => Promise<InspectionHistoryShard | undefined>;
+  /**
+   * Encode caller-provided rows into the inspections parquet sidecar at
+   * `entities/inspections/index.parquet`. Sorted by `urlHash` so DuckDB
+   * row-group stats can prune URL-keyed JOINs efficiently. One PUT.
+   *
+   * D1 is the source of truth in the 2026-05-19 redesign; this rebuilds
+   * the parquet from D1 rows the caller streams in (engine has no D1
+   * access). Triggered by `indexing/complete` post-hook.
+   *
+   * Returns the parquet object key (matches {@link parquetUri} after write).
+   */
+  materialize: (ctx: TenantCtx, rows: Iterable<InspectionParquetRow>) => Promise<{
+    key: string;
+    rowCount: number;
+    bytes: number;
+  }>;
+  /**
+   * Append a batch of inspection results as an immutable per-batch parquet
+   * under `events/<YYYY-MM>/<batchId>.parquet`, partitioned by the `YYYY-MM`
+   * of each row's `inspectedAt` (a batch spanning a month boundary writes one
+   * file per month). No read-before-write; idempotent under job retry (same
+   * `batchId` → same key → whole-file overwrite). Rows carry the FULL column
+   * set ({@link INSPECTION_EVENT_COLUMNS}); this is the append-only
+   * source-of-truth that supersedes {@link InspectionStore.materialize}.
+   *
+   * Returns the keys written + total row count. Empty input is a no-op.
+   */
+  appendInspectionEvents: (ctx: TenantCtx, rows: readonly InspectionEventRow[], opts?: {
+    batchId?: string;
+  }) => Promise<{
+    keys: string[];
+    rowCount: number;
+  }>;
+  /**
+   * Fold every outstanding event file into the `base.parquet`: latest-per-url
+   * by max `inspectedAt` (newest-wins), preserving the earliest non-null
+   * `firstCheckedAt` per url. Writes the new base then deletes the consumed
+   * event files — file-level only, never row-level (ADR-0002). Idempotent +
+   * re-runnable: a crash after the base write but before the delete just
+   * re-folds the same events (newest-wins makes that a no-op). A real read
+   * failure on the existing base propagates rather than rebuilding from events
+   * alone (which would drop URLs only the base held).
+   *
+   * No-op (no base rewrite) when there are zero outstanding events.
+   */
+  compactInspections: (ctx: TenantCtx, opts?: {
+    /**
+     * Also record state changes into the durable transitions sidecar.
+     * Default OFF so publishing this is inert until a canary opts in.
+     */
+    transitions?: boolean;
+  }) => Promise<{
+    baseRowCount: number;
+    eventsFolded: number;
+    eventFilesDeleted: number;
+    transitionsWritten: number;
+  }>;
+  /**
+   * Rewrite a legacy latest-only base with the canonical kind derived from the
+   * canonical pair it already retains. Outstanding events must be compacted first.
+   */
+  backfillCanonicalMismatchKinds: (ctx: TenantCtx) => Promise<{
+    baseRowCount: number;
+    rowsBackfilled: number;
+    rewritten: boolean;
+  }>;
+  /**
+   * DuckDB-resolvable URI for the materialised parquet sidecar, or
+   * `undefined` if the underlying `DataSource` has no native URI shape
+   * (in-memory tests). When defined, read paths can `read_parquet(<uri>)`
+   * directly without staging bytes through JS.
+   *
+   * Does not check existence — caller is responsible for ensuring
+   * `materialize` has run at least once. Returning a URI for a missing key
+   * is safe; DuckDB will surface a 404 / not-found at query time.
+   */
+  parquetUri: (ctx: TenantCtx) => string | undefined;
+}
+interface CreateInspectionStoreOptions {
+  dataSource: DataSource;
+}
+declare function createInspectionStore(opts: CreateInspectionStoreOptions): InspectionStore;
+export { CreateInspectionStoreOptions, INSPECTION_HISTORY_MAX_BYTES, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, createInspectionStore };