@gscdump/engine 0.26.1 → 0.26.3

package/dist/_chunks/entities.mjs

@@ -26,6 +26,16 @@ function emptyTypesKey(ctx) {
 function inspectionParquetKey(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/index.parquet` : `u_${ctx.userId}/entities/inspections/index.parquet`;
 }
+function inspectionEventsPrefix(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/events` : `u_${ctx.userId}/entities/inspections/events`;
+}
+function inspectionEventKey(ctx, yearMonth, batchId) {
+	return `${inspectionEventsPrefix(ctx)}/${yearMonth}/${batchId}.parquet`;
+}
+function inspectionBaseKey(ctx) {
+	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/base.parquet` : `u_${ctx.userId}/entities/inspections/base.parquet`;
+}
+const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
 function inspectionHistoryPrefix(ctx, yearMonth) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/inspections/history/${yearMonth}` : `u_${ctx.userId}/entities/inspections/history/${yearMonth}`;
 }
@@ -129,6 +139,49 @@ const INSPECTION_PARQUET_COLUMNS = [
 		nullable: true
 	}
 ];
+const INSPECTION_EVENT_COLUMNS = [
+	...INSPECTION_PARQUET_COLUMNS,
+	{
+		name: "crawlingUserAgent",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "richResultsItems",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "sitemaps",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "referringUrls",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "mobileIssues",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "inspectionResultLink",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "firstCheckedAt",
+		type: "VARCHAR",
+		nullable: true
+	},
+	{
+		name: "checkCount",
+		type: "INTEGER",
+		nullable: true
+	}
+];
 function createInspectionStore(opts) {
 	const ds = opts.dataSource;
 	function shardFor(record) {
@@ -193,6 +246,89 @@ function createInspectionStore(opts) {
 				bytes: bytes.byteLength
 			};
 		},
+		async appendInspectionEvents(ctx, rows, options) {
+			if (rows.length === 0) return {
+				keys: [],
+				rowCount: 0
+			};
+			const batchId = options?.batchId ?? randomBatchId();
+			const byMonth = /* @__PURE__ */ new Map();
+			for (const r of rows) {
+				const m = YEAR_MONTH_RE.exec(r.inspectedAt);
+				const month = m ? `${m[1]}-${m[2]}` : "unknown";
+				const bucket = byMonth.get(month) ?? [];
+				bucket.push(r);
+				byMonth.set(month, bucket);
+			}
+			const keys = [];
+			for (const [month, batch] of byMonth) {
+				const bytes = encodeRowsToParquetFlex(batch, {
+					columns: INSPECTION_EVENT_COLUMNS,
+					sortKey: ["urlHash"]
+				});
+				const key = inspectionEventKey(ctx, month, batchId);
+				await ds.write(key, bytes);
+				keys.push(key);
+			}
+			return {
+				keys,
+				rowCount: rows.length
+			};
+		},
+		async compactInspections(ctx) {
+			const eventKeys = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((k) => INSPECTION_EVENT_KEY_RE.test(k));
+			if (eventKeys.length === 0) return {
+				baseRowCount: 0,
+				eventsFolded: 0,
+				eventFilesDeleted: 0
+			};
+			const baseKey = inspectionBaseKey(ctx);
+			const baseBytes = await readOptional(ds, baseKey);
+			const baseRows = baseBytes ? await decodeParquetToRows(baseBytes) : [];
+			const latest = /* @__PURE__ */ new Map();
+			const earliestChecked = /* @__PURE__ */ new Map();
+			const consider = (row) => {
+				const h = String(row.urlHash);
+				const prev = latest.get(h);
+				if (!prev || String(row.inspectedAt ?? "") > String(prev.inspectedAt ?? "")) latest.set(h, row);
+				const fc = row.firstCheckedAt;
+				if (fc != null) {
+					const fcStr = String(fc);
+					const cur = earliestChecked.get(h);
+					if (cur === void 0 || fcStr < cur) earliestChecked.set(h, fcStr);
+				}
+			};
+			for (const row of baseRows) consider(row);
+			let eventsFolded = 0;
+			const consumed = [];
+			for (const key of eventKeys.sort()) {
+				const bytes = await readOptional(ds, key);
+				if (!bytes) continue;
+				consumed.push(key);
+				const rows = await decodeParquetToRows(bytes);
+				for (const row of rows) {
+					consider(row);
+					eventsFolded++;
+				}
+			}
+			const merged = [];
+			for (const [h, row] of latest) {
+				const fc = earliestChecked.get(h);
+				if (fc !== void 0) row.firstCheckedAt = fc;
+				merged.push(row);
+			}
+			const bytes = encodeRowsToParquetFlex(merged, {
+				columns: INSPECTION_EVENT_COLUMNS,
+				sortKey: ["urlHash"]
+			});
+			await ds.write(baseKey, bytes);
+			if (consumed.length > 0) await ds.delete(consumed);
+			return {
+				baseRowCount: merged.length,
+				eventsFolded,
+				eventFilesDeleted: consumed.length
+			};
+		},
 		parquetUri(ctx) {
 			return ds.uri?.(inspectionParquetKey(ctx));
 		}
@@ -661,4 +797,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/_chunks/schema.d.mts

@@ -1,4 +1,4 @@
-import { ColumnDef, ColumnType, Row, TableName, TableSchema, TableSchema as TableSchema$1 } from "@gscdump/contracts";
+import { ColumnDef as ColumnDef$1, ColumnType, Row, TableName, TableSchema, TableSchema as TableSchema$1 } from "@gscdump/contracts";
 declare const pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
   name: "pages";
   schema: undefined;
@@ -2259,4 +2259,4 @@ declare function naturalKeyColumns(table: TableName): readonly string[];
  */
 declare function dedupeByNaturalKey(table: TableName, rows: readonly Row[]): Row[];
 declare function dimensionToColumn(dim: string, _table: TableName): string;
-export { type ColumnDef, type ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema$1 as TableSchema, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+export { type ColumnDef$1 as ColumnDef, type ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema$1 as TableSchema, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/entities.d.mts

@@ -1,6 +1,6 @@
 import { DataSource } from "./_chunks/storage.mjs";
 import { ScheduleState } from "./schedule.mjs";
-import { TenantCtx } from "@gscdump/contracts";
+import { ColumnDef, TenantCtx } from "@gscdump/contracts";
 /**
  * GSC URL inspection result fields we persist. Mirrors the
  *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON
@@ -60,6 +60,16 @@ interface InspectionHistoryShard {
 declare function inspectionIndexKey(ctx: TenantCtx): string;
 declare function emptyTypesKey(ctx: TenantCtx): string;
 declare function inspectionParquetKey(ctx: TenantCtx): string;
+/** Directory prefix holding a tenant's immutable inspection-event parquets. */
+declare function inspectionEventsPrefix(ctx: TenantCtx): string;
+/**
+ * Object key for one immutable inspection-event batch, partitioned by the
+ * `YYYY-MM` of the records' `inspectedAt`. The `batchId` is caller-supplied so
+ * a job retry re-writes the SAME key (idempotent whole-file overwrite).
+ */
+declare function inspectionEventKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
+/** Compacted latest-per-url base produced by `compactInspections`. */
+declare function inspectionBaseKey(ctx: TenantCtx): string;
 /**
  * Directory prefix for a month's history shards. Each shard is a UUID-keyed
  * blob under this prefix; `appendHistory` writes one per call, `loadHistory`
@@ -99,6 +109,36 @@ interface InspectionParquetRow {
   scheduleConsecutiveUnchanged: number | null;
   schedulePolicyVersion: number | null;
 }
+/**
+ * Row shape for the append-only inspection-event store. Superset of
+ * {@link InspectionParquetRow}: carries the full-fidelity columns the lossy
+ * `materialize` parquet dropped (`crawlingUserAgent`, `richResultsItems`,
+ * `sitemaps`, `referringUrls`, `mobileIssues`, `inspectionResultLink`,
+ * `firstCheckedAt`, `checkCount`). Object/array fields are persisted as JSON
+ * strings — read paths unpack them with DuckDB's JSON functions.
+ *
+ * `firstCheckedAt` / `checkCount` are caller-managed: the writer carries the
+ * earliest-seen timestamp + running observation count forward. Compaction
+ * preserves the EARLIEST `firstCheckedAt` per url (mirrors the sitemap store's
+ * `firstSeenAt` preservation); every other column is taken from the
+ * newest-by-`inspectedAt` event.
+ */
+interface InspectionEventRow extends InspectionParquetRow {
+  crawlingUserAgent: string | null;
+  /** JSON-encoded `RichResultsItem[]`. */
+  richResultsItems: string | null;
+  /** JSON-encoded list of sitemap URLs referencing this page. */
+  sitemaps: string | null;
+  /** JSON-encoded list of referring URLs. */
+  referringUrls: string | null;
+  /** JSON-encoded mobile-usability issues. */
+  mobileIssues: string | null;
+  inspectionResultLink: string | null;
+  /** ISO-8601 timestamp of the first inspection we ever recorded for this url. */
+  firstCheckedAt: string | null;
+  /** Total number of inspections recorded for this url. */
+  checkCount: number | null;
+}
 /**
  * Hard cap on a single `appendHistory` shard payload. Encoded bytes >
  * this threshold throws — the caller logs and moves on (D1 is
@@ -141,6 +181,40 @@ interface InspectionStore {
     rowCount: number;
     bytes: number;
   }>;
+  /**
+   * Append a batch of inspection results as an immutable per-batch parquet
+   * under `events/<YYYY-MM>/<batchId>.parquet`, partitioned by the `YYYY-MM`
+   * of each row's `inspectedAt` (a batch spanning a month boundary writes one
+   * file per month). No read-before-write; idempotent under job retry (same
+   * `batchId` → same key → whole-file overwrite). Rows carry the FULL column
+   * set ({@link INSPECTION_EVENT_COLUMNS}); this is the append-only
+   * source-of-truth that supersedes {@link InspectionStore.materialize}.
+   *
+   * Returns the keys written + total row count. Empty input is a no-op.
+   */
+  appendInspectionEvents: (ctx: TenantCtx, rows: readonly InspectionEventRow[], opts?: {
+    batchId?: string;
+  }) => Promise<{
+    keys: string[];
+    rowCount: number;
+  }>;
+  /**
+   * Fold every outstanding event file into the `base.parquet`: latest-per-url
+   * by max `inspectedAt` (newest-wins), preserving the earliest non-null
+   * `firstCheckedAt` per url. Writes the new base then deletes the consumed
+   * event files — file-level only, never row-level (ADR-0002). Idempotent +
+   * re-runnable: a crash after the base write but before the delete just
+   * re-folds the same events (newest-wins makes that a no-op). A real read
+   * failure on the existing base propagates rather than rebuilding from events
+   * alone (which would drop URLs only the base held).
+   *
+   * No-op (no base rewrite) when there are zero outstanding events.
+   */
+  compactInspections: (ctx: TenantCtx) => Promise<{
+    baseRowCount: number;
+    eventsFolded: number;
+    eventFilesDeleted: number;
+  }>;
   /**
    * DuckDB-resolvable URI for the materialised parquet sidecar, or
    * `undefined` if the underlying `DataSource` has no native URI shape
@@ -156,6 +230,14 @@ interface InspectionStore {
 interface CreateInspectionStoreOptions {
   dataSource: DataSource;
 }
+/**
+ * Column schema for the append-only inspection-event store + its compacted
+ * base. Superset of {@link INSPECTION_PARQUET_COLUMNS}: the 16 promoted columns
+ * plus the 8 full-fidelity ones the lossy `materialize` parquet dropped. The
+ * event files and `base.parquet` share this schema so DuckDB
+ * `read_parquet([...], union_by_name = true)` merges base + events cleanly.
+ */
+declare const INSPECTION_EVENT_COLUMNS: readonly ColumnDef[];
 declare function createInspectionStore(opts: CreateInspectionStoreOptions): InspectionStore;
 /** GSC sitemap record we persist. Matches `Schema$WmxSitemap` but as plain JSON. */
 interface SitemapRecord {
@@ -331,4 +413,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/entities.mjs

@@ -1,2 +1,2 @@
-import { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
-export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+import { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix } from "./_chunks/entities.mjs";
+export { INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/rollups.d.mts

@@ -1,5 +1,5 @@
 import { DataSource, FileSetRef, Row as Row$1 } from "./_chunks/storage.mjs";
-import { ColumnDef } from "./_chunks/schema.mjs";
+import { ColumnDef as ColumnDef$1 } from "./_chunks/schema.mjs";
 import { EngineError } from "./_chunks/errors.mjs";
 import { SearchType } from "gscdump/query";
 import { TenantCtx } from "@gscdump/contracts";
@@ -70,7 +70,7 @@ interface RollupDef {
    * Types map the same way as the fact-table encoder: VARCHAR / DATE go
    * through BYTE_ARRAY/UTF8; BIGINT → INT64; INTEGER → INT32; DOUBLE → DOUBLE.
    */
-  parquetColumns?: readonly ColumnDef[];
+  parquetColumns?: readonly ColumnDef$1[];
   /** Sort-key column names for parquet row-group stats. Optional. */
   parquetSortKey?: readonly string[];
   /**

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.26.1",
+  "version": "0.26.3",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -190,8 +190,8 @@
     "drizzle-orm": "1.0.0-rc.3",
     "icebird": "^0.8.8",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.26.1",
-    "gscdump": "0.26.1"
+    "gscdump": "0.26.3",
+    "@gscdump/contracts": "0.26.3"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",