@gscdump/engine 1.4.7 → 1.4.10

package/dist/adapters/node-harness.d.mts

@@ -1,6 +1,6 @@
 import { DataSource, StorageEngine } from "../storage.mjs";
-import { SearchType } from "gscdump/query";
 import { Row, TableName, TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 interface NodeHarnessOptions {
   dataDir: string;
   /** Tenant user id. Defaults to `'local'` for single-user CLI installs. */

package/dist/entities.d.mts

@@ -4,6 +4,7 @@ import { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUr
 import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
 import { TenantCtx } from "@gscdump/contracts";
+import { CanonicalDifferenceKind } from "gscdump";
 /**
  * GSC URL inspection result fields we persist. Mirrors the
  *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON
@@ -106,6 +107,8 @@ interface InspectionParquetRow {
  * newest-by-`inspectedAt` event.
  */
 interface InspectionEventRow extends InspectionParquetRow {
+  /** Declared-vs-selected canonical classification, computed at inspection ingest. */
+  canonicalMismatchKind: CanonicalDifferenceKind;
   crawlingUserAgent: string | null;
   /** JSON-encoded `RichResultsItem[]`. */
   richResultsItems: string | null;
@@ -212,6 +215,15 @@ interface InspectionStore {
     eventFilesDeleted: number;
     transitionsWritten: number;
   }>;
+  /**
+   * Rewrite a legacy latest-only base with the canonical kind derived from the
+   * canonical pair it already retains. Outstanding events must be compacted first.
+   */
+  backfillCanonicalMismatchKinds: (ctx: TenantCtx) => Promise<{
+    baseRowCount: number;
+    rowsBackfilled: number;
+    rewritten: boolean;
+  }>;
   /**
    * DuckDB-resolvable URI for the materialised parquet sidecar, or
    * `undefined` if the underlying `DataSource` has no native URI shape

package/dist/entities.mjs

@@ -4,6 +4,8 @@ import { readOptional } from "./adapters/read-optional.mjs";
 import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
 import { buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+import { GSCDUMP_INDEXING_TRANSITION_FIELDS } from "@gscdump/contracts";
+import { classifyCanonicalDifference } from "gscdump";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 const ENTITY_IO_CONCURRENCY = 8;
 async function mapEntityIo(items, fn) {
@@ -106,6 +108,11 @@ const INSPECTION_PARQUET_COLUMNS = [
 ];
 const INSPECTION_EVENT_COLUMNS = [
 	...INSPECTION_PARQUET_COLUMNS,
+	{
+		name: "canonicalMismatchKind",
+		type: "VARCHAR",
+		nullable: false
+	},
 	{
 		name: "crawlingUserAgent",
 		type: "VARCHAR",
@@ -157,14 +164,19 @@ const INSPECTION_EVENT_COLUMNS = [
 		nullable: true
 	}
 ];
-const TRANSITION_STATE_FIELDS = [
-	"indexStatus",
-	"coverageState",
-	"robotsTxtState",
-	"indexingState",
-	"pageFetchState",
-	"googleCanonical"
-];
+const CANONICAL_DIFFERENCE_KINDS = /* @__PURE__ */ new Set([
+	"none",
+	"formatting",
+	"path",
+	"cross_domain"
+]);
+function populateCanonicalMismatchKind(row) {
+	const current = row.canonicalMismatchKind;
+	const expected = classifyCanonicalDifference(typeof row.userCanonical === "string" ? row.userCanonical : null, typeof row.googleCanonical === "string" ? row.googleCanonical : null);
+	const missingOrInvalid = typeof current !== "string" || !CANONICAL_DIFFERENCE_KINDS.has(current);
+	row.canonicalMismatchKind = expected;
+	return missingOrInvalid || current !== expected;
+}
 const INSPECTION_TRANSITION_COLUMNS = [
 	{
 		name: "urlHash",
@@ -186,7 +198,7 @@ const INSPECTION_TRANSITION_COLUMNS = [
 		type: "VARCHAR",
 		nullable: false
 	},
-	...TRANSITION_STATE_FIELDS.flatMap((field) => {
+	...GSCDUMP_INDEXING_TRANSITION_FIELDS.flatMap((field) => {
 		const capped = field[0].toUpperCase() + field.slice(1);
 		return [{
 			name: `from${capped}`,
@@ -200,7 +212,7 @@ const INSPECTION_TRANSITION_COLUMNS = [
 	})
 ];
 function isStateTransition(before, after) {
-	return TRANSITION_STATE_FIELDS.some((field) => (before[field] ?? null) !== (after[field] ?? null));
+	return GSCDUMP_INDEXING_TRANSITION_FIELDS.some((field) => (before[field] ?? null) !== (after[field] ?? null));
 }
 function buildTransitionRow(before, after) {
 	const row = {
@@ -209,7 +221,7 @@ function buildTransitionRow(before, after) {
 		changedAfter: String(before.inspectedAt ?? ""),
 		changedBefore: String(after.inspectedAt ?? "")
 	};
-	for (const field of TRANSITION_STATE_FIELDS) {
+	for (const field of GSCDUMP_INDEXING_TRANSITION_FIELDS) {
 		const capped = field[0].toUpperCase() + field.slice(1);
 		row[`from${capped}`] = before[field] ?? null;
 		row[`to${capped}`] = after[field] ?? null;
@@ -396,6 +408,7 @@ function createInspectionStore(opts) {
 			for (const [h, row] of latest) {
 				const fc = earliestChecked.get(h);
 				if (fc !== void 0) row.firstCheckedAt = fc;
+				populateCanonicalMismatchKind(row);
 				merged.push(row);
 			}
 			const bytes = encodeRowsToParquetFlex(merged, {
@@ -412,6 +425,32 @@ function createInspectionStore(opts) {
 				transitionsWritten
 			};
 		},
+		async backfillCanonicalMismatchKinds(ctx) {
+			const baseKey = inspectionBaseKey(ctx);
+			const baseBytes = await readOptional(ds, baseKey);
+			if (!baseBytes) return {
+				baseRowCount: 0,
+				rowsBackfilled: 0,
+				rewritten: false
+			};
+			const rows = await decodeParquetToRows(baseBytes);
+			let rowsBackfilled = 0;
+			for (const row of rows) if (populateCanonicalMismatchKind(row)) rowsBackfilled++;
+			if (rowsBackfilled === 0) return {
+				baseRowCount: rows.length,
+				rowsBackfilled: 0,
+				rewritten: false
+			};
+			await ds.write(baseKey, encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_EVENT_COLUMNS,
+				sortKey: ["urlHash"]
+			}));
+			return {
+				baseRowCount: rows.length,
+				rowsBackfilled,
+				rewritten: true
+			};
+		},
 		parquetUri(ctx) {
 			return ds.uri?.(inspectionParquetKey(ctx));
 		}

package/dist/layout.d.mts

@@ -1,6 +1,6 @@
 import { CompactionTier, ManifestEntry } from "./storage.mjs";
-import { SearchType } from "gscdump/query";
 import { TableName, TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 declare function dayPartition(date: string): string;
 /**
  * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet

package/dist/resolver/run-query.d.mts

@@ -1,7 +1,7 @@
 import { SearchType as SearchType$1, TableName as TableName$1 } from "../storage.mjs";
 import { ComparisonFilter } from "./types.mjs";
-import { BuilderState } from "gscdump/query";
 import { Grain } from "@gscdump/contracts";
+import { BuilderState } from "gscdump/query";
 interface RunQueryCtx {
   userId: string;
   siteId: string;

package/dist/rollups.d.mts

@@ -2,8 +2,8 @@ import { DataSource, FileSetRef, Row as Row$1, TableName as TableName$1 } from "
 import { ColumnDef } from "./schema.mjs";
 import { EngineError } from "./errors.mjs";
 import "./contracts.mjs";
-import { SearchType } from "gscdump/query";
 import { TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 interface RollupCtx extends TenantCtx {
   /** When the rollup was built. Stamped into payload + filename. */
   builtAt: number;

package/dist/rollups.mjs

@@ -978,7 +978,7 @@ const indexingHealthRollup = {
         SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'NOT_FOUND' THEN 1 ELSE 0 END)::BIGINT AS not_found,
         SUM(CASE WHEN CAST(mobileUsabilityVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS mobile_passes,
         SUM(CASE WHEN CAST(richResultsVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
-        SUM(CASE WHEN userCanonical IS NOT NULL AND googleCanonical IS NOT NULL AND CAST(userCanonical AS VARCHAR) <> CAST(googleCanonical AS VARCHAR) THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
+        SUM(CASE WHEN canonicalMismatchKind IN ('path', 'cross_domain') THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
       FROM read_parquet({{INSPECTIONS}}, union_by_name = true)
       WHERE substr(CAST(inspectedAt AS VARCHAR), 1, 10) >= '${utcDateMinusDays(windowAnchorMs, 90)}'
       GROUP BY 1

package/dist/source/source-types.d.mts

@@ -1,7 +1,7 @@
 import { ResolverAdapter } from "../resolver/types.mjs";
 import { PlannerCapabilities } from "gscdump/query/plan";
-import { BuilderState } from "gscdump/query";
 import { TableName } from "@gscdump/contracts";
+import { BuilderState } from "gscdump/query";
 type QueryRow = Record<string, unknown>;
 interface FileSet {
   table: TableName;

package/dist/storage.d.mts

@@ -1,8 +1,8 @@
 import { CompactionThresholds } from "./compaction.mjs";
 import { ParquetQueryFilter } from "./node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/index.mjs";
 import "./node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/node.mjs";
-import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 import { Grain, Grain as Grain$1, Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
+import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 interface WriteCtx extends TenantCtx {
   table: TableName;
   date?: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "1.4.7",
+  "version": "1.4.10",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "^1.4.7",
-    "@gscdump/lakehouse": "^1.4.7",
-    "gscdump": "^1.4.7"
+    "@gscdump/lakehouse": "^1.4.10",
+    "gscdump": "^1.4.10",
+    "@gscdump/contracts": "^1.4.10"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",