@gscdump/engine 1.4.8 → 1.4.11

package/dist/entities.d.mts

@@ -4,6 +4,7 @@ import { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUr
 import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
 import { TenantCtx } from "@gscdump/contracts";
+import { CanonicalDifferenceKind } from "gscdump";
 /**
  * GSC URL inspection result fields we persist. Mirrors the
  *  `searchconsole_v1.Schema$UrlInspectionResult` shape but as plain JSON
@@ -106,6 +107,8 @@ interface InspectionParquetRow {
  * newest-by-`inspectedAt` event.
  */
 interface InspectionEventRow extends InspectionParquetRow {
+  /** Declared-vs-selected canonical classification, computed at inspection ingest. */
+  canonicalMismatchKind: CanonicalDifferenceKind;
   crawlingUserAgent: string | null;
   /** JSON-encoded `RichResultsItem[]`. */
   richResultsItems: string | null;
@@ -212,6 +215,15 @@ interface InspectionStore {
     eventFilesDeleted: number;
     transitionsWritten: number;
   }>;
+  /**
+   * Rewrite a legacy latest-only base with the canonical kind derived from the
+   * canonical pair it already retains. Outstanding events must be compacted first.
+   */
+  backfillCanonicalMismatchKinds: (ctx: TenantCtx) => Promise<{
+    baseRowCount: number;
+    rowsBackfilled: number;
+    rewritten: boolean;
+  }>;
   /**
    * DuckDB-resolvable URI for the materialised parquet sidecar, or
    * `undefined` if the underlying `DataSource` has no native URI shape

package/dist/entities.mjs

@@ -5,6 +5,7 @@ import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySite
 import { buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 import { GSCDUMP_INDEXING_TRANSITION_FIELDS } from "@gscdump/contracts";
+import { classifyCanonicalDifference } from "gscdump";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 const ENTITY_IO_CONCURRENCY = 8;
 async function mapEntityIo(items, fn) {
@@ -107,6 +108,11 @@ const INSPECTION_PARQUET_COLUMNS = [
 ];
 const INSPECTION_EVENT_COLUMNS = [
 	...INSPECTION_PARQUET_COLUMNS,
+	{
+		name: "canonicalMismatchKind",
+		type: "VARCHAR",
+		nullable: false
+	},
 	{
 		name: "crawlingUserAgent",
 		type: "VARCHAR",
@@ -158,6 +164,19 @@ const INSPECTION_EVENT_COLUMNS = [
 		nullable: true
 	}
 ];
+const CANONICAL_DIFFERENCE_KINDS = /* @__PURE__ */ new Set([
+	"none",
+	"formatting",
+	"path",
+	"cross_domain"
+]);
+function populateCanonicalMismatchKind(row) {
+	const current = row.canonicalMismatchKind;
+	const expected = classifyCanonicalDifference(typeof row.userCanonical === "string" ? row.userCanonical : null, typeof row.googleCanonical === "string" ? row.googleCanonical : null);
+	const missingOrInvalid = typeof current !== "string" || !CANONICAL_DIFFERENCE_KINDS.has(current);
+	row.canonicalMismatchKind = expected;
+	return missingOrInvalid || current !== expected;
+}
 const INSPECTION_TRANSITION_COLUMNS = [
 	{
 		name: "urlHash",
@@ -389,6 +408,7 @@ function createInspectionStore(opts) {
 			for (const [h, row] of latest) {
 				const fc = earliestChecked.get(h);
 				if (fc !== void 0) row.firstCheckedAt = fc;
+				populateCanonicalMismatchKind(row);
 				merged.push(row);
 			}
 			const bytes = encodeRowsToParquetFlex(merged, {
@@ -405,6 +425,32 @@ function createInspectionStore(opts) {
 				transitionsWritten
 			};
 		},
+		async backfillCanonicalMismatchKinds(ctx) {
+			const baseKey = inspectionBaseKey(ctx);
+			const baseBytes = await readOptional(ds, baseKey);
+			if (!baseBytes) return {
+				baseRowCount: 0,
+				rowsBackfilled: 0,
+				rewritten: false
+			};
+			const rows = await decodeParquetToRows(baseBytes);
+			let rowsBackfilled = 0;
+			for (const row of rows) if (populateCanonicalMismatchKind(row)) rowsBackfilled++;
+			if (rowsBackfilled === 0) return {
+				baseRowCount: rows.length,
+				rowsBackfilled: 0,
+				rewritten: false
+			};
+			await ds.write(baseKey, encodeRowsToParquetFlex(rows, {
+				columns: INSPECTION_EVENT_COLUMNS,
+				sortKey: ["urlHash"]
+			}));
+			return {
+				baseRowCount: rows.length,
+				rowsBackfilled,
+				rewritten: true
+			};
+		},
 		parquetUri(ctx) {
 			return ds.uri?.(inspectionParquetKey(ctx));
 		}

package/dist/rollups.mjs

@@ -978,7 +978,7 @@ const indexingHealthRollup = {
         SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'NOT_FOUND' THEN 1 ELSE 0 END)::BIGINT AS not_found,
         SUM(CASE WHEN CAST(mobileUsabilityVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS mobile_passes,
         SUM(CASE WHEN CAST(richResultsVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
-        SUM(CASE WHEN userCanonical IS NOT NULL AND googleCanonical IS NOT NULL AND CAST(userCanonical AS VARCHAR) <> CAST(googleCanonical AS VARCHAR) THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
+        SUM(CASE WHEN canonicalMismatchKind IN ('path', 'cross_domain') THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
       FROM read_parquet({{INSPECTIONS}}, union_by_name = true)
       WHERE substr(CAST(inspectedAt AS VARCHAR), 1, 10) >= '${utcDateMinusDays(windowAnchorMs, 90)}'
       GROUP BY 1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "1.4.8",
+  "version": "1.4.11",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "^1.4.8",
-    "@gscdump/lakehouse": "^1.4.8",
-    "gscdump": "^1.4.8"
+    "@gscdump/contracts": "^1.4.11",
+    "@gscdump/lakehouse": "^1.4.11",
+    "gscdump": "^1.4.11"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",