@gscdump/engine 1.4.6 → 1.4.8

package/dist/adapters/node-harness.d.mts

@@ -1,6 +1,6 @@
 import { DataSource, StorageEngine } from "../storage.mjs";
-import { SearchType } from "gscdump/query";
 import { Row, TableName, TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 interface NodeHarnessOptions {
   dataDir: string;
   /** Tenant user id. Defaults to `'local'` for single-user CLI installs. */

package/dist/entities.mjs

@@ -4,6 +4,7 @@ import { readOptional } from "./adapters/read-optional.mjs";
 import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./sitemap-projection.mjs";
 import { buildQueryDimRecords, createQueryDimStore } from "./query-dim.mjs";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+import { GSCDUMP_INDEXING_TRANSITION_FIELDS } from "@gscdump/contracts";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
 const ENTITY_IO_CONCURRENCY = 8;
 async function mapEntityIo(items, fn) {
@@ -157,14 +158,6 @@ const INSPECTION_EVENT_COLUMNS = [
 		nullable: true
 	}
 ];
-const TRANSITION_STATE_FIELDS = [
-	"indexStatus",
-	"coverageState",
-	"robotsTxtState",
-	"indexingState",
-	"pageFetchState",
-	"googleCanonical"
-];
 const INSPECTION_TRANSITION_COLUMNS = [
 	{
 		name: "urlHash",
@@ -186,7 +179,7 @@ const INSPECTION_TRANSITION_COLUMNS = [
 		type: "VARCHAR",
 		nullable: false
 	},
-	...TRANSITION_STATE_FIELDS.flatMap((field) => {
+	...GSCDUMP_INDEXING_TRANSITION_FIELDS.flatMap((field) => {
 		const capped = field[0].toUpperCase() + field.slice(1);
 		return [{
 			name: `from${capped}`,
@@ -200,7 +193,7 @@ const INSPECTION_TRANSITION_COLUMNS = [
 	})
 ];
 function isStateTransition(before, after) {
-	return TRANSITION_STATE_FIELDS.some((field) => (before[field] ?? null) !== (after[field] ?? null));
+	return GSCDUMP_INDEXING_TRANSITION_FIELDS.some((field) => (before[field] ?? null) !== (after[field] ?? null));
 }
 function buildTransitionRow(before, after) {
 	const row = {
@@ -209,7 +202,7 @@ function buildTransitionRow(before, after) {
 		changedAfter: String(before.inspectedAt ?? ""),
 		changedBefore: String(after.inspectedAt ?? "")
 	};
-	for (const field of TRANSITION_STATE_FIELDS) {
+	for (const field of GSCDUMP_INDEXING_TRANSITION_FIELDS) {
 		const capped = field[0].toUpperCase() + field.slice(1);
 		row[`from${capped}`] = before[field] ?? null;
 		row[`to${capped}`] = after[field] ?? null;
@@ -374,6 +367,7 @@ function createInspectionStore(opts) {
 			for (const row of baseRows) consider(row);
 			let eventsFolded = 0;
 			const consumed = [];
+			const eventRows = [];
 			const eventFiles = await mapEntityIo(eventKeys.sort(), async (key) => {
 				const bytes = await readOptional(ds, key);
 				if (!bytes) return void 0;
@@ -386,11 +380,11 @@ function createInspectionStore(opts) {
 				if (!file) continue;
 				const { key, rows } = file;
 				consumed.push(key);
-				for (const row of rows) {
-					consider(row);
-					eventsFolded++;
-				}
+				eventRows.push(...rows);
+				eventsFolded += rows.length;
 			}
+			eventRows.sort((a, b) => String(a.urlHash).localeCompare(String(b.urlHash)) || String(a.inspectedAt ?? "").localeCompare(String(b.inspectedAt ?? "")));
+			for (const row of eventRows) consider(row);
 			const merged = [];
 			for (const [h, row] of latest) {
 				const fc = earliestChecked.get(h);
@@ -401,8 +395,8 @@ function createInspectionStore(opts) {
 				columns: INSPECTION_EVENT_COLUMNS,
 				sortKey: ["urlHash"]
 			});
-			await ds.write(baseKey, bytes);
 			const transitionsWritten = opts?.transitions ? await appendTransitions(ds, ctx, transitions) : 0;
+			await ds.write(baseKey, bytes);
 			if (consumed.length > 0) await ds.delete(consumed);
 			return {
 				baseRowCount: merged.length,

package/dist/layout.d.mts

@@ -1,6 +1,6 @@
 import { CompactionTier, ManifestEntry } from "./storage.mjs";
-import { SearchType } from "gscdump/query";
 import { TableName, TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 declare function dayPartition(date: string): string;
 /**
  * Hourly partition keyed by the PT calendar day (`YYYY-MM-DD`). One parquet

package/dist/resolver/run-query.d.mts

@@ -1,7 +1,7 @@
 import { SearchType as SearchType$1, TableName as TableName$1 } from "../storage.mjs";
 import { ComparisonFilter } from "./types.mjs";
-import { BuilderState } from "gscdump/query";
 import { Grain } from "@gscdump/contracts";
+import { BuilderState } from "gscdump/query";
 interface RunQueryCtx {
   userId: string;
   siteId: string;

package/dist/rollups.d.mts

@@ -2,8 +2,8 @@ import { DataSource, FileSetRef, Row as Row$1, TableName as TableName$1 } from "
 import { ColumnDef } from "./schema.mjs";
 import { EngineError } from "./errors.mjs";
 import "./contracts.mjs";
-import { SearchType } from "gscdump/query";
 import { TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
 interface RollupCtx extends TenantCtx {
   /** When the rollup was built. Stamped into payload + filename. */
   builtAt: number;

package/dist/source/source-types.d.mts

@@ -1,7 +1,7 @@
 import { ResolverAdapter } from "../resolver/types.mjs";
 import { PlannerCapabilities } from "gscdump/query/plan";
-import { BuilderState } from "gscdump/query";
 import { TableName } from "@gscdump/contracts";
+import { BuilderState } from "gscdump/query";
 type QueryRow = Record<string, unknown>;
 interface FileSet {
   table: TableName;

package/dist/storage.d.mts

@@ -1,8 +1,8 @@
 import { CompactionThresholds } from "./compaction.mjs";
 import { ParquetQueryFilter } from "./node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/index.mjs";
 import "./node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/types/node.mjs";
-import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 import { Grain, Grain as Grain$1, Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
+import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 interface WriteCtx extends TenantCtx {
   table: TableName;
   date?: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "1.4.6",
+  "version": "1.4.8",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "^1.4.6",
-    "gscdump": "^1.4.6",
-    "@gscdump/lakehouse": "^1.4.6"
+    "@gscdump/contracts": "^1.4.8",
+    "@gscdump/lakehouse": "^1.4.8",
+    "gscdump": "^1.4.8"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",