@gscdump/engine 0.26.3 → 0.26.6

package/dist/_chunks/engine.mjs

@@ -4,7 +4,7 @@ import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from
 import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
-import { normalizeUrl } from "gscdump";
+import { normalizeUrl } from "gscdump/normalize";
 async function encodeBytes(db, table, rows) {
 	const inName = db.makeTempPath("json");
 	const outName = db.makeTempPath("parquet");

package/dist/_chunks/entities.mjs

@@ -180,6 +180,16 @@ const INSPECTION_EVENT_COLUMNS = [
 		name: "checkCount",
 		type: "INTEGER",
 		nullable: true
+	},
+	{
+		name: "nextCheckAfter",
+		type: "BIGINT",
+		nullable: true
+	},
+	{
+		name: "nextCheckPriority",
+		type: "VARCHAR",
+		nullable: true
 	}
 ];
 function createInspectionStore(opts) {
@@ -710,6 +720,92 @@ function createSitemapStore(opts) {
 				await ds.write(indexKey, bytes);
 				if (consumed.length > 0) await ds.delete(consumed);
 			}
+		},
+		async reconcile(ctx, { liveFeedpaths, at: atOpt }) {
+			const at = atOpt ?? now();
+			const liveHashes = new Set(liveFeedpaths.map((fp) => hash(fp)));
+			const present = /* @__PURE__ */ new Set();
+			for (const key of await ds.list(`${sitemapUrlsIndexPrefix(ctx)}/`)) {
+				const m = /\/by-feed\/([0-9a-f]+)\/index\.parquet$/.exec(key);
+				if (m) present.add(m[1]);
+			}
+			const deltasByFeed = /* @__PURE__ */ new Map();
+			for (const key of await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)) {
+				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
+				if (!m) continue;
+				present.add(m[2]);
+				const list = deltasByFeed.get(m[2]) ?? [];
+				list.push(key);
+				deltasByFeed.set(m[2], list);
+			}
+			let feedpathsPruned = 0;
+			let urlsRemoved = 0;
+			for (const fpHash of present) {
+				if (liveHashes.has(fpHash)) continue;
+				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
+				const indexBytes = await readOptional(ds, indexKey);
+				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+				const live = /* @__PURE__ */ new Map();
+				const removed = /* @__PURE__ */ new Map();
+				for (const row of indexRows) {
+					const r = rowToUrlRecord(row);
+					if (r.removedAt != null) removed.set(r.urlHash, r);
+					else live.set(r.urlHash, r);
+				}
+				const consumed = [];
+				for (const key of (deltasByFeed.get(fpHash) ?? []).sort()) {
+					const bytes = await readOptional(ds, key);
+					if (!bytes) continue;
+					consumed.push(key);
+					const rows = await decodeParquetToRows(bytes);
+					for (const r of rows) {
+						const urlHash = String(r.url_hash);
+						const dat = Number(r.at);
+						if (String(r.op) === "added") {
+							const prev = live.get(urlHash) ?? removed.get(urlHash);
+							removed.delete(urlHash);
+							live.set(urlHash, {
+								feedpath: String(r.feedpath),
+								feedpathHash: fpHash,
+								urlHash,
+								loc: String(r.loc),
+								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+								firstSeenAt: prev?.firstSeenAt ?? dat,
+								lastSeenAt: dat
+							});
+						} else if (String(r.op) === "removed") {
+							const prev = live.get(urlHash);
+							live.delete(urlHash);
+							if (prev) removed.set(urlHash, {
+								...prev,
+								removedAt: dat
+							});
+						}
+					}
+				}
+				const hadLive = live.size > 0;
+				if (!hadLive && consumed.length === 0) continue;
+				for (const [urlHash, r] of live) {
+					removed.set(urlHash, {
+						...r,
+						removedAt: at
+					});
+					urlsRemoved++;
+				}
+				const merged = [...removed.values()];
+				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+					columns: URLS_INDEX_COLUMNS,
+					sortKey: ["feedpath_hash", "url_hash"]
+				});
+				await ds.write(indexKey, bytes);
+				if (consumed.length > 0) await ds.delete(consumed);
+				if (hadLive) feedpathsPruned++;
+			}
+			return {
+				feedpathsPruned,
+				urlsRemoved
+			};
 		}
 	};
 }

package/dist/_chunks/resolver.mjs

@@ -3,8 +3,8 @@ import { enumeratePartitions } from "./compaction.mjs";
 import { escapeLike } from "../sql-fragments.mjs";
 import "../planner.mjs";
 import { UnresolvableDatasetError, buildLogicalComparisonPlan, buildLogicalPlan, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
-import { normalizeUrl } from "gscdump";
 import { PgDialect, pgTable, varchar } from "drizzle-orm/pg-core";
+import { normalizeUrl } from "gscdump/normalize";
 import { sql } from "drizzle-orm";
 const DIMENSION_SURFACES = {
 	page: ["api", "stored"],

package/dist/adapters/node.mjs

@@ -2,9 +2,9 @@ import { engineErrors } from "../errors.mjs";
 import { createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "../_chunks/engine.mjs";
 import { createNodeDuckDBHandle, resetNodeDuckDB } from "./duckdb-node.mjs";
 import { createFilesystemDataSource, createFilesystemManifestStore } from "./filesystem.mjs";
-import { encodeSiteId } from "gscdump";
 import { err, ok, unwrapResult } from "gscdump/result";
 import path from "node:path";
+import { encodeSiteId } from "gscdump/tenant";
 function createNodeHarness(opts) {
 	const dataDir = opts.dataDir;
 	const userId = opts.userId ?? "local";

package/dist/entities.d.mts

@@ -138,6 +138,14 @@ interface InspectionEventRow extends InspectionParquetRow {
   firstCheckedAt: string | null;
   /** Total number of inspections recorded for this url. */
   checkCount: number | null;
+  /**
+   * Stored next-recheck unix-seconds + priority as computed AT INSPECT TIME.
+   * Carried verbatim (NOT recomputed at read) because the scheduling policy can
+   * change over time — `__gsc/inspections` must replay the historical value to
+   * keep its frozen wire shape byte-stable.
+   */
+  nextCheckAfter: number | null;
+  nextCheckPriority: string | null;
 }
 /**
  * Hard cap on a single `appendHistory` shard payload. Encoded bytes >
@@ -311,6 +319,12 @@ interface SnapshotUrlsResult {
   /** True when contentHash matched prior; the call performed zero writes. */
   unchanged: boolean;
 }
+interface ReconcileResult {
+  /** Feedpaths that were absent from the live set and had their live URLs pruned. */
+  feedpathsPruned: number;
+  /** Total URL rows transitioned live → removed across pruned feedpaths. */
+  urlsRemoved: number;
+}
 interface DeltaEntry {
   feedpath: string;
   feedpathHash: string;
@@ -362,6 +376,21 @@ interface SitemapStore {
    * regardless of total site URL count.
    */
   compactUrls: (ctx: TenantCtx) => Promise<void>;
+  /**
+   * Site-wide convergence: mark every still-live URL whose owning feedpath is
+   * absent from `liveFeedpaths` as removed. `compactUrls`/`snapshotUrls` only
+   * prune URLs *inside* a feedpath that was re-observed; a whole feed dropped
+   * from the sitemap list (no `snapshotUrls` call) leaves its URLs frozen-live
+   * forever. This is the sidecar mirror of the D1 generation sweep: it rewrites
+   * each dropped feedpath's `by-feed/<hash>/index.parquet` with `removedAt` set
+   * and deletes its outstanding deltas (write-new-base + delete-deltas,
+   * ADR-0002). Bounded per feedpath, so memory stays flat regardless of site
+   * size. Live feedpaths are never touched.
+   */
+  reconcile: (ctx: TenantCtx, opts: {
+    liveFeedpaths: readonly string[];
+    at?: number;
+  }) => Promise<ReconcileResult>;
 }
 interface CreateSitemapStoreOptions {
   dataSource: DataSource;
@@ -413,4 +442,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.26.3",
+  "version": "0.26.6",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -190,8 +190,8 @@
     "drizzle-orm": "1.0.0-rc.3",
     "icebird": "^0.8.8",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.26.3",
-    "@gscdump/contracts": "0.26.3"
+    "@gscdump/contracts": "0.26.6",
+    "gscdump": "0.26.6"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",