@gscdump/engine 5.2.0 → 5.3.0

package/dist/entities/inspection.d.mts

@@ -200,16 +200,25 @@ export interface InspectionStore {
     rowCount: number;
   }>;
   /**
-   * Fold every outstanding event file into the `base.parquet`: latest-per-url
-   * by max `inspectedAt` (newest-wins), preserving the earliest non-null
-   * `firstCheckedAt` per url. Writes the new base then deletes the consumed
-   * event files — file-level only, never row-level (ADR-0002). Idempotent +
-   * re-runnable: a crash after the base write but before the delete just
-   * re-folds the same events (newest-wins makes that a no-op). A real read
-   * failure on the existing base propagates rather than rebuilding from events
-   * alone (which would drop URLs only the base held).
+   * Fold every outstanding event file into a new base: latest-per-url by max
+   * `inspectedAt` (newest-wins), preserving the earliest non-null
+   * `firstCheckedAt` per url. The first fold reads the legacy in-place
+   * `base.parquet`; later folds read the base the manifest names.
    *
-   * No-op (no base rewrite) when there are zero outstanding events.
+   * Writes the new base under a new immutable key, then publishes the inspection
+   * base manifest that names it. The superseded base and the folded event files
+   * are retired, not deleted: reads that resolved the previous manifest keep
+   * scanning them. A later compaction deletes each retired object once the
+   * grace has passed. It also deletes a base that an interrupted compaction
+   * wrote but never published. File-level only, never row-level (ADR-0002).
+   *
+   * Callers serialize compaction per site. Re-runnable: a crash before the
+   * manifest write leaves the previous manifest current, and the retry
+   * re-folds the same events. A real read failure on the current base
+   * propagates rather than rebuilding from events alone.
+   *
+   * `eventFilesDeleted` counts event files this run deleted, which an earlier
+   * run folded. No-op when no event file exists.
    */
   compactInspections: (ctx: TenantCtx, opts?: {
     /**
@@ -224,8 +233,9 @@ export interface InspectionStore {
     transitionsWritten: number;
   }>;
   /**
-   * Rewrite a legacy latest-only base with the canonical kind derived from the
-   * canonical pair it already retains. Outstanding events must be compacted first.
+   * Publish a copy of the current base with the canonical kind derived from the
+   * canonical pair it already retains. Outstanding events must be compacted
+   * first. Like compaction, it writes a new immutable base and retires the old one.
    */
   backfillCanonicalMismatchKinds: (ctx: TenantCtx) => Promise<{
     baseRowCount: number;
@@ -246,5 +256,50 @@ export interface InspectionStore {
 }
 export interface CreateInspectionStoreOptions {
   dataSource: DataSource;
+  /** Clock for retirement times and the grace check. Defaults to `Date.now`. */
+  now?: () => number;
+  /** How long a retired object stays readable. Defaults to {@link INSPECTION_RETIRED_GRACE_MS}. */
+  graceMs?: number;
 }
+/**
+ * How long compaction keeps a superseded base or a folded event file after it
+ * retires it. Every read must finish within this window after it resolves its
+ * {@link InspectionReadPlan}.
+ *
+ * The longest hosted reader is a queue job that holds one plan across several
+ * DuckDB queries. The Workers queue wall clock caps that job at 15 minutes, so
+ * one hour leaves a wide margin.
+ */
+export declare const INSPECTION_RETIRED_GRACE_MS: number;
+/**
+ * The base one read scans.
+ *
+ * - `legacy`: no manifest exists yet. The key is the in-place base an older
+ *   engine wrote, which may be absent. It stays readable until the first
+ *   compaction under the manifest retires it.
+ * - `published`: an immutable base the manifest names. It never changes, and it
+ *   exists for at least the grace after a later manifest supersedes it.
+ */
+export type InspectionBaseRef = {
+  _tag: 'legacy';
+  key: string;
+} | {
+  _tag: 'published';
+  key: string;
+};
+/**
+ * What one read scans. Resolve it once per read, then list event files, then
+ * scan. Skip each listed event key in `foldedEventKeys`: the base already holds
+ * its rows, and the file remains only for reads that resolved an older plan.
+ */
+export interface InspectionReadPlan {
+  base: InspectionBaseRef;
+  foldedEventKeys: ReadonlySet<string>;
+}
+/**
+ * Resolve the base a read scans and the event files it must skip. Reads one
+ * small manifest object; never the base itself. Throws when the manifest is
+ * corrupt, because no base choice is safe then.
+ */
+export declare function resolveInspectionReadPlan(source: Pick<DataSource, 'read'>, ctx: TenantCtx): Promise<InspectionReadPlan>;
 export declare function createInspectionStore(opts: CreateInspectionStoreOptions): InspectionStore;

package/dist/entities/inspection.mjs

@@ -1,4 +1,4 @@
-import { inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionParquetKey, inspectionTransitionsMonthKey } from "../entity-keys.mjs";
+import { inspectionBaseKey, inspectionBaseManifestKey, inspectionBasesPrefix, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionParquetKey, inspectionTransitionsMonthKey } from "../entity-keys.mjs";
 import { readOptional } from "../adapters/read-optional.mjs";
 import { decodeParquetToRows, encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
 import { mapEntityIo } from "./io.mjs";
@@ -6,8 +6,82 @@ import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 import { GSCDUMP_INDEXING_TRANSITION_FIELDS } from "@gscdump/contracts";
 import { classifyCanonicalDifference } from "gscdump";
 const YEAR_MONTH_RE = /^(\d{4})-(\d{2})-/;
-const INSPECTION_EVENT_KEY_RE = /\/inspections\/events\/\d{4}-\d{2}\/[^/]+\.parquet$/;
 const INSPECTION_HISTORY_MAX_BYTES = 5242880;
+const INSPECTION_RETIRED_GRACE_MS = 36e5;
+const EVENT_FILE_RE = /^\d{4}-\d{2}\/[^/]+\.parquet$/;
+const IMMUTABLE_BASE_FILE_RE = /^(\d{13,})__[\w-]+\.parquet$/;
+function isInspectionEventKey(ctx, key) {
+	const prefix = `${inspectionEventsPrefix(ctx)}/`;
+	return key.startsWith(prefix) && EVENT_FILE_RE.test(key.slice(prefix.length));
+}
+function immutableBaseKey(ctx, writtenAt, id) {
+	return `${inspectionBasesPrefix(ctx)}/${String(writtenAt).padStart(13, "0")}__${id}.parquet`;
+}
+function immutableBaseWrittenAt(ctx, key) {
+	const prefix = `${inspectionBasesPrefix(ctx)}/`;
+	if (!key.startsWith(prefix)) return void 0;
+	const match = IMMUTABLE_BASE_FILE_RE.exec(key.slice(prefix.length));
+	return match ? Number(match[1]) : void 0;
+}
+function invalidManifest(key, reason) {
+	throw new Error(`invalid inspection base manifest ${key}: ${reason}`);
+}
+function parseBaseManifest(ctx, key, bytes) {
+	let input;
+	try {
+		input = JSON.parse(new TextDecoder().decode(bytes));
+	} catch {
+		return invalidManifest(key, "not JSON");
+	}
+	if (!input || typeof input !== "object" || Array.isArray(input)) return invalidManifest(key, "not an object");
+	const manifest = input;
+	if (manifest.version !== 1) return invalidManifest(key, `unknown version ${String(manifest.version)}`);
+	const baseKey = manifest.baseKey;
+	if (typeof baseKey !== "string" || immutableBaseWrittenAt(ctx, baseKey) === void 0) return invalidManifest(key, "base key is not an immutable base of this tenant");
+	if (!Array.isArray(manifest.retired)) return invalidManifest(key, "retired is not an array");
+	return {
+		version: 1,
+		baseKey,
+		retired: manifest.retired.map((entry) => {
+			if (!entry || typeof entry !== "object") return invalidManifest(key, "retired entry is not an object");
+			const { key: retiredKey, retiredAt } = entry;
+			if (typeof retiredKey !== "string" || retiredKey === baseKey || !(retiredKey === inspectionBaseKey(ctx) || immutableBaseWrittenAt(ctx, retiredKey) !== void 0 || isInspectionEventKey(ctx, retiredKey))) return invalidManifest(key, `retired key ${String(retiredKey)} is not a superseded inspection file`);
+			if (typeof retiredAt !== "number" || !Number.isSafeInteger(retiredAt) || retiredAt < 0) return invalidManifest(key, `retired time for ${retiredKey} is invalid`);
+			return {
+				key: retiredKey,
+				retiredAt
+			};
+		})
+	};
+}
+async function readBaseState(source, ctx) {
+	const key = inspectionBaseManifestKey(ctx);
+	const bytes = await readOptional(source, key);
+	return bytes ? {
+		_tag: "published",
+		manifest: parseBaseManifest(ctx, key, bytes)
+	} : {
+		_tag: "legacy",
+		key: inspectionBaseKey(ctx)
+	};
+}
+async function resolveInspectionReadPlan(source, ctx) {
+	const state = await readBaseState(source, ctx);
+	if (state._tag === "legacy") return {
+		base: {
+			_tag: "legacy",
+			key: state.key
+		},
+		foldedEventKeys: /* @__PURE__ */ new Set()
+	};
+	return {
+		base: {
+			_tag: "published",
+			key: state.manifest.baseKey
+		},
+		foldedEventKeys: new Set(state.manifest.retired.map((entry) => entry.key).filter((key) => isInspectionEventKey(ctx, key)))
+	};
+}
 const INSPECTION_PARQUET_COLUMNS = [
 	{
 		name: "urlHash",
@@ -246,8 +320,40 @@ async function appendTransitions(ds, ctx, rows) {
 function transitionMonth(row) {
 	return String(row.changedBefore ?? "").slice(0, 7) || "unknown";
 }
+const EMPTY_COMPACTION = {
+	baseRowCount: 0,
+	eventsFolded: 0,
+	eventFilesDeleted: 0,
+	transitionsWritten: 0
+};
 function createInspectionStore(opts) {
 	const ds = opts.dataSource;
+	const now = () => Math.floor((opts.now ?? Date.now)());
+	const graceMs = opts.graceMs ?? 36e5;
+	async function writeImmutableBase(ctx, bytes) {
+		const key = immutableBaseKey(ctx, now(), randomBatchId());
+		await ds.write(key, bytes);
+		return key;
+	}
+	async function publishManifest(ctx, manifest) {
+		await ds.write(inspectionBaseManifestKey(ctx), encodeJsonBigintSafe(manifest));
+	}
+	async function sweepRetired(ctx, retired, live, at) {
+		const retiredKeys = new Set(retired.map((entry) => entry.key));
+		const expired = retired.filter((entry) => entry.retiredAt + graceMs <= at).map((entry) => entry.key);
+		const unpublished = (await ds.list(`${inspectionBasesPrefix(ctx)}/`)).filter((key) => {
+			const writtenAt = immutableBaseWrittenAt(ctx, key);
+			return writtenAt !== void 0 && !retiredKeys.has(key) && writtenAt + graceMs <= at;
+		});
+		const doomed = [...expired, ...unpublished].filter((key) => !live.has(key));
+		if (doomed.length > 0) await ds.delete(doomed);
+		return new Set(doomed);
+	}
+	function countEventKeys(ctx, keys) {
+		let count = 0;
+		for (const key of keys) if (isInspectionEventKey(ctx, key)) count++;
+		return count;
+	}
 	function shardFor(record) {
 		const m = YEAR_MONTH_RE.exec(record.inspectedAt);
 		return m ? `${m[1]}-${m[2]}` : "unknown";
@@ -358,15 +464,28 @@ function createInspectionStore(opts) {
 			};
 		},
 		async compactInspections(ctx, opts) {
-			const eventKeys = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((k) => INSPECTION_EVENT_KEY_RE.test(k));
-			if (eventKeys.length === 0) return {
-				baseRowCount: 0,
-				eventsFolded: 0,
-				eventFilesDeleted: 0,
-				transitionsWritten: 0
-			};
-			const baseKey = inspectionBaseKey(ctx);
+			const listed = (await ds.list(`${inspectionEventsPrefix(ctx)}/`)).filter((key) => isInspectionEventKey(ctx, key));
+			if (listed.length === 0) return EMPTY_COMPACTION;
+			const startedAt = now();
+			const state = await readBaseState(ds, ctx);
+			const retired = state._tag === "published" ? state.manifest.retired : [];
+			const retiredKeys = new Set(retired.map((entry) => entry.key));
+			const eventKeys = listed.filter((key) => !retiredKeys.has(key));
+			const baseKey = state._tag === "published" ? state.manifest.baseKey : state.key;
+			if (eventKeys.length === 0) {
+				const deleted = await sweepRetired(ctx, retired, /* @__PURE__ */ new Set([baseKey]), startedAt);
+				if (state._tag === "published" && retired.some((entry) => deleted.has(entry.key))) await publishManifest(ctx, {
+					version: 1,
+					baseKey,
+					retired: retired.filter((entry) => !deleted.has(entry.key))
+				});
+				return {
+					...EMPTY_COMPACTION,
+					eventFilesDeleted: countEventKeys(ctx, deleted)
+				};
+			}
 			const baseBytes = await readOptional(ds, baseKey);
+			if (!baseBytes && state._tag === "published") throw new Error(`published inspection base missing: ${baseKey}`);
 			const baseRows = baseBytes ? await decodeParquetToRows(baseBytes) : [];
 			const latest = /* @__PURE__ */ new Map();
 			const earliestChecked = /* @__PURE__ */ new Map();
@@ -418,23 +537,43 @@ function createInspectionStore(opts) {
 				sortKey: ["urlHash"]
 			});
 			const transitionsWritten = opts?.transitions ? await appendTransitions(ds, ctx, transitions) : 0;
-			await ds.write(baseKey, bytes);
-			if (consumed.length > 0) await ds.delete(consumed);
+			const nextBaseKey = await writeImmutableBase(ctx, bytes);
+			const deleted = await sweepRetired(ctx, retired, /* @__PURE__ */ new Set([baseKey, nextBaseKey]), startedAt);
+			const retiredAt = now();
+			await publishManifest(ctx, {
+				version: 1,
+				baseKey: nextBaseKey,
+				retired: [
+					...retired.filter((entry) => !deleted.has(entry.key)),
+					...baseBytes ? [{
+						key: baseKey,
+						retiredAt
+					}] : [],
+					...consumed.map((key) => ({
+						key,
+						retiredAt
+					}))
+				]
+			});
 			return {
 				baseRowCount: merged.length,
 				eventsFolded,
-				eventFilesDeleted: consumed.length,
+				eventFilesDeleted: countEventKeys(ctx, deleted),
 				transitionsWritten
 			};
 		},
 		async backfillCanonicalMismatchKinds(ctx) {
-			const baseKey = inspectionBaseKey(ctx);
+			const state = await readBaseState(ds, ctx);
+			const baseKey = state._tag === "published" ? state.manifest.baseKey : state.key;
 			const baseBytes = await readOptional(ds, baseKey);
-			if (!baseBytes) return {
-				baseRowCount: 0,
-				rowsBackfilled: 0,
-				rewritten: false
-			};
+			if (!baseBytes) {
+				if (state._tag === "published") throw new Error(`published inspection base missing: ${baseKey}`);
+				return {
+					baseRowCount: 0,
+					rowsBackfilled: 0,
+					rewritten: false
+				};
+			}
 			const rows = await decodeParquetToRows(baseBytes);
 			let rowsBackfilled = 0;
 			for (const row of rows) if (populateCanonicalMismatchKind(row)) rowsBackfilled++;
@@ -443,10 +582,17 @@ function createInspectionStore(opts) {
 				rowsBackfilled: 0,
 				rewritten: false
 			};
-			await ds.write(baseKey, encodeRowsToParquetFlex(rows, {
-				columns: INSPECTION_EVENT_COLUMNS,
-				sortKey: ["urlHash"]
-			}));
+			await publishManifest(ctx, {
+				version: 1,
+				baseKey: await writeImmutableBase(ctx, encodeRowsToParquetFlex(rows, {
+					columns: INSPECTION_EVENT_COLUMNS,
+					sortKey: ["urlHash"]
+				})),
+				retired: [...state._tag === "published" ? state.manifest.retired : [], {
+					key: baseKey,
+					retiredAt: now()
+				}]
+			});
 			return {
 				baseRowCount: rows.length,
 				rowsBackfilled,
@@ -458,4 +604,4 @@ function createInspectionStore(opts) {
 		}
 	};
 }
-export { INSPECTION_HISTORY_MAX_BYTES, createInspectionStore };
+export { INSPECTION_HISTORY_MAX_BYTES, INSPECTION_RETIRED_GRACE_MS, createInspectionStore, resolveInspectionReadPlan };

package/dist/entities.d.mts

@@ -1,10 +1,10 @@
 import { CreateEmptyTypesStoreOptions, EmptyTypesDoc, EmptyTypesStore, createEmptyTypesStore } from "./entities/empty-types.mjs";
 import { CreateIndexingMetadataStoreOptions, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, createIndexingMetadataStore } from "./entities/indexing-metadata.mjs";
-import { CreateInspectionStoreOptions, INSPECTION_HISTORY_MAX_BYTES, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, createInspectionStore } from "./entities/inspection.mjs";
-import { SitemapFeedIdentityResult, SitemapGenerationKey, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import { CreateInspectionStoreOptions, INSPECTION_HISTORY_MAX_BYTES, INSPECTION_RETIRED_GRACE_MS, InspectionBaseRef, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionReadPlan, InspectionRecord, InspectionStore, createInspectionStore, resolveInspectionReadPlan } from "./entities/inspection.mjs";
+import { SitemapFeedIdentityResult, SitemapGenerationKey, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionBaseManifestKey, inspectionBasesPrefix, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
 import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./entities/query-dim.mjs";
 import { CompleteSitemapGeneration, CompleteSitemapTraversal, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, FinalizeSitemapGenerationResult, IterateSitemapGenerationUrlsOptions, IterateSitemapGenerationUrlsResult, ListSitemapGenerationUrlsOptions, ListSitemapGenerationUrlsResult, ParsedUrl, QuerySitemapGenerationMembershipResult, RecordSitemapLegacyImportResult, SitemapGenerationFeedManifest, SitemapGenerationReadResult, SitemapLegacyImportEvidence, SitemapMembershipEvent, SitemapMutation, SitemapReadStore, SitemapSiteGenerationManifest, SitemapStagedFeed, SitemapStore, SitemapUrlRecord, StageSitemapGenerationFeedResult } from "./entities/sitemap-shared.mjs";
 import { createSitemapReadStore } from "./entities/sitemap.mjs";
 import { CreateSitemapListStoreOptions, SitemapListContent, SitemapListDoc, SitemapListEntry, SitemapListStore, createSitemapListStore } from "./entities/sitemap-list.mjs";
 import { createSitemapStore } from "./entities/sitemap-write.mjs";
-export { type CompleteSitemapGeneration, type CompleteSitemapTraversal, type CreateEmptyTypesStoreOptions, type CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, type CreateSitemapListStoreOptions, type CreateSitemapReadStoreOptions, type CreateSitemapStoreOptions, type DateRange, type EmptyTypesDoc, type EmptyTypesStore, type FinalizeSitemapGenerationResult, INSPECTION_HISTORY_MAX_BYTES, type IndexingMetadataIndex, type IndexingMetadataRecord, type IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, type IterateSitemapGenerationUrlsOptions, type IterateSitemapGenerationUrlsResult, type ListSitemapGenerationUrlsOptions, type ListSitemapGenerationUrlsResult, type ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, type QuerySitemapGenerationMembershipResult, type RecordSitemapLegacyImportResult, SitemapFeedIdentityResult, type SitemapGenerationFeedManifest, SitemapGenerationKey, type SitemapGenerationReadResult, type SitemapLegacyImportEvidence, type SitemapListContent, type SitemapListDoc, type SitemapListEntry, type SitemapListStore, type SitemapMembershipEvent, type SitemapMutation, type SitemapReadStore, type SitemapSiteGenerationManifest, type SitemapStagedFeed, type SitemapStore, type SitemapUrlRecord, type StageSitemapGenerationFeedResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapListStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };
+export { type CompleteSitemapGeneration, type CompleteSitemapTraversal, type CreateEmptyTypesStoreOptions, type CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, type CreateSitemapListStoreOptions, type CreateSitemapReadStoreOptions, type CreateSitemapStoreOptions, type DateRange, type EmptyTypesDoc, type EmptyTypesStore, type FinalizeSitemapGenerationResult, INSPECTION_HISTORY_MAX_BYTES, INSPECTION_RETIRED_GRACE_MS, type IndexingMetadataIndex, type IndexingMetadataRecord, type IndexingMetadataStore, InspectionBaseRef, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionReadPlan, InspectionRecord, InspectionStore, type IterateSitemapGenerationUrlsOptions, type IterateSitemapGenerationUrlsResult, type ListSitemapGenerationUrlsOptions, type ListSitemapGenerationUrlsResult, type ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, type QuerySitemapGenerationMembershipResult, type RecordSitemapLegacyImportResult, SitemapFeedIdentityResult, type SitemapGenerationFeedManifest, SitemapGenerationKey, type SitemapGenerationReadResult, type SitemapLegacyImportEvidence, type SitemapListContent, type SitemapListDoc, type SitemapListEntry, type SitemapListStore, type SitemapMembershipEvent, type SitemapMutation, type SitemapReadStore, type SitemapSiteGenerationManifest, type SitemapStagedFeed, type SitemapStore, type SitemapUrlRecord, type StageSitemapGenerationFeedResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapListStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionBaseManifestKey, inspectionBasesPrefix, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, resolveInspectionReadPlan, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/entities.mjs

@@ -1,9 +1,9 @@
-import { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
+import { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionBaseManifestKey, inspectionBasesPrefix, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
 import { createEmptyTypesStore } from "./entities/empty-types.mjs";
 import { createIndexingMetadataStore } from "./entities/indexing-metadata.mjs";
-import { INSPECTION_HISTORY_MAX_BYTES, createInspectionStore } from "./entities/inspection.mjs";
+import { INSPECTION_HISTORY_MAX_BYTES, INSPECTION_RETIRED_GRACE_MS, createInspectionStore, resolveInspectionReadPlan } from "./entities/inspection.mjs";
 import { buildQueryDimRecords, createQueryDimStore } from "./entities/query-dim.mjs";
 import { createSitemapReadStore } from "./entities/sitemap.mjs";
 import { createSitemapListStore } from "./entities/sitemap-list.mjs";
 import { createSitemapStore } from "./entities/sitemap-write.mjs";
-export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapListStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };
+export { INSPECTION_HISTORY_MAX_BYTES, INSPECTION_RETIRED_GRACE_MS, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapListStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionBaseManifestKey, inspectionBasesPrefix, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, resolveInspectionReadPlan, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/entity-keys.d.mts

@@ -4,7 +4,19 @@ export declare function emptyTypesKey(ctx: TenantCtx): string;
 export declare function inspectionParquetKey(ctx: TenantCtx): string;
 export declare function inspectionEventsPrefix(ctx: TenantCtx): string;
 export declare function inspectionEventKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
+/**
+ * @deprecated Names only the legacy base that compaction rewrote in place.
+ * Compaction now publishes immutable bases through the inspection base
+ * manifest. Resolve the current base with `resolveInspectionReadPlan`.
+ */
 export declare function inspectionBaseKey(ctx: TenantCtx): string;
+/**
+ * The inspection base manifest: the mutable pointer to the current immutable
+ * base. Compaction writes it last, so it is the only publication point.
+ */
+export declare function inspectionBaseManifestKey(ctx: TenantCtx): string;
+/** Directory of immutable inspection bases. Each compaction writes one new key. */
+export declare function inspectionBasesPrefix(ctx: TenantCtx): string;
 /**
  * Append-only record of URL-inspection STATE CHANGES, partitioned by month.
  *

package/dist/entity-keys.mjs

@@ -19,6 +19,12 @@ function inspectionEventKey(ctx, yearMonth, batchId) {
 function inspectionBaseKey(ctx) {
 	return `${tenantEntityPrefix(ctx)}/inspections/base.parquet`;
 }
+function inspectionBaseManifestKey(ctx) {
+	return `${tenantEntityPrefix(ctx)}/inspections/base-manifest.json`;
+}
+function inspectionBasesPrefix(ctx) {
+	return `${tenantEntityPrefix(ctx)}/inspections/bases`;
+}
 function inspectionTransitionsPrefix(ctx) {
 	return `${tenantEntityPrefix(ctx)}/inspections/transitions`;
 }
@@ -144,4 +150,4 @@ function queryDimParquetKey(ctx) {
 function queryDimMetaKey(ctx) {
 	return `${queryDimPrefix(ctx)}/index.json`;
 }
-export { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };
+export { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionBaseManifestKey, inspectionBasesPrefix, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapListKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "5.2.0",
+  "version": "5.3.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^5.2.0",
-    "@gscdump/lakehouse": "^5.2.0",
+    "@gscdump/contracts": "^5.3.0",
+    "@gscdump/lakehouse": "^5.3.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^5.2.0",
+    "gscdump": "^5.3.0",
     "pathe": "^2.0.3",
     "proper-lockfile": "^4.1.2"
   },