@gscdump/engine 1.5.4 → 2.0.0

package/dist/entities.d.mts

@@ -1,10 +1,9 @@
 import { CreateEmptyTypesStoreOptions, EmptyTypesDoc, EmptyTypesStore, createEmptyTypesStore } from "./entities/empty-types.mjs";
 import { CreateIndexingMetadataStoreOptions, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, createIndexingMetadataStore } from "./entities/indexing-metadata.mjs";
 import { CreateInspectionStoreOptions, INSPECTION_HISTORY_MAX_BYTES, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, createInspectionStore } from "./entities/inspection.mjs";
-import { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
+import { SitemapFeedIdentityResult, SitemapGenerationKey, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
 import { QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, buildQueryDimRecords, createQueryDimStore } from "./entities/query-dim.mjs";
-import { CompactUrlsOptions, CompactUrlsResult, CompleteSitemapGeneration, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, LoadUrlsOptions, ParsedUrl, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapMembershipEvent, SitemapMutation, SitemapReadStore, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult } from "./entities/sitemap-shared.mjs";
+import { CompleteSitemapGeneration, CompleteSitemapTraversal, CreateSitemapReadStoreOptions, CreateSitemapStoreOptions, DateRange, FinalizeSitemapGenerationResult, IterateSitemapGenerationUrlsOptions, IterateSitemapGenerationUrlsResult, ListSitemapGenerationUrlsOptions, ListSitemapGenerationUrlsResult, ParsedUrl, QuerySitemapGenerationMembershipResult, RecordSitemapLegacyImportResult, SitemapGenerationFeedManifest, SitemapGenerationReadResult, SitemapLegacyImportEvidence, SitemapMembershipEvent, SitemapMutation, SitemapReadStore, SitemapSiteGenerationManifest, SitemapStagedFeed, SitemapStore, SitemapUrlRecord, StageSitemapGenerationFeedResult } from "./entities/sitemap-shared.mjs";
 import { createSitemapReadStore } from "./entities/sitemap.mjs";
-import { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./entities/sitemap-projection.mjs";
 import { createSitemapStore } from "./entities/sitemap-write.mjs";
-export { type CompactUrlsOptions, type CompactUrlsResult, type CompleteSitemapGeneration, type CreateEmptyTypesStoreOptions, type CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, type CreateSitemapReadStoreOptions, type CreateSitemapStoreOptions, type DateRange, type DeltaEntry, type EmptyTypesDoc, type EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, type IndexingMetadataIndex, type IndexingMetadataRecord, type IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, type LoadUrlsOptions, type ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, type ReconcileResult, SITEMAP_PROJECTION_GRACE_MS, SitemapGenerationKey, type SitemapHistoryDoc, type SitemapIndex, type SitemapMembershipEvent, type SitemapMutation, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, type SitemapReadStore, type SitemapRecord, type SitemapStore, type SitemapUrlRecord, type SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };
+export { type CompleteSitemapGeneration, type CompleteSitemapTraversal, type CreateEmptyTypesStoreOptions, type CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, type CreateSitemapReadStoreOptions, type CreateSitemapStoreOptions, type DateRange, type EmptyTypesDoc, type EmptyTypesStore, type FinalizeSitemapGenerationResult, INSPECTION_HISTORY_MAX_BYTES, type IndexingMetadataIndex, type IndexingMetadataRecord, type IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, type IterateSitemapGenerationUrlsOptions, type IterateSitemapGenerationUrlsResult, type ListSitemapGenerationUrlsOptions, type ListSitemapGenerationUrlsResult, type ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, type QuerySitemapGenerationMembershipResult, type RecordSitemapLegacyImportResult, SitemapFeedIdentityResult, type SitemapGenerationFeedManifest, SitemapGenerationKey, type SitemapGenerationReadResult, type SitemapLegacyImportEvidence, type SitemapMembershipEvent, type SitemapMutation, type SitemapReadStore, type SitemapSiteGenerationManifest, type SitemapStagedFeed, type SitemapStore, type SitemapUrlRecord, type StageSitemapGenerationFeedResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/entities.mjs

@@ -1,9 +1,8 @@
-import { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey } from "./entity-keys.mjs";
+import { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix } from "./entity-keys.mjs";
 import { createEmptyTypesStore } from "./entities/empty-types.mjs";
 import { createIndexingMetadataStore } from "./entities/indexing-metadata.mjs";
 import { INSPECTION_HISTORY_MAX_BYTES, createInspectionStore } from "./entities/inspection.mjs";
 import { buildQueryDimRecords, createQueryDimStore } from "./entities/query-dim.mjs";
-import { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed } from "./entities/sitemap-projection.mjs";
 import { createSitemapReadStore } from "./entities/sitemap.mjs";
 import { createSitemapStore } from "./entities/sitemap-write.mjs";
-export { INSPECTION_HISTORY_MAX_BYTES, SITEMAP_PROJECTION_GRACE_MS, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapProjectionManifest, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, selectSitemapProjectionFiles, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey, withSitemapProjectionFeed };
+export { INSPECTION_HISTORY_MAX_BYTES, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapReadStore, createSitemapStore, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/entity-keys.d.mts

@@ -20,35 +20,35 @@ declare function inspectionHistoryPrefix(ctx: TenantCtx, yearMonth: string): str
 declare function inspectionHistoryShardKey(ctx: TenantCtx, yearMonth: string, batchId: string): string;
 /** Stable, URL-safe FNV-1a hash used for entity keys. */
 declare function hashUrl(url: string): string;
-declare function sitemapIndexKey(ctx: TenantCtx): string;
-declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
 declare function sitemapUrlsPrefix(ctx: TenantCtx): string;
-declare function sitemapUrlsIndexPrefix(ctx: TenantCtx): string;
-declare function sitemapUrlsIndexKey(ctx: TenantCtx, feedpathHash: string): string;
-declare function sitemapUrlsProjectionManifestKey(ctx: TenantCtx): string;
 interface SitemapGenerationKey {
   id: string;
   observedAt: number;
 }
-declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, generation: SitemapGenerationKey): string;
-declare function parseSitemapUrlsDeltaKey(key: string): {
-  date: string;
-  feedpathHash: string;
-} | undefined;
 declare function sitemapUrlsEventsPrefix(ctx: TenantCtx): string;
 declare function sitemapUrlsEventKey(ctx: TenantCtx, feedpathHash: string, generation: SitemapGenerationKey): string;
-declare function sitemapUrlsEventSeedKey(ctx: TenantCtx, feedpathHash: string): string;
-declare function sitemapUrlsGenerationKey(ctx: TenantCtx, feedpathHash: string): string;
-declare function sitemapUrlsPendingGenerationsPrefix(ctx: TenantCtx): string;
-declare function sitemapUrlsPendingGenerationKey(ctx: TenantCtx, feedpathHash: string): string;
-declare function sitemapUrlsReconcileGenerationKey(ctx: TenantCtx): string;
-/** Hash a URL list for deterministic change detection. */
-declare function hashUrlList(urls: readonly {
+declare function sitemapSiteManifestKey(ctx: TenantCtx): string;
+declare function sitemapSiteManifestsPrefix(ctx: TenantCtx): string;
+declare function sitemapSiteGenerationManifestKey(ctx: TenantCtx, generation: SitemapGenerationKey): string;
+declare function sitemapStagedGenerationPrefix(ctx: TenantCtx, generation: SitemapGenerationKey): string;
+declare function sitemapStagedFeedKey(ctx: TenantCtx, generation: SitemapGenerationKey, feedpathHash: string): string;
+declare function sitemapImmutableBaseKey(ctx: TenantCtx, generation: SitemapGenerationKey, feedpathHash: string): string;
+declare function sitemapLegacyImportKey(ctx: TenantCtx): string;
+/** WHATWG-serialized HTTP(S) feed identity with the fragment removed. */
+type SitemapFeedIdentityResult = {
+  _tag: 'ok';
+  url: string;
+} | {
+  _tag: 'invalid';
+  reason: 'credentials' | 'empty' | 'invalid_url' | 'unsupported_protocol';
+};
+declare function parseSitemapFeedIdentity(input: string): SitemapFeedIdentityResult;
+/** Versioned exact payload hash. Lastmod-only changes are intentionally visible. */
+declare function sitemapPayloadHashV1(urls: readonly {
   loc: string;
+  lastmod?: string;
 }[]): string;
-/** Hash sorted URL strings as though joined by a newline, without allocating the join. */
-declare function hashSortedUrlList(locs: readonly string[]): string;
 declare function indexingMetadataIndexKey(ctx: TenantCtx): string;
 declare function queryDimParquetKey(ctx: TenantCtx): string;
 declare function queryDimMetaKey(ctx: TenantCtx): string;
-export { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };
+export { SitemapFeedIdentityResult, SitemapGenerationKey, emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/entity-keys.mjs

@@ -45,61 +45,71 @@ function hashUrl(url) {
 	}
 	return (hi >>> 0).toString(16).padStart(8, "0") + (lo >>> 0).toString(16).padStart(8, "0");
 }
-function sitemapIndexKey(ctx) {
-	return `${tenantEntityPrefix(ctx)}/sitemaps/index.json`;
-}
-function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
-	return `${tenantEntityPrefix(ctx)}/sitemaps/history/${feedpathHash}__${capturedAtMs}.json`;
-}
 function sitemapUrlsPrefix(ctx) {
 	return `${tenantEntityPrefix(ctx)}/sitemaps/urls`;
 }
-function sitemapUrlsIndexPrefix(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/by-feed`;
-}
-function sitemapUrlsIndexKey(ctx, feedpathHash) {
-	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
-}
-function sitemapUrlsProjectionManifestKey(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/projection.json`;
-}
 function sitemapGenerationDate(generation) {
 	return new Date(generation.observedAt).toISOString().slice(0, 10);
 }
-function sitemapUrlsDeltaKey(ctx, feedpathHash, generation) {
-	return `${sitemapUrlsPrefix(ctx)}/deltas/${sitemapGenerationDate(generation)}__${feedpathHash}__${String(generation.observedAt).padStart(13, "0")}__${hashUrl(generation.id)}.parquet`;
-}
-const SITEMAP_URLS_DELTA_KEY_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)(?:__\d+__[0-9a-f]+)?\.parquet$/;
-function parseSitemapUrlsDeltaKey(key) {
-	const match = SITEMAP_URLS_DELTA_KEY_RE.exec(key);
-	return match?.[1] && match[2] ? {
-		date: match[1],
-		feedpathHash: match[2]
-	} : void 0;
-}
 function sitemapUrlsEventsPrefix(ctx) {
 	return `${sitemapUrlsPrefix(ctx)}/events`;
 }
 function sitemapUrlsEventKey(ctx, feedpathHash, generation) {
 	return `${sitemapUrlsEventsPrefix(ctx)}/${sitemapGenerationDate(generation)}__${feedpathHash}__${String(generation.observedAt).padStart(13, "0")}__${hashUrl(generation.id)}.parquet`;
 }
-function sitemapUrlsEventSeedKey(ctx, feedpathHash) {
-	return `${sitemapUrlsPrefix(ctx)}/event-seeds/${feedpathHash}.json`;
+function sitemapSiteManifestKey(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/site-manifest.json`;
+}
+function sitemapSiteManifestsPrefix(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/site-manifests`;
 }
-function sitemapUrlsGenerationKey(ctx, feedpathHash) {
-	return `${sitemapUrlsPrefix(ctx)}/generations/by-feed/${feedpathHash}.json`;
+function sitemapSiteGenerationManifestKey(ctx, generation) {
+	return `${sitemapSiteManifestsPrefix(ctx)}/${String(generation.observedAt).padStart(13, "0")}__${hashUrl(generation.id)}.json`;
 }
-function sitemapUrlsPendingGenerationsPrefix(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/generations/pending`;
+function sitemapStagedGenerationPrefix(ctx, generation) {
+	return `${sitemapUrlsPrefix(ctx)}/staged/${String(generation.observedAt).padStart(13, "0")}__${hashUrl(generation.id)}`;
 }
-function sitemapUrlsPendingGenerationKey(ctx, feedpathHash) {
-	return `${sitemapUrlsPendingGenerationsPrefix(ctx)}/${feedpathHash}.json`;
+function sitemapStagedFeedKey(ctx, generation, feedpathHash) {
+	return `${sitemapStagedGenerationPrefix(ctx, generation)}/${feedpathHash}.json`;
 }
-function sitemapUrlsReconcileGenerationKey(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/generations/reconcile.json`;
+function sitemapImmutableBaseKey(ctx, generation, feedpathHash) {
+	return `${sitemapUrlsPrefix(ctx)}/bases/${feedpathHash}/${String(generation.observedAt).padStart(13, "0")}__${hashUrl(generation.id)}.parquet`;
 }
-function hashUrlList(urls) {
-	return hashSortedUrlList(urls.map((url) => url.loc).sort());
+function sitemapLegacyImportKey(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/legacy-import.json`;
+}
+function parseSitemapFeedIdentity(input) {
+	if (!input) return {
+		_tag: "invalid",
+		reason: "empty"
+	};
+	let parsed;
+	try {
+		parsed = new URL(input);
+	} catch {
+		return {
+			_tag: "invalid",
+			reason: "invalid_url"
+		};
+	}
+	if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return {
+		_tag: "invalid",
+		reason: "unsupported_protocol"
+	};
+	if (parsed.username || parsed.password) return {
+		_tag: "invalid",
+		reason: "credentials"
+	};
+	parsed.hash = "";
+	return {
+		_tag: "ok",
+		url: parsed.toString()
+	};
+}
+function sitemapPayloadHashV1(urls) {
+	const byLoc = /* @__PURE__ */ new Map();
+	for (const url of urls) byLoc.set(url.loc, url.lastmod ?? null);
+	return `v1:${hashSortedUrlList([...byLoc].map(([loc, lastmod]) => JSON.stringify([loc, lastmod])).sort())}`;
 }
 function hashSortedUrlList(locs) {
 	let hi = 2166136261;
@@ -131,4 +141,4 @@ function queryDimParquetKey(ctx) {
 function queryDimMetaKey(ctx) {
 	return `${queryDimPrefix(ctx)}/index.json`;
 }
-export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };
+export { emptyTypesKey, hashUrl, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, inspectionTransitionsMonthKey, inspectionTransitionsPrefix, parseSitemapFeedIdentity, queryDimMetaKey, queryDimParquetKey, sitemapImmutableBaseKey, sitemapLegacyImportKey, sitemapPayloadHashV1, sitemapSiteGenerationManifestKey, sitemapSiteManifestKey, sitemapSiteManifestsPrefix, sitemapStagedFeedKey, sitemapStagedGenerationPrefix, sitemapUrlsEventKey, sitemapUrlsEventsPrefix, sitemapUrlsPrefix };

package/dist/rollups/indexing.d.mts

@@ -22,18 +22,14 @@ declare const indexingMetadataRollup: RollupDef;
 declare const indexingHealthRollup: RollupDef;
 /**
  * Per-day index-percent: ratio of (sitemap URLs that received GSC clicks on
- * that date) / (total live sitemap URLs). Uses a DuckDB JOIN between the
- * sitemap urls parquet (`SitemapStore.urlsParquetUri`) and the `pages` fact
- * parquet. Total denominator is the count of live URLs in the urls index;
- * numerator is per-day distinct loc count where pages.clicks > 0.
+ * that date) / (total published sitemap URLs). The published generation
+ * manifest supplies the exact immutable feed bases. No storage listing or
+ * derived projection participates in authority.
  */
 declare const indexPercentRollup: RollupDef;
 /**
- * Sitemap-health per-day series materialized from the sitemap-store JSON
- * index. Each `SitemapRecord` carries `urlCount`, `errors`, `warnings`,
- * `contentHash`, and `lastDownloaded`. We bucket records by the day of their
- * `capturedAt` (or `lastDownloaded` fallback) and emit per-day aggregates plus
- * a snapshot of per-feed stats at the most recent capture.
+ * Sitemap-health history derives exclusively from published complete
+ * generations. The newest generation observed on each day is retained.
  */
 declare const sitemapHealthRollup: RollupDef;
 /**

package/dist/rollups/indexing.mjs

@@ -1,8 +1,7 @@
 import "../layout.mjs";
-import { inspectionParquetKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey } from "../entity-keys.mjs";
+import { inspectionParquetKey } from "../entity-keys.mjs";
 import { readOptional } from "../adapters/read-optional.mjs";
 import { createIndexingMetadataStore } from "../entities/indexing-metadata.mjs";
-import { decodeSitemapProjectionManifest, selectSitemapProjectionFiles } from "../entities/sitemap-projection.mjs";
 import { createSitemapReadStore } from "../entities/sitemap.mjs";
 import "../entities.mjs";
 import { utcDateMinusDays } from "./dates.mjs";
@@ -95,71 +94,26 @@ const indexingHealthRollup = {
 		})) };
 	}
 };
-function currentSitemapProjectionRelation(hasIndexes, hasDeltas) {
-	const sources = [];
-	if (hasIndexes) sources.push(`
-      SELECT
-        feedpath_hash,
-        url_hash,
-        loc,
-        removed_at,
-        greatest(
-          coalesce(removed_at, 0),
-          coalesce(last_seen_at, 0),
-          coalesce(first_seen_at, 0)
-        )::BIGINT AS observed_at,
-        0::INTEGER AS source_order
-      FROM read_parquet({{URLS_INDEX}}, union_by_name = true)
-    `);
-	if (hasDeltas) sources.push(`
-      SELECT
-        feedpath_hash,
-        url_hash,
-        loc,
-        CASE WHEN op = 'removed' THEN "at" ELSE NULL END AS removed_at,
-        "at"::BIGINT AS observed_at,
-        1::INTEGER AS source_order
-      FROM read_parquet({{URLS_DELTA}}, union_by_name = true)
-      WHERE op IN ('added', 'removed')
-    `);
-	return `(
-    WITH membership_events AS (
-      ${sources.join("\nUNION ALL\n")}
-    )
-    SELECT feedpath_hash, url_hash, loc, removed_at
-    FROM membership_events
-    QUALIFY row_number() OVER (
-      PARTITION BY feedpath_hash, url_hash
-      ORDER BY observed_at DESC, source_order DESC
-    ) = 1
-  )`;
-}
 const indexPercentRollup = {
 	id: "index_percent",
 	windowDays: 90,
 	sliceOrthogonal: true,
 	async build({ engine, ctx, dataSource, windowAnchorMs, searchType }) {
-		const [listedIndexKeys, listedDeltaKeys, manifestBytes] = await Promise.all([
-			dataSource.list(sitemapUrlsIndexPrefix(ctx)),
-			dataSource.list(`${sitemapUrlsPrefix(ctx)}/deltas/`),
-			readOptional(dataSource, sitemapUrlsProjectionManifestKey(ctx))
-		]);
-		const projection = selectSitemapProjectionFiles(listedIndexKeys, listedDeltaKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0);
-		if (projection.indexKeys.length === 0 && projection.deltaKeys.length === 0) return {
+		const generation = await createSitemapReadStore({ dataSource }).getSitemapGeneration(ctx);
+		if (generation._tag === "unavailable") return {
 			totalSitemapUrls: 0,
 			days: []
 		};
-		const sitemapFileSets = {
-			...projection.indexKeys.length > 0 ? { URLS_INDEX: {
-				table: "pages",
-				keys: projection.indexKeys
-			} } : {},
-			...projection.deltaKeys.length > 0 ? { URLS_DELTA: {
-				table: "pages",
-				keys: projection.deltaKeys
-			} } : {}
+		const baseKeys = Object.values(generation.manifest.feeds).sort((left, right) => left.feedpath.localeCompare(right.feedpath)).map((feed) => feed.baseKey);
+		if (baseKeys.length === 0) return {
+			totalSitemapUrls: 0,
+			days: []
 		};
-		const currentMembership = currentSitemapProjectionRelation(projection.indexKeys.length > 0, projection.deltaKeys.length > 0);
+		const sitemapFileSets = { SITEMAP_BASES: {
+			table: "pages",
+			keys: baseKeys
+		} };
+		const currentMembership = "read_parquet({{SITEMAP_BASES}}, union_by_name = true)";
 		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
 		const factSearchType = searchType ?? "web";
 		const pagesPartitions = partitionsInRange(await engine.listPartitions({
@@ -184,7 +138,7 @@ const indexPercentRollup = {
           COUNT(DISTINCT p.url)::BIGINT AS clicked_urls
         FROM read_parquet({{PAGES}}, union_by_name = true) p
         INNER JOIN ${currentMembership} s
-          ON s.loc = p.url AND s.removed_at IS NULL
+          ON s.loc = p.url
         WHERE p.clicks > 0 AND p.date >= '${cutoff}'
         GROUP BY p.date
         ORDER BY p.date
@@ -197,7 +151,6 @@ const indexPercentRollup = {
 			sql: `
         SELECT COUNT(DISTINCT loc)::BIGINT AS total
         FROM ${currentMembership}
-        WHERE removed_at IS NULL
       `
 		});
 		const total = Number(denom.rows[0]?.total ?? 0);
@@ -215,47 +168,64 @@ const indexPercentRollup = {
 		};
 	}
 };
+const SITEMAP_HEALTH_GENERATION_LIMIT = 1e4;
+async function loadSitemapGenerationAncestry(dataSource, current, cutoff) {
+	const ancestry = [];
+	const seen = /* @__PURE__ */ new Set();
+	let cursor = current;
+	while (cursor) {
+		if (new Date(cursor.observedAt).toISOString().slice(0, 10) < cutoff) break;
+		if (ancestry.length >= SITEMAP_HEALTH_GENERATION_LIMIT) throw new Error("sitemap health generation ancestry limit exceeded");
+		ancestry.push(cursor);
+		if (!cursor.previousManifestKey) break;
+		if (seen.has(cursor.previousManifestKey)) throw new Error("sitemap generation manifest ancestry cycle");
+		seen.add(cursor.previousManifestKey);
+		const bytes = await readOptional(dataSource, cursor.previousManifestKey);
+		if (!bytes) throw new Error("sitemap generation manifest ancestry is incomplete");
+		const previous = JSON.parse(new TextDecoder().decode(bytes));
+		if (previous.generationId !== cursor.previousGenerationId) throw new Error("sitemap generation manifest ancestry is incomplete");
+		cursor = previous;
+	}
+	return ancestry;
+}
 const sitemapHealthRollup = {
 	id: "sitemap_health",
 	windowDays: 90,
 	sliceOrthogonal: true,
 	async build({ dataSource, ctx, windowAnchorMs }) {
-		const index = await createSitemapReadStore({ dataSource }).loadIndex(ctx);
-		const records = Object.values(index.records);
+		const generation = await createSitemapReadStore({ dataSource }).getSitemapGeneration(ctx);
+		if (generation._tag === "unavailable") return {
+			days: [],
+			feeds: []
+		};
 		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
+		const ancestry = await loadSitemapGenerationAncestry(dataSource, generation.manifest, cutoff);
 		const byDay = /* @__PURE__ */ new Map();
-		const feeds = [];
-		for (const r of records) {
-			const day = (r.capturedAt ?? r.lastDownloaded ?? "").slice(0, 10);
-			if (!day || day < cutoff) continue;
-			const errors = Number(r.errors ?? 0);
-			const warnings = Number(r.warnings ?? 0);
-			const urlCount = Number(r.urlCount ?? 0);
-			const bucket = byDay.get(day) ?? {
+		for (const manifest of ancestry) {
+			const day = new Date(manifest.observedAt).toISOString().slice(0, 10);
+			if (byDay.has(day)) continue;
+			const manifestFeeds = Object.values(manifest.feeds);
+			byDay.set(day, {
 				day,
-				feeds: 0,
-				total_urls: 0,
+				feeds: manifestFeeds.length,
+				total_urls: manifestFeeds.reduce((total, feed) => total + feed.urlCount, 0),
 				errors: 0,
 				warnings: 0
-			};
-			bucket.feeds += 1;
-			bucket.total_urls += urlCount;
-			bucket.errors += errors;
-			bucket.warnings += warnings;
-			byDay.set(day, bucket);
-			feeds.push({
-				path: r.path,
-				urlCount,
-				errors,
-				warnings,
-				contentHash: r.contentHash ?? null,
-				lastDownloaded: r.lastDownloaded ?? null,
-				capturedAt: r.capturedAt
 			});
 		}
+		const days = Array.from(byDay.values()).sort((a, b) => a.day < b.day ? -1 : 1);
+		const capturedAt = new Date(generation.manifest.observedAt).toISOString();
 		return {
-			days: Array.from(byDay.values()).sort((a, b) => a.day < b.day ? -1 : 1),
-			feeds
+			days,
+			feeds: Object.values(generation.manifest.feeds).sort((left, right) => left.feedpath.localeCompare(right.feedpath)).map((feed) => ({
+				path: feed.feedpath,
+				urlCount: feed.urlCount,
+				errors: 0,
+				warnings: 0,
+				contentHash: feed.payloadHash,
+				lastDownloaded: null,
+				capturedAt
+			}))
 		};
 	}
 };
@@ -312,6 +282,7 @@ const sitemapChanges28dRollup = {
 			from,
 			to
 		})) {
+			if (d.op === "updated") continue;
 			const day = new Date(d.observedAt).toISOString().slice(0, 10);
 			const k = key({
 				day,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "1.5.4",
+  "version": "2.0.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -186,9 +186,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "^1.5.4",
-    "@gscdump/lakehouse": "^1.5.4",
-    "gscdump": "^1.5.4"
+    "@gscdump/lakehouse": "^2.0.0",
+    "@gscdump/contracts": "^2.0.0",
+    "gscdump": "^2.0.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",

package/dist/entities/sitemap-projection.d.mts

@@ -1,20 +0,0 @@
-declare const SITEMAP_PROJECTION_GRACE_MS: number;
-interface SitemapProjectionFeed {
-  compactedThrough: string;
-  /** Unix epoch milliseconds when readers started filtering through this key. */
-  publishedAt: number;
-}
-interface SitemapProjectionManifest {
-  version: 1;
-  feeds: Record<string, SitemapProjectionFeed>;
-}
-interface SitemapProjectionFiles {
-  indexKeys: string[];
-  deltaKeys: string[];
-}
-declare function emptySitemapProjectionManifest(): SitemapProjectionManifest;
-declare function parseSitemapProjectionManifest(value: unknown): SitemapProjectionManifest;
-declare function decodeSitemapProjectionManifest(text: string): SitemapProjectionManifest;
-declare function selectSitemapProjectionFiles(indexKeys: readonly string[], deltaKeys: readonly string[], manifest: SitemapProjectionManifest | undefined): SitemapProjectionFiles;
-declare function withSitemapProjectionFeed(manifest: SitemapProjectionManifest, feedpathHash: string, feed: SitemapProjectionFeed): SitemapProjectionManifest;
-export { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed };

package/dist/entities/sitemap-projection.mjs

@@ -1,57 +0,0 @@
-import { parseSitemapUrlsDeltaKey } from "../entity-keys.mjs";
-const SITEMAP_PROJECTION_GRACE_MS = 900 * 1e3;
-function emptySitemapProjectionManifest() {
-	return {
-		version: 1,
-		feeds: {}
-	};
-}
-function objectRecord(value) {
-	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
-}
-function parseSitemapProjectionManifest(value) {
-	const manifest = objectRecord(value);
-	const rawFeeds = objectRecord(manifest?.feeds);
-	if (manifest?.version !== 1 || !rawFeeds) throw new Error("invalid sitemap projection manifest");
-	const feeds = {};
-	for (const [feedpathHash, rawFeed] of Object.entries(rawFeeds)) {
-		const feed = objectRecord(rawFeed);
-		const compactedThrough = feed?.compactedThrough;
-		const publishedAt = feed?.publishedAt;
-		if (typeof compactedThrough !== "string") throw new Error(`invalid sitemap projection manifest feed: ${feedpathHash}`);
-		const parsedKey = parseSitemapUrlsDeltaKey(compactedThrough);
-		if (!/^[0-9a-f]+$/.test(feedpathHash) || parsedKey?.feedpathHash !== feedpathHash || !Number.isSafeInteger(publishedAt) || Number(publishedAt) < 0) throw new Error(`invalid sitemap projection manifest feed: ${feedpathHash}`);
-		feeds[feedpathHash] = {
-			compactedThrough,
-			publishedAt: Number(publishedAt)
-		};
-	}
-	return {
-		version: 1,
-		feeds
-	};
-}
-function decodeSitemapProjectionManifest(text) {
-	return parseSitemapProjectionManifest(JSON.parse(text));
-}
-function selectSitemapProjectionFiles(indexKeys, deltaKeys, manifest) {
-	return {
-		indexKeys: indexKeys.filter((key) => key.endsWith("/index.parquet")),
-		deltaKeys: deltaKeys.filter((key) => {
-			const parsed = parseSitemapUrlsDeltaKey(key);
-			if (!parsed) return false;
-			const feed = manifest?.feeds[parsed.feedpathHash];
-			return !feed || key > feed.compactedThrough;
-		})
-	};
-}
-function withSitemapProjectionFeed(manifest, feedpathHash, feed) {
-	return {
-		version: 1,
-		feeds: {
-			...manifest.feeds,
-			[feedpathHash]: feed
-		}
-	};
-}
-export { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed };