@gscdump/engine 1.4.2 → 1.4.4

package/dist/entity-keys.d.mts

@@ -14,7 +14,23 @@ declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capture
 declare function sitemapUrlsPrefix(ctx: TenantCtx): string;
 declare function sitemapUrlsIndexPrefix(ctx: TenantCtx): string;
 declare function sitemapUrlsIndexKey(ctx: TenantCtx, feedpathHash: string): string;
-declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, date: string): string;
+declare function sitemapUrlsProjectionManifestKey(ctx: TenantCtx): string;
+interface SitemapGenerationKey {
+  id: string;
+  observedAt: number;
+}
+declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, generation: SitemapGenerationKey): string;
+declare function parseSitemapUrlsDeltaKey(key: string): {
+  date: string;
+  feedpathHash: string;
+} | undefined;
+declare function sitemapUrlsEventsPrefix(ctx: TenantCtx): string;
+declare function sitemapUrlsEventKey(ctx: TenantCtx, feedpathHash: string, generation: SitemapGenerationKey): string;
+declare function sitemapUrlsEventSeedKey(ctx: TenantCtx, feedpathHash: string): string;
+declare function sitemapUrlsGenerationKey(ctx: TenantCtx, feedpathHash: string): string;
+declare function sitemapUrlsPendingGenerationsPrefix(ctx: TenantCtx): string;
+declare function sitemapUrlsPendingGenerationKey(ctx: TenantCtx, feedpathHash: string): string;
+declare function sitemapUrlsReconcileGenerationKey(ctx: TenantCtx): string;
 /** Hash a URL list for deterministic change detection. */
 declare function hashUrlList(urls: readonly {
   loc: string;
@@ -24,4 +40,4 @@ declare function hashSortedUrlList(locs: readonly string[]): string;
 declare function indexingMetadataIndexKey(ctx: TenantCtx): string;
 declare function queryDimParquetKey(ctx: TenantCtx): string;
 declare function queryDimMetaKey(ctx: TenantCtx): string;
-export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };
+export { SitemapGenerationKey, emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };

package/dist/entity-keys.mjs

@@ -54,8 +54,43 @@ function sitemapUrlsIndexPrefix(ctx) {
 function sitemapUrlsIndexKey(ctx, feedpathHash) {
 	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
 }
-function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
-	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
+function sitemapUrlsProjectionManifestKey(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/projection.json`;
+}
+function sitemapGenerationDate(generation) {
+	return new Date(generation.observedAt).toISOString().slice(0, 10);
+}
+function sitemapUrlsDeltaKey(ctx, feedpathHash, generation) {
+	return `${sitemapUrlsPrefix(ctx)}/deltas/${sitemapGenerationDate(generation)}__${feedpathHash}__${String(generation.observedAt).padStart(13, "0")}__${hashUrl(generation.id)}.parquet`;
+}
+const SITEMAP_URLS_DELTA_KEY_RE = /\/urls\/deltas\/(\d{4}-\d{2}-\d{2})__([0-9a-f]+)(?:__\d+__[0-9a-f]+)?\.parquet$/;
+function parseSitemapUrlsDeltaKey(key) {
+	const match = SITEMAP_URLS_DELTA_KEY_RE.exec(key);
+	return match?.[1] && match[2] ? {
+		date: match[1],
+		feedpathHash: match[2]
+	} : void 0;
+}
+function sitemapUrlsEventsPrefix(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/events`;
+}
+function sitemapUrlsEventKey(ctx, feedpathHash, generation) {
+	return `${sitemapUrlsEventsPrefix(ctx)}/${sitemapGenerationDate(generation)}__${feedpathHash}__${String(generation.observedAt).padStart(13, "0")}__${hashUrl(generation.id)}.parquet`;
+}
+function sitemapUrlsEventSeedKey(ctx, feedpathHash) {
+	return `${sitemapUrlsPrefix(ctx)}/event-seeds/${feedpathHash}.json`;
+}
+function sitemapUrlsGenerationKey(ctx, feedpathHash) {
+	return `${sitemapUrlsPrefix(ctx)}/generations/by-feed/${feedpathHash}.json`;
+}
+function sitemapUrlsPendingGenerationsPrefix(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/generations/pending`;
+}
+function sitemapUrlsPendingGenerationKey(ctx, feedpathHash) {
+	return `${sitemapUrlsPendingGenerationsPrefix(ctx)}/${feedpathHash}.json`;
+}
+function sitemapUrlsReconcileGenerationKey(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/generations/reconcile.json`;
 }
 function hashUrlList(urls) {
 	return hashSortedUrlList(urls.map((url) => url.loc).sort());
@@ -90,4 +125,4 @@ function queryDimParquetKey(ctx) {
 function queryDimMetaKey(ctx) {
 	return `${queryDimPrefix(ctx)}/index.json`;
 }
-export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix };
+export { emptyTypesKey, hashSortedUrlList, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, parseSitemapUrlsDeltaKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsEventKey, sitemapUrlsEventSeedKey, sitemapUrlsEventsPrefix, sitemapUrlsGenerationKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix, sitemapUrlsPendingGenerationKey, sitemapUrlsPendingGenerationsPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey, sitemapUrlsReconcileGenerationKey };

package/dist/errors.d.mts

@@ -97,7 +97,7 @@ declare const engineErrors: {
   readonly invalidYearMonth: (value: string) => EngineError;
   readonly missingAttachUrl: (fileName: string) => EngineError;
   readonly manifestCasRoundLost: (siteId: string, table: string, attempt: number) => EngineError;
-  readonly icebergTableOpFailed: (op: 'create' | 'drop', table: string, cause: unknown) => EngineError;
+  readonly icebergTableOpFailed: (op: "create" | "drop", table: string, cause: unknown) => EngineError;
   readonly sinkTableFlushFailed: (table: string, cause: unknown) => EngineError;
   readonly rollupBuildFailed: (id: string, cause: unknown) => EngineError;
   readonly lockAcquireTimeout: (scope: string, timeoutMs: number) => EngineError;

package/dist/rollups.d.mts

@@ -423,8 +423,8 @@ declare const sitemapHealthRollup: RollupDef;
 /**
  * Trailing-28-day sitemap URL changes: per-day per-feedpath {added, removed}
  * counts plus rolling top-200 added and removed URLs. Streams from
- * `SitemapStore.loadDeltas()` so it scales independently of how many feeds
- * exist on the site.
+ * retained `SitemapReadStore.loadEvents()` history. State compaction cannot
+ * erase analytics input; memory scales independently of total site state.
  */
 declare const sitemapChanges28dRollup: RollupDef;
 /**

package/dist/rollups.mjs

@@ -1,9 +1,11 @@
 import { engineErrors } from "./errors.mjs";
 import "./layout.mjs";
-import { inspectionParquetKey, sitemapUrlsIndexPrefix } from "./entity-keys.mjs";
+import { inspectionParquetKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey } from "./entity-keys.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
+import { readOptional } from "./adapters/read-optional.mjs";
+import { decodeSitemapProjectionManifest, selectSitemapProjectionFiles } from "./sitemap-projection.mjs";
 import { createQueryDimStore } from "./query-dim.mjs";
-import { createIndexingMetadataStore, createSitemapStore } from "./entities.mjs";
+import { createIndexingMetadataStore, createSitemapReadStore } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump/dates";
 import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
 function rollupPrefix(ctx, searchType) {
@@ -1003,16 +1005,71 @@ const indexingHealthRollup = {
 		})) };
 	}
 };
+function currentSitemapProjectionRelation(hasIndexes, hasDeltas) {
+	const sources = [];
+	if (hasIndexes) sources.push(`
+      SELECT
+        feedpath_hash,
+        url_hash,
+        loc,
+        removed_at,
+        greatest(
+          coalesce(removed_at, 0),
+          coalesce(last_seen_at, 0),
+          coalesce(first_seen_at, 0)
+        )::BIGINT AS observed_at,
+        0::INTEGER AS source_order
+      FROM read_parquet({{URLS_INDEX}}, union_by_name = true)
+    `);
+	if (hasDeltas) sources.push(`
+      SELECT
+        feedpath_hash,
+        url_hash,
+        loc,
+        CASE WHEN op = 'removed' THEN "at" ELSE NULL END AS removed_at,
+        "at"::BIGINT AS observed_at,
+        1::INTEGER AS source_order
+      FROM read_parquet({{URLS_DELTA}}, union_by_name = true)
+      WHERE op IN ('added', 'removed')
+    `);
+	return `(
+    WITH membership_events AS (
+      ${sources.join("\nUNION ALL\n")}
+    )
+    SELECT feedpath_hash, url_hash, loc, removed_at
+    FROM membership_events
+    QUALIFY row_number() OVER (
+      PARTITION BY feedpath_hash, url_hash
+      ORDER BY observed_at DESC, source_order DESC
+    ) = 1
+  )`;
+}
 const indexPercentRollup = {
 	id: "index_percent",
 	windowDays: 90,
 	sliceOrthogonal: true,
 	async build({ engine, ctx, dataSource, windowAnchorMs, searchType }) {
-		const urlsKeys = await dataSource.list(sitemapUrlsIndexPrefix(ctx));
-		if (urlsKeys.length === 0) return {
+		const [listedIndexKeys, listedDeltaKeys, manifestBytes] = await Promise.all([
+			dataSource.list(sitemapUrlsIndexPrefix(ctx)),
+			dataSource.list(`${sitemapUrlsPrefix(ctx)}/deltas/`),
+			readOptional(dataSource, sitemapUrlsProjectionManifestKey(ctx))
+		]);
+		const projection = selectSitemapProjectionFiles(listedIndexKeys, listedDeltaKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0);
+		if (projection.indexKeys.length === 0 && projection.deltaKeys.length === 0) return {
 			totalSitemapUrls: 0,
 			days: []
 		};
+		const sitemapFileSets = {
+			...projection.indexKeys.length > 0 ? { URLS_INDEX: {
+				table: "pages",
+				keys: projection.indexKeys
+			} } : {},
+			...projection.deltaKeys.length > 0 ? { URLS_DELTA: {
+				table: "pages",
+				keys: projection.deltaKeys
+			} } : {}
+		};
+		const currentMembership = currentSitemapProjectionRelation(projection.indexKeys.length > 0, projection.deltaKeys.length > 0);
 		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
 		const factSearchType = searchType ?? "web";
 		const pagesPartitions = partitionsInRange(await engine.listPartitions({
@@ -1028,10 +1085,7 @@ const indexPercentRollup = {
 					table: "pages",
 					partitions: pagesPartitions
 				},
-				URLS: {
-					table: "pages",
-					keys: urlsKeys
-				}
+				...sitemapFileSets
 			},
 			searchType: factSearchType,
 			sql: `
@@ -1039,7 +1093,7 @@ const indexPercentRollup = {
           p.date AS date,
           COUNT(DISTINCT p.url)::BIGINT AS clicked_urls
         FROM read_parquet({{PAGES}}, union_by_name = true) p
-        INNER JOIN read_parquet({{URLS}}, union_by_name = true) s
+        INNER JOIN ${currentMembership} s
           ON s.loc = p.url AND s.removed_at IS NULL
         WHERE p.clicks > 0 AND p.date >= '${cutoff}'
         GROUP BY p.date
@@ -1049,13 +1103,10 @@ const indexPercentRollup = {
 		const denom = await engine.runSQL({
 			ctx,
 			table: "pages",
-			fileSets: { URLS: {
-				table: "pages",
-				keys: urlsKeys
-			} },
+			fileSets: sitemapFileSets,
 			sql: `
-        SELECT COUNT(*)::BIGINT AS total
-        FROM read_parquet({{URLS}}, union_by_name = true)
+        SELECT COUNT(DISTINCT loc)::BIGINT AS total
+        FROM ${currentMembership}
         WHERE removed_at IS NULL
       `
 		});
@@ -1079,7 +1130,7 @@ const sitemapHealthRollup = {
 	windowDays: 90,
 	sliceOrthogonal: true,
 	async build({ dataSource, ctx, windowAnchorMs }) {
-		const index = await createSitemapStore({ dataSource }).loadIndex(ctx);
+		const index = await createSitemapReadStore({ dataSource }).loadIndex(ctx);
 		const records = Object.values(index.records);
 		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
 		const byDay = /* @__PURE__ */ new Map();
@@ -1157,7 +1208,7 @@ const sitemapChanges28dRollup = {
 	windowDays: 28,
 	sliceOrthogonal: true,
 	async build({ dataSource, ctx, windowAnchorMs }) {
-		const store = createSitemapStore({ dataSource });
+		const store = createSitemapReadStore({ dataSource });
 		const from = utcDateMinusDays(windowAnchorMs, 28);
 		const to = utcDateMinusDays(windowAnchorMs, 0);
 		const counts = /* @__PURE__ */ new Map();
@@ -1167,11 +1218,11 @@ const sitemapChanges28dRollup = {
 		function key(k) {
 			return `${k.day}\x00${k.feedpath}`;
 		}
-		for await (const d of store.loadDeltas(ctx, {
+		for await (const d of store.loadEvents(ctx, {
 			from,
 			to
 		})) {
-			const day = new Date(d.at).toISOString().slice(0, 10);
+			const day = new Date(d.observedAt).toISOString().slice(0, 10);
 			const k = key({
 				day,
 				feedpath: d.feedpath
@@ -1185,7 +1236,7 @@ const sitemapChanges28dRollup = {
 			const change = {
 				loc: d.loc,
 				feedpath: d.feedpath,
-				at: d.at,
+				at: d.observedAt,
 				sequence: sequence++
 			};
 			if (d.op === "added") {

package/dist/sitemap-projection.d.mts

@@ -0,0 +1,20 @@
+declare const SITEMAP_PROJECTION_GRACE_MS: number;
+interface SitemapProjectionFeed {
+  compactedThrough: string;
+  /** Unix epoch milliseconds when readers started filtering through this key. */
+  publishedAt: number;
+}
+interface SitemapProjectionManifest {
+  version: 1;
+  feeds: Record<string, SitemapProjectionFeed>;
+}
+interface SitemapProjectionFiles {
+  indexKeys: string[];
+  deltaKeys: string[];
+}
+declare function emptySitemapProjectionManifest(): SitemapProjectionManifest;
+declare function parseSitemapProjectionManifest(value: unknown): SitemapProjectionManifest;
+declare function decodeSitemapProjectionManifest(text: string): SitemapProjectionManifest;
+declare function selectSitemapProjectionFiles(indexKeys: readonly string[], deltaKeys: readonly string[], manifest: SitemapProjectionManifest | undefined): SitemapProjectionFiles;
+declare function withSitemapProjectionFeed(manifest: SitemapProjectionManifest, feedpathHash: string, feed: SitemapProjectionFeed): SitemapProjectionManifest;
+export { SITEMAP_PROJECTION_GRACE_MS, SitemapProjectionFeed, SitemapProjectionFiles, SitemapProjectionManifest, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed };

package/dist/sitemap-projection.mjs

@@ -0,0 +1,57 @@
+import { parseSitemapUrlsDeltaKey } from "./entity-keys.mjs";
+const SITEMAP_PROJECTION_GRACE_MS = 900 * 1e3;
+function emptySitemapProjectionManifest() {
+	return {
+		version: 1,
+		feeds: {}
+	};
+}
+function objectRecord(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value) ? value : void 0;
+}
+function parseSitemapProjectionManifest(value) {
+	const manifest = objectRecord(value);
+	const rawFeeds = objectRecord(manifest?.feeds);
+	if (manifest?.version !== 1 || !rawFeeds) throw new Error("invalid sitemap projection manifest");
+	const feeds = {};
+	for (const [feedpathHash, rawFeed] of Object.entries(rawFeeds)) {
+		const feed = objectRecord(rawFeed);
+		const compactedThrough = feed?.compactedThrough;
+		const publishedAt = feed?.publishedAt;
+		if (typeof compactedThrough !== "string") throw new Error(`invalid sitemap projection manifest feed: ${feedpathHash}`);
+		const parsedKey = parseSitemapUrlsDeltaKey(compactedThrough);
+		if (!/^[0-9a-f]+$/.test(feedpathHash) || parsedKey?.feedpathHash !== feedpathHash || !Number.isSafeInteger(publishedAt) || Number(publishedAt) < 0) throw new Error(`invalid sitemap projection manifest feed: ${feedpathHash}`);
+		feeds[feedpathHash] = {
+			compactedThrough,
+			publishedAt: Number(publishedAt)
+		};
+	}
+	return {
+		version: 1,
+		feeds
+	};
+}
+function decodeSitemapProjectionManifest(text) {
+	return parseSitemapProjectionManifest(JSON.parse(text));
+}
+function selectSitemapProjectionFiles(indexKeys, deltaKeys, manifest) {
+	return {
+		indexKeys: indexKeys.filter((key) => key.endsWith("/index.parquet")),
+		deltaKeys: deltaKeys.filter((key) => {
+			const parsed = parseSitemapUrlsDeltaKey(key);
+			if (!parsed) return false;
+			const feed = manifest?.feeds[parsed.feedpathHash];
+			return !feed || key > feed.compactedThrough;
+		})
+	};
+}
+function withSitemapProjectionFeed(manifest, feedpathHash, feed) {
+	return {
+		version: 1,
+		feeds: {
+			...manifest.feeds,
+			[feedpathHash]: feed
+		}
+	};
+}
+export { SITEMAP_PROJECTION_GRACE_MS, decodeSitemapProjectionManifest, emptySitemapProjectionManifest, parseSitemapProjectionManifest, selectSitemapProjectionFiles, withSitemapProjectionFeed };

package/dist/sync-config.d.mts

@@ -13,11 +13,11 @@ declare function parseEnabledSearchTypes(raw: string | null | undefined): Search
 declare function validateEnabledSearchTypesResult(value: unknown): Result<SearchType[], EngineError>;
 declare function validateEnabledSearchTypes(value: unknown): SearchType[];
 declare const TABLE_TIERS: {
-  readonly pages: 'critical';
-  readonly queries: 'critical';
-  readonly countries: 'standard';
-  readonly dates: 'standard';
-  readonly page_queries: 'extended';
+  readonly pages: "critical";
+  readonly queries: "critical";
+  readonly countries: "standard";
+  readonly dates: "standard";
+  readonly page_queries: "extended";
 };
 type TieredTableName = keyof typeof TABLE_TIERS;
 type TableTier = 'critical' | 'standard' | 'extended';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "1.4.2",
+  "version": "1.4.4",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "^1.4.2",
-    "gscdump": "^1.4.2",
-    "@gscdump/lakehouse": "^1.4.2"
+    "@gscdump/lakehouse": "^1.4.4",
+    "gscdump": "^1.4.4",
+    "@gscdump/contracts": "^1.4.4"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "1.33.1-dev57.0",