@gscdump/engine 0.19.6 → 0.20.0

package/dist/entities.d.mts

@@ -200,7 +200,8 @@ interface SitemapHistoryDoc {
 }
 declare function sitemapIndexKey(ctx: TenantCtx): string;
 declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
-declare function sitemapUrlsIndexKey(ctx: TenantCtx): string;
+declare function sitemapUrlsIndexPrefix(ctx: TenantCtx): string;
+declare function sitemapUrlsIndexKey(ctx: TenantCtx, feedpathHash: string): string;
 declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, date: string): string;
 /** Parsed URL entry from a sitemap XML. */
 interface ParsedUrl {
@@ -273,12 +274,12 @@ interface SitemapStore {
   /** Stream all delta entries within `[from, to]` (YYYY-MM-DD inclusive). */
   loadDeltas: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<DeltaEntry>;
   /**
-   * Fold every accumulated delta into the prior index; writes a fresh
-   * `urls/index.parquet` and deletes the consumed delta files.
+   * Fold accumulated deltas into the prior index, one feedpath at a time:
+   * rewrites each touched feedpath's `by-feed/<hash>/index.parquet` and deletes
+   * the consumed delta files. Bounded per feedpath, so it stays within memory
+   * regardless of total site URL count.
    */
   compactUrls: (ctx: TenantCtx) => Promise<void>;
-  /** DuckDB-resolvable URI for the URLs index; `undefined` if backend lacks one. */
-  urlsParquetUri: (ctx: TenantCtx) => string | undefined;
 }
 interface CreateSitemapStoreOptions {
   dataSource: DataSource;
@@ -330,4 +331,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/entities.mjs

@@ -190,8 +190,11 @@ function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
 function sitemapUrlsPrefix(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/urls` : `u_${ctx.userId}/entities/sitemaps/urls`;
 }
-function sitemapUrlsIndexKey(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/index.parquet`;
+function sitemapUrlsIndexPrefix(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/by-feed`;
+}
+function sitemapUrlsIndexKey(ctx, feedpathHash) {
+	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
 }
 function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
 	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
@@ -417,13 +420,12 @@ function createSitemapStore(opts) {
 		async *loadUrls(ctx, feedpath, opts) {
 			const fpHash = hash(feedpath);
 			const includeRemoved = opts?.includeRemoved ?? false;
-			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx)).catch(() => void 0);
-			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes, { filter: { feedpath_hash: { $eq: fpHash } } }) : [];
+			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx, fpHash)).catch(() => void 0);
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
 			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
 			const live = /* @__PURE__ */ new Map();
 			const removedMap = /* @__PURE__ */ new Map();
 			for (const row of indexRows) {
-				if (row.feedpath_hash !== fpHash) continue;
 				const rec = rowToUrlRecord(row);
 				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
 				else live.set(rec.urlHash, rec);
@@ -492,71 +494,67 @@ function createSitemapStore(opts) {
 			}
 		},
 		async compactUrls(ctx) {
-			const indexKey = sitemapUrlsIndexKey(ctx);
-			const indexBytes = await ds.read(indexKey).catch(() => void 0);
-			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
-			const stateKey = (fp, u) => `${fp}::${u}`;
-			const live = /* @__PURE__ */ new Map();
-			const removed = /* @__PURE__ */ new Map();
-			for (const row of indexRows) {
-				const rec = rowToUrlRecord(row);
-				const k = stateKey(rec.feedpathHash, rec.urlHash);
-				if (rec.removedAt != null) removed.set(k, rec);
-				else live.set(k, rec);
-			}
-			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
-			const consumed = [];
+			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
+			const deltasByFeed = /* @__PURE__ */ new Map();
 			for (const key of deltaKeys) {
 				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
 				if (!m) continue;
-				const fpHash = m[2];
-				const bytes = await ds.read(key).catch(() => void 0);
-				if (!bytes) continue;
-				consumed.push(key);
-				const rows = await decodeParquetToRows(bytes);
-				for (const r of rows) {
-					const urlHash = String(r.url_hash);
-					const at = Number(r.at);
-					const k = stateKey(fpHash, urlHash);
-					const op = String(r.op);
-					if (op === "added") {
-						const prev = live.get(k) ?? removed.get(k);
-						removed.delete(k);
-						live.set(k, {
-							feedpath: String(r.feedpath),
-							feedpathHash: fpHash,
-							urlHash,
-							loc: String(r.loc),
-							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-							firstSeenAt: prev?.firstSeenAt ?? at,
-							lastSeenAt: at
-						});
-					} else if (op === "removed") {
-						const prev = live.get(k);
-						live.delete(k);
-						if (prev) removed.set(k, {
-							...prev,
-							removedAt: at
-						});
+				const list = deltasByFeed.get(m[2]) ?? [];
+				list.push(key);
+				deltasByFeed.set(m[2], list);
+			}
+			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
+				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
+				const indexBytes = await ds.read(indexKey).catch(() => void 0);
+				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+				const live = /* @__PURE__ */ new Map();
+				const removed = /* @__PURE__ */ new Map();
+				for (const row of indexRows) {
+					const rec = rowToUrlRecord(row);
+					if (rec.removedAt != null) removed.set(rec.urlHash, rec);
+					else live.set(rec.urlHash, rec);
+				}
+				const consumed = [];
+				for (const key of feedDeltaKeys.sort()) {
+					const bytes = await ds.read(key).catch(() => void 0);
+					if (!bytes) continue;
+					consumed.push(key);
+					const rows = await decodeParquetToRows(bytes);
+					for (const r of rows) {
+						const urlHash = String(r.url_hash);
+						const at = Number(r.at);
+						const op = String(r.op);
+						if (op === "added") {
+							const prev = live.get(urlHash) ?? removed.get(urlHash);
+							removed.delete(urlHash);
+							live.set(urlHash, {
+								feedpath: String(r.feedpath),
+								feedpathHash: fpHash,
+								urlHash,
+								loc: String(r.loc),
+								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+								firstSeenAt: prev?.firstSeenAt ?? at,
+								lastSeenAt: at
+							});
+						} else if (op === "removed") {
+							const prev = live.get(urlHash);
+							live.delete(urlHash);
+							if (prev) removed.set(urlHash, {
+								...prev,
+								removedAt: at
+							});
+						}
 					}
 				}
+				const merged = [...live.values(), ...removed.values()];
+				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+					columns: URLS_INDEX_COLUMNS,
+					sortKey: ["feedpath_hash", "url_hash"]
+				});
+				await ds.write(indexKey, bytes);
+				if (consumed.length > 0) await ds.delete(consumed);
 			}
-			const merged = [...live.values(), ...removed.values()];
-			merged.sort((a, b) => {
-				if (a.feedpathHash !== b.feedpathHash) return a.feedpathHash < b.feedpathHash ? -1 : 1;
-				if (a.urlHash !== b.urlHash) return a.urlHash < b.urlHash ? -1 : 1;
-				return 0;
-			});
-			const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
-				columns: URLS_INDEX_COLUMNS,
-				sortKey: ["feedpath_hash", "url_hash"]
-			});
-			await ds.write(indexKey, bytes);
-			if (consumed.length > 0) await ds.delete(consumed);
-		},
-		urlsParquetUri(ctx) {
-			const key = sitemapUrlsIndexKey(ctx);
-			return ds.uri ? ds.uri(key) : void 0;
 		}
 	};
 }
@@ -640,4 +638,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };
+export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/resolver/index.d.mts

@@ -119,7 +119,8 @@ interface RunSQLFn {
     table: TableName$1;
     fileSets: Record<string, {
       table: TableName$1;
-      partitions: string[];
+      partitions?: string[];
+      keys?: string[];
     }>;
     sql: string;
     params: unknown[];

package/dist/rollups.d.mts

@@ -30,9 +30,9 @@ interface RollupEngine {
   /**
    * Read the live manifest for a (tenant, table[, searchType]) cohort —
    * cheap, no parquet decode. Builders use this to chunk a full-history scan
-   * into byte-bounded windows so a single `runSQL` call never has to ship
-   * more than ~14MB of decoded rows across the Workers service-binding RPC
-   * (32MiB hard cap).
+   * into byte-bounded windows (see `WINDOW_BYTE_BUDGET`) so a single `runSQL`
+   * call never ships an oversized Arrow IPC payload across the Workers
+   * service-binding RPC (32MiB hard cap).
    */
   listPartitions: (opts: {
     ctx: TenantCtx;
@@ -174,8 +174,15 @@ interface RebuildRollupResult {
 }
 declare function rebuildRollups(opts: RebuildRollupsOptions): Promise<RebuildRollupResult[]>;
 /**
- * Target decoded-bytes budget per window. Sits well under the 28MiB executor
- *  guard so headroom remains for SQL + result rows.
+ * Per-window budget, measured in *parquet* bytes (manifest `bytes`), used by
+ * `planRollupWindows` to chunk a full-history scan.
+ *
+ * The executor decodes a window's parquet and ships it as an Arrow IPC stream
+ * over the service binding; that IPC is hard-guarded at 28MiB
+ * (`IPC_PLACEHOLDER_BUDGET` in @gscdump/cloudflare). Parquet is compressed and
+ * the IPC stream is not, so a window inflates on the wire — keep this
+ * conservatively below the guard. Re-measure the parquet→IPC ratio against
+ * production and raise if headroom allows.
  */
 declare const WINDOW_BYTE_BUDGET: number;
 /**

package/dist/rollups.mjs

@@ -1,5 +1,6 @@
+import "./_chunks/storage.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexKey } from "./entities.mjs";
+import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx, searchType) {
 	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
@@ -111,7 +112,7 @@ function utcDateMinusDays(at, days) {
 	const d = new Date(at - days * MS_PER_DAY);
 	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
 }
-const WINDOW_BYTE_BUDGET = 14 * 1024 * 1024;
+const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
 const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
 const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
 const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
@@ -612,16 +613,17 @@ const indexPercentRollup = {
 	windowDays: 90,
 	sliceOrthogonal: true,
 	async build({ engine, ctx, dataSource, builtAt, searchType }) {
-		const urlsKey = sitemapUrlsIndexKey(ctx);
-		if (!await dataSource.head?.(urlsKey)) return {
+		const urlsKeys = await dataSource.list(sitemapUrlsIndexPrefix(ctx));
+		if (urlsKeys.length === 0) return {
 			totalSitemapUrls: 0,
 			days: []
 		};
 		const cutoff = utcDateMinusDays(builtAt, 90);
+		const factSearchType = searchType ?? "web";
 		const pagesPartitions = partitionsInRange(await engine.listPartitions({
 			ctx,
 			table: "pages",
-			...searchType !== void 0 ? { searchType } : {}
+			searchType: factSearchType
 		}), cutoff, utcDateMinusDays(builtAt, 0));
 		const numerator = await engine.runSQL({
 			ctx,
@@ -633,10 +635,10 @@ const indexPercentRollup = {
 				},
 				URLS: {
 					table: "pages",
-					keys: [urlsKey]
+					keys: urlsKeys
 				}
 			},
-			...searchType !== void 0 ? { searchType } : {},
+			searchType: factSearchType,
 			sql: `
         SELECT
           p.date AS date,
@@ -654,7 +656,7 @@ const indexPercentRollup = {
 			table: "pages",
 			fileSets: { URLS: {
 				table: "pages",
-				keys: [urlsKey]
+				keys: urlsKeys
 			} },
 			sql: `
         SELECT COUNT(*)::BIGINT AS total

package/dist/source/index.d.mts

@@ -90,8 +90,8 @@ interface EngineQuerySourceOptions {
 declare function createEngineQuerySource(options: EngineQuerySourceOptions): AnalysisQuerySource;
 /**
  * Convenience: wrap a storage engine + tenant ctx in a source and dispatch.
- * Equivalent to
- * `runAnalyzerFromSource(createEngineQuerySource({ engine, ctx }), params, registry)`.
+ * Equivalent to wrapping `createEngineQuerySource`, with omitted searchType
+ * defaulted to web at this public helper boundary.
  */
 declare function runAnalyzerWithEngine(deps: {
   engine: StorageEngine;

package/dist/source/index.mjs

@@ -1,4 +1,5 @@
 import { n as coerceRows } from "../_chunks/coerce.mjs";
+import "../_chunks/storage.mjs";
 import { T as assertDimensionsSupported, a as pgResolverAdapter, c as getFilterDimensions, v as resolveToSQL } from "../_chunks/resolver.mjs";
 import { n as runAnalyzerFromSource } from "../_chunks/dispatch.mjs";
 var AttachedTableMissingError = class extends Error {
@@ -127,7 +128,8 @@ function createEngineQuerySource(options) {
 async function runAnalyzerWithEngine(deps, ctx, params, registry) {
 	return runAnalyzerFromSource(createEngineQuerySource({
 		engine: deps.engine,
-		ctx
+		ctx,
+		searchType: params.searchType ?? "web"
 	}), params, registry);
 }
 function typedQuery(state) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.19.6",
+  "version": "0.20.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.19.6",
-    "@gscdump/contracts": "0.19.6"
+    "@gscdump/contracts": "0.20.0",
+    "gscdump": "0.20.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",