@gscdump/engine 0.19.6 → 0.19.7

package/dist/entities.d.mts

@@ -200,7 +200,8 @@ interface SitemapHistoryDoc {
 }
 declare function sitemapIndexKey(ctx: TenantCtx): string;
 declare function sitemapHistoryKey(ctx: TenantCtx, feedpathHash: string, capturedAtMs: number): string;
-declare function sitemapUrlsIndexKey(ctx: TenantCtx): string;
+declare function sitemapUrlsIndexPrefix(ctx: TenantCtx): string;
+declare function sitemapUrlsIndexKey(ctx: TenantCtx, feedpathHash: string): string;
 declare function sitemapUrlsDeltaKey(ctx: TenantCtx, feedpathHash: string, date: string): string;
 /** Parsed URL entry from a sitemap XML. */
 interface ParsedUrl {
@@ -273,12 +274,12 @@ interface SitemapStore {
   /** Stream all delta entries within `[from, to]` (YYYY-MM-DD inclusive). */
   loadDeltas: (ctx: TenantCtx, dateRange?: DateRange) => AsyncIterable<DeltaEntry>;
   /**
-   * Fold every accumulated delta into the prior index; writes a fresh
-   * `urls/index.parquet` and deletes the consumed delta files.
+   * Fold accumulated deltas into the prior index, one feedpath at a time:
+   * rewrites each touched feedpath's `by-feed/<hash>/index.parquet` and deletes
+   * the consumed delta files. Bounded per feedpath, so it stays within memory
+   * regardless of total site URL count.
    */
   compactUrls: (ctx: TenantCtx) => Promise<void>;
-  /** DuckDB-resolvable URI for the URLs index; `undefined` if backend lacks one. */
-  urlsParquetUri: (ctx: TenantCtx) => string | undefined;
 }
 interface CreateSitemapStoreOptions {
   dataSource: DataSource;
@@ -330,4 +331,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };
+export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/entities.mjs

@@ -190,8 +190,11 @@ function sitemapHistoryKey(ctx, feedpathHash, capturedAtMs) {
 function sitemapUrlsPrefix(ctx) {
 	return ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/entities/sitemaps/urls` : `u_${ctx.userId}/entities/sitemaps/urls`;
 }
-function sitemapUrlsIndexKey(ctx) {
-	return `${sitemapUrlsPrefix(ctx)}/index.parquet`;
+function sitemapUrlsIndexPrefix(ctx) {
+	return `${sitemapUrlsPrefix(ctx)}/by-feed`;
+}
+function sitemapUrlsIndexKey(ctx, feedpathHash) {
+	return `${sitemapUrlsIndexPrefix(ctx)}/${feedpathHash}/index.parquet`;
 }
 function sitemapUrlsDeltaKey(ctx, feedpathHash, date) {
 	return `${sitemapUrlsPrefix(ctx)}/deltas/${date}__${feedpathHash}.parquet`;
@@ -417,13 +420,12 @@ function createSitemapStore(opts) {
 		async *loadUrls(ctx, feedpath, opts) {
 			const fpHash = hash(feedpath);
 			const includeRemoved = opts?.includeRemoved ?? false;
-			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx)).catch(() => void 0);
-			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes, { filter: { feedpath_hash: { $eq: fpHash } } }) : [];
+			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx, fpHash)).catch(() => void 0);
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
 			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
 			const live = /* @__PURE__ */ new Map();
 			const removedMap = /* @__PURE__ */ new Map();
 			for (const row of indexRows) {
-				if (row.feedpath_hash !== fpHash) continue;
 				const rec = rowToUrlRecord(row);
 				if (rec.removedAt != null) removedMap.set(rec.urlHash, rec);
 				else live.set(rec.urlHash, rec);
@@ -492,71 +494,67 @@ function createSitemapStore(opts) {
 			}
 		},
 		async compactUrls(ctx) {
-			const indexKey = sitemapUrlsIndexKey(ctx);
-			const indexBytes = await ds.read(indexKey).catch(() => void 0);
-			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
-			const stateKey = (fp, u) => `${fp}::${u}`;
-			const live = /* @__PURE__ */ new Map();
-			const removed = /* @__PURE__ */ new Map();
-			for (const row of indexRows) {
-				const rec = rowToUrlRecord(row);
-				const k = stateKey(rec.feedpathHash, rec.urlHash);
-				if (rec.removedAt != null) removed.set(k, rec);
-				else live.set(k, rec);
-			}
-			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
-			const consumed = [];
+			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
+			const deltasByFeed = /* @__PURE__ */ new Map();
 			for (const key of deltaKeys) {
 				const m = SITEMAP_URLS_DELTA_PREFIX_RE.exec(key);
 				if (!m) continue;
-				const fpHash = m[2];
-				const bytes = await ds.read(key).catch(() => void 0);
-				if (!bytes) continue;
-				consumed.push(key);
-				const rows = await decodeParquetToRows(bytes);
-				for (const r of rows) {
-					const urlHash = String(r.url_hash);
-					const at = Number(r.at);
-					const k = stateKey(fpHash, urlHash);
-					const op = String(r.op);
-					if (op === "added") {
-						const prev = live.get(k) ?? removed.get(k);
-						removed.delete(k);
-						live.set(k, {
-							feedpath: String(r.feedpath),
-							feedpathHash: fpHash,
-							urlHash,
-							loc: String(r.loc),
-							lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
-							firstSeenAt: prev?.firstSeenAt ?? at,
-							lastSeenAt: at
-						});
-					} else if (op === "removed") {
-						const prev = live.get(k);
-						live.delete(k);
-						if (prev) removed.set(k, {
-							...prev,
-							removedAt: at
-						});
+				const list = deltasByFeed.get(m[2]) ?? [];
+				list.push(key);
+				deltasByFeed.set(m[2], list);
+			}
+			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
+				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
+				const indexBytes = await ds.read(indexKey).catch(() => void 0);
+				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+				const live = /* @__PURE__ */ new Map();
+				const removed = /* @__PURE__ */ new Map();
+				for (const row of indexRows) {
+					const rec = rowToUrlRecord(row);
+					if (rec.removedAt != null) removed.set(rec.urlHash, rec);
+					else live.set(rec.urlHash, rec);
+				}
+				const consumed = [];
+				for (const key of feedDeltaKeys.sort()) {
+					const bytes = await ds.read(key).catch(() => void 0);
+					if (!bytes) continue;
+					consumed.push(key);
+					const rows = await decodeParquetToRows(bytes);
+					for (const r of rows) {
+						const urlHash = String(r.url_hash);
+						const at = Number(r.at);
+						const op = String(r.op);
+						if (op === "added") {
+							const prev = live.get(urlHash) ?? removed.get(urlHash);
+							removed.delete(urlHash);
+							live.set(urlHash, {
+								feedpath: String(r.feedpath),
+								feedpathHash: fpHash,
+								urlHash,
+								loc: String(r.loc),
+								lastmod: r.lastmod == null ? void 0 : String(r.lastmod),
+								firstSeenAt: prev?.firstSeenAt ?? at,
+								lastSeenAt: at
+							});
+						} else if (op === "removed") {
+							const prev = live.get(urlHash);
+							live.delete(urlHash);
+							if (prev) removed.set(urlHash, {
+								...prev,
+								removedAt: at
+							});
+						}
 					}
 				}
+				const merged = [...live.values(), ...removed.values()];
+				merged.sort((a, b) => a.urlHash < b.urlHash ? -1 : a.urlHash > b.urlHash ? 1 : 0);
+				const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
+					columns: URLS_INDEX_COLUMNS,
+					sortKey: ["feedpath_hash", "url_hash"]
+				});
+				await ds.write(indexKey, bytes);
+				if (consumed.length > 0) await ds.delete(consumed);
 			}
-			const merged = [...live.values(), ...removed.values()];
-			merged.sort((a, b) => {
-				if (a.feedpathHash !== b.feedpathHash) return a.feedpathHash < b.feedpathHash ? -1 : 1;
-				if (a.urlHash !== b.urlHash) return a.urlHash < b.urlHash ? -1 : 1;
-				return 0;
-			});
-			const bytes = encodeRowsToParquetFlex(merged.map(urlRecordToRow), {
-				columns: URLS_INDEX_COLUMNS,
-				sortKey: ["feedpath_hash", "url_hash"]
-			});
-			await ds.write(indexKey, bytes);
-			if (consumed.length > 0) await ds.delete(consumed);
-		},
-		urlsParquetUri(ctx) {
-			const key = sitemapUrlsIndexKey(ctx);
-			return ds.uri ? ds.uri(key) : void 0;
 		}
 	};
 }
@@ -640,4 +638,4 @@ function createEmptyTypesStore(opts) {
 		}
 	};
 }
-export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey };
+export { INSPECTION_HISTORY_MAX_BYTES, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/dist/rollups.mjs

@@ -1,5 +1,5 @@
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexKey } from "./entities.mjs";
+import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump";
 function rollupPrefix(ctx, searchType) {
 	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
@@ -612,8 +612,8 @@ const indexPercentRollup = {
 	windowDays: 90,
 	sliceOrthogonal: true,
 	async build({ engine, ctx, dataSource, builtAt, searchType }) {
-		const urlsKey = sitemapUrlsIndexKey(ctx);
-		if (!await dataSource.head?.(urlsKey)) return {
+		const urlsKeys = await dataSource.list(sitemapUrlsIndexPrefix(ctx));
+		if (urlsKeys.length === 0) return {
 			totalSitemapUrls: 0,
 			days: []
 		};
@@ -633,7 +633,7 @@ const indexPercentRollup = {
 				},
 				URLS: {
 					table: "pages",
-					keys: [urlsKey]
+					keys: urlsKeys
 				}
 			},
 			...searchType !== void 0 ? { searchType } : {},
@@ -654,7 +654,7 @@ const indexPercentRollup = {
 			table: "pages",
 			fileSets: { URLS: {
 				table: "pages",
-				keys: [urlsKey]
+				keys: urlsKeys
 			} },
 			sql: `
         SELECT COUNT(*)::BIGINT AS total

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.19.6",
+  "version": "0.19.7",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.19.6",
-    "@gscdump/contracts": "0.19.6"
+    "gscdump": "0.19.7",
+    "@gscdump/contracts": "0.19.7"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",