@gscdump/engine 0.37.2 → 0.37.3

package/dist/_chunks/entities.mjs

@@ -758,7 +758,10 @@ function createSitemapStore(opts) {
 				}
 			}
 		},
-		async compactUrls(ctx) {
+		async compactUrls(ctx, opts = {}) {
+			const startedAt = now();
+			const deadlineMs = opts.deadlineMs ?? Number.POSITIVE_INFINITY;
+			const maxFeedpaths = opts.maxFeedpaths ?? Number.POSITIVE_INFINITY;
 			const deltaKeys = await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`);
 			const deltasByFeed = /* @__PURE__ */ new Map();
 			for (const key of deltaKeys) {
@@ -770,7 +773,10 @@ function createSitemapStore(opts) {
 				list.push(key);
 				deltasByFeed.set(feedpathHash, list);
 			}
+			const totalFeedpaths = deltasByFeed.size;
+			let compactedFeedpaths = 0;
 			for (const [fpHash, feedDeltaKeys] of deltasByFeed) {
+				if (compactedFeedpaths > 0 && (compactedFeedpaths >= maxFeedpaths || now() - startedAt >= deadlineMs)) break;
 				const indexKey = sitemapUrlsIndexKey(ctx, fpHash);
 				const indexBytes = await readOptional(ds, indexKey);
 				const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
@@ -821,7 +827,12 @@ function createSitemapStore(opts) {
 				});
 				await ds.write(indexKey, bytes);
 				if (consumed.length > 0) await ds.delete(consumed);
+				compactedFeedpaths++;
 			}
+			return {
+				compactedFeedpaths,
+				remainingFeedpaths: totalFeedpaths - compactedFeedpaths
+			};
 		},
 		async reconcile(ctx, { liveFeedpaths, at: atOpt }) {
 			const at = atOpt ?? now();

package/dist/entities.d.mts

@@ -379,6 +379,29 @@ interface ReconcileResult {
   /** Total URL rows transitioned live → removed across pruned feedpaths. */
   urlsRemoved: number;
 }
+/**
+ * Bounds on a single `compactUrls` call. `compactUrls` is memory-bounded (one
+ * feedpath at a time) but was previously unbounded in TIME: a tenant with a
+ * large accumulated delta backlog could run past a caller's execution budget.
+ * Both bounds are checked BETWEEN feedpaths and only after at least one has
+ * been compacted, so a call always makes forward progress.
+ */
+interface CompactUrlsOptions {
+  /** Stop before starting another feedpath once this many ms have elapsed. */
+  deadlineMs?: number;
+  /** Stop after compacting this many feedpaths. */
+  maxFeedpaths?: number;
+}
+interface CompactUrlsResult {
+  /** Feedpaths whose deltas were folded and deleted by this call. */
+  compactedFeedpaths: number;
+  /**
+   * Feedpaths that still hold outstanding deltas. Call `compactUrls` again to
+   * continue — no cursor is needed, because a compacted feedpath's deltas are
+   * deleted, so the next call simply doesn't see it.
+   */
+  remainingFeedpaths: number;
+}
 interface DeltaEntry {
   feedpath: string;
   feedpathHash: string;
@@ -428,8 +451,13 @@ interface SitemapStore {
    * rewrites each touched feedpath's `by-feed/<hash>/index.parquet` and deletes
    * the consumed delta files. Bounded per feedpath, so it stays within memory
    * regardless of total site URL count.
+   *
+   * Optionally bounded in TIME too (`opts.deadlineMs` / `opts.maxFeedpaths`).
+   * A bounded call is safe to stop mid-way: each feedpath's deltas are deleted
+   * as soon as its index is rewritten, so the remainder is simply what the next
+   * call finds. Callers drive this from `remainingFeedpaths`, not a cursor.
    */
-  compactUrls: (ctx: TenantCtx) => Promise<void>;
+  compactUrls: (ctx: TenantCtx, opts?: CompactUrlsOptions) => Promise<CompactUrlsResult>;
   /**
    * Site-wide convergence: mark every still-live URL whose owning feedpath is
    * absent from `liveFeedpaths` as removed. `compactUrls`/`snapshotUrls` only
@@ -496,4 +524,4 @@ interface CreateEmptyTypesStoreOptions {
   now?: () => number;
 }
 declare function createEmptyTypesStore(opts: CreateEmptyTypesStoreOptions): EmptyTypesStore;
-export { CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };
+export { CompactUrlsOptions, CompactUrlsResult, CreateEmptyTypesStoreOptions, CreateIndexingMetadataStoreOptions, CreateInspectionStoreOptions, CreateSitemapStoreOptions, DateRange, DeltaEntry, EmptyTypesDoc, EmptyTypesStore, INSPECTION_EVENT_COLUMNS, INSPECTION_HISTORY_MAX_BYTES, IndexingMetadataIndex, IndexingMetadataRecord, IndexingMetadataStore, InspectionEventRow, InspectionHistoryShard, InspectionIndex, InspectionParquetRow, InspectionRecord, InspectionStore, LoadUrlsOptions, ParsedUrl, QueryDimDeps, QueryDimMeta, QueryDimRecord, QueryDimStore, ReconcileResult, SitemapHistoryDoc, SitemapIndex, SitemapRecord, SitemapStore, SitemapUrlRecord, SnapshotUrlsResult, buildQueryDimRecords, createEmptyTypesStore, createIndexingMetadataStore, createInspectionStore, createQueryDimStore, createSitemapStore, emptyTypesKey, hashUrl, hashUrlList, indexingMetadataIndexKey, inspectionBaseKey, inspectionEventKey, inspectionEventsPrefix, inspectionHistoryPrefix, inspectionHistoryShardKey, inspectionIndexKey, inspectionParquetKey, queryDimMetaKey, queryDimParquetKey, sitemapHistoryKey, sitemapIndexKey, sitemapUrlsDeltaKey, sitemapUrlsIndexKey, sitemapUrlsIndexPrefix };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.37.2",
+  "version": "0.37.3",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,9 +181,9 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.37.2",
-    "gscdump": "0.37.2",
-    "@gscdump/lakehouse": "0.37.2"
+    "@gscdump/lakehouse": "0.37.3",
+    "@gscdump/contracts": "0.37.3",
+    "gscdump": "0.37.3"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",