@gscdump/engine 0.19.7 → 0.20.1

package/dist/resolver/index.d.mts

@@ -119,7 +119,8 @@ interface RunSQLFn {
     table: TableName$1;
     fileSets: Record<string, {
       table: TableName$1;
-      partitions: string[];
+      partitions?: string[];
+      keys?: string[];
     }>;
     sql: string;
     params: unknown[];

package/dist/rollups.d.mts

@@ -30,9 +30,9 @@ interface RollupEngine {
   /**
    * Read the live manifest for a (tenant, table[, searchType]) cohort —
    * cheap, no parquet decode. Builders use this to chunk a full-history scan
-   * into byte-bounded windows so a single `runSQL` call never has to ship
-   * more than ~14MB of decoded rows across the Workers service-binding RPC
-   * (32MiB hard cap).
+   * into byte-bounded windows (see `WINDOW_BYTE_BUDGET`) so a single `runSQL`
+   * call never ships an oversized Arrow IPC payload across the Workers
+   * service-binding RPC (32MiB hard cap).
    */
   listPartitions: (opts: {
     ctx: TenantCtx;
@@ -174,8 +174,15 @@ interface RebuildRollupResult {
 }
 declare function rebuildRollups(opts: RebuildRollupsOptions): Promise<RebuildRollupResult[]>;
 /**
- * Target decoded-bytes budget per window. Sits well under the 28MiB executor
- *  guard so headroom remains for SQL + result rows.
+ * Per-window budget, measured in *parquet* bytes (manifest `bytes`), used by
+ * `planRollupWindows` to chunk a full-history scan.
+ *
+ * The executor decodes a window's parquet and ships it as an Arrow IPC stream
+ * over the service binding; that IPC is hard-guarded at 28MiB
+ * (`IPC_PLACEHOLDER_BUDGET` in @gscdump/cloudflare). Parquet is compressed and
+ * the IPC stream is not, so a window inflates on the wire — keep this
+ * conservatively below the guard. Re-measure the parquet→IPC ratio against
+ * production and raise if headroom allows.
  */
 declare const WINDOW_BYTE_BUDGET: number;
 /**

package/dist/rollups.mjs

@@ -1,3 +1,4 @@
+import "./_chunks/storage.mjs";
 import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
 import { createIndexingMetadataStore, createSitemapStore, inspectionParquetKey, sitemapUrlsIndexPrefix } from "./entities.mjs";
 import { MS_PER_DAY } from "gscdump";
@@ -111,7 +112,7 @@ function utcDateMinusDays(at, days) {
 	const d = new Date(at - days * MS_PER_DAY);
 	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
 }
-const WINDOW_BYTE_BUDGET = 14 * 1024 * 1024;
+const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
 const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
 const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
 const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
@@ -618,10 +619,11 @@ const indexPercentRollup = {
 			days: []
 		};
 		const cutoff = utcDateMinusDays(builtAt, 90);
+		const factSearchType = searchType ?? "web";
 		const pagesPartitions = partitionsInRange(await engine.listPartitions({
 			ctx,
 			table: "pages",
-			...searchType !== void 0 ? { searchType } : {}
+			searchType: factSearchType
 		}), cutoff, utcDateMinusDays(builtAt, 0));
 		const numerator = await engine.runSQL({
 			ctx,
@@ -636,7 +638,7 @@ const indexPercentRollup = {
 					keys: urlsKeys
 				}
 			},
-			...searchType !== void 0 ? { searchType } : {},
+			searchType: factSearchType,
 			sql: `
         SELECT
           p.date AS date,

package/dist/source/index.d.mts

@@ -90,8 +90,8 @@ interface EngineQuerySourceOptions {
 declare function createEngineQuerySource(options: EngineQuerySourceOptions): AnalysisQuerySource;
 /**
  * Convenience: wrap a storage engine + tenant ctx in a source and dispatch.
- * Equivalent to
- * `runAnalyzerFromSource(createEngineQuerySource({ engine, ctx }), params, registry)`.
+ * Equivalent to wrapping `createEngineQuerySource`, with omitted searchType
+ * defaulted to web at this public helper boundary.
  */
 declare function runAnalyzerWithEngine(deps: {
   engine: StorageEngine;

package/dist/source/index.mjs

@@ -1,4 +1,5 @@
 import { n as coerceRows } from "../_chunks/coerce.mjs";
+import "../_chunks/storage.mjs";
 import { T as assertDimensionsSupported, a as pgResolverAdapter, c as getFilterDimensions, v as resolveToSQL } from "../_chunks/resolver.mjs";
 import { n as runAnalyzerFromSource } from "../_chunks/dispatch.mjs";
 var AttachedTableMissingError = class extends Error {
@@ -127,7 +128,8 @@ function createEngineQuerySource(options) {
 async function runAnalyzerWithEngine(deps, ctx, params, registry) {
 	return runAnalyzerFromSource(createEngineQuerySource({
 		engine: deps.engine,
-		ctx
+		ctx,
+		searchType: params.searchType ?? "web"
 	}), params, registry);
 }
 function typedQuery(state) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.19.7",
+  "version": "0.20.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.19.7",
-    "@gscdump/contracts": "0.19.7"
+    "gscdump": "0.20.1",
+    "@gscdump/contracts": "0.20.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
@@ -178,7 +178,7 @@
     "aws4fetch": "^1.0.20",
     "hyparquet": "^1.25.8",
     "hyparquet-writer": "^0.15.1",
-    "tsx": "^4.22.2",
+    "tsx": "^4.22.3",
     "vitest": "^4.1.6"
   },
   "scripts": {