@gscdump/engine 0.31.4 → 0.31.6

package/dist/_chunks/libs/icebird.d.mts

@@ -1,4 +1,4 @@
-import { AsyncBuffer } from "hyparquet";
+import { AsyncBuffer } from "./hyparquet.mjs";
 import { Writer } from "hyparquet-writer/src/types.js";
 interface WriterOptions {
   /**

package/dist/_chunks/libs/icebird.mjs

@@ -1,7 +1,6 @@
+import { asyncBufferFromUrl, cachedAsyncBuffer, parseDecimal } from "./hyparquet.mjs";
+import { ByteWriter, parquetWrite } from "./hyparquet-writer.mjs";
 import { gunzip } from "./hyparquet-compressors.mjs";
-import { asyncBufferFromUrl, cachedAsyncBuffer } from "hyparquet";
-import { ByteWriter, parquetWrite } from "hyparquet-writer";
-import { parseDecimal } from "hyparquet/src/convert.js";
 function readZigZag(reader) {
 	let result = 0;
 	let shift = 0;

package/dist/_chunks/storage.d.mts

@@ -1,4 +1,4 @@
-import { ParquetQueryFilter } from "hyparquet";
+import { ParquetQueryFilter } from "./libs/hyparquet.mjs";
 import { BuilderState, SearchType, SearchType as SearchType$1 } from "gscdump/query";
 import { Grain, Grain as Grain$1, Row, Row as Row$1, TableName, TableName as TableName$1, TenantCtx, TenantCtx as TenantCtx$1 } from "@gscdump/contracts";
 /**

package/dist/adapters/hyparquet.d.mts

@@ -1,6 +1,6 @@
 import { CodecCtx, DataSource, ParquetCodec, Row, TableName } from "../_chunks/storage.mjs";
+import { ParquetQueryFilter } from "../_chunks/libs/hyparquet.mjs";
 import { ColumnDef } from "../_chunks/schema.mjs";
-import { ParquetQueryFilter } from "hyparquet";
 declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
 interface EncodeFlexOptions {
   /** Columns defining the output schema + order. */

package/dist/adapters/hyparquet.mjs

@@ -1,6 +1,6 @@
 import { SCHEMAS, TABLE_METADATA, dedupeByNaturalKey } from "../_chunks/schema.mjs";
-import { parquetReadObjects } from "hyparquet";
-import { ByteWriter, parquetWriteRows } from "hyparquet-writer";
+import { parquetReadObjects } from "../_chunks/libs/hyparquet.mjs";
+import { ByteWriter$1 as ByteWriter, parquetWriteRows } from "../_chunks/libs/hyparquet-writer.mjs";
 const ROW_GROUP_SIZE = 25e3;
 function basicTypeFor(colType) {
 	if (colType === "VARCHAR") return "STRING";

package/dist/rollups.mjs

@@ -603,44 +603,61 @@ const queryCanonicalVariantsRollup = {
 		});
 		if (parts.length === 0) return [];
 		const partitions = parts.map((p) => p.partition);
-		return (await runPagedQuery({
-			engine,
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {},
-			fileSets: { FILES: {
+		const byCanonical = /* @__PURE__ */ new Map();
+		let cursor = null;
+		for (;;) {
+			const after = cursor === null ? "" : `AND query > '${cursor.replace(/'/g, "''")}'`;
+			const { rows } = await engine.runSQL({
+				ctx,
 				table: "queries",
-				partitions
-			} },
-			orderBy: "joinKey",
-			pageRows: ROLLUP_PAGE_ROWS_WIDE,
-			coreSql: `
-        WITH per_variant AS (
+				...searchType !== void 0 ? { searchType } : {},
+				fileSets: { FILES: {
+					table: "queries",
+					partitions
+				} },
+				sql: `
           SELECT
             COALESCE(NULLIF(query_canonical, ''), query) AS joinKey,
             query AS query,
             SUM(clicks) AS clicks,
             SUM(impressions) AS impressions,
-            SUM(sum_position) AS sum_pos,
-            ROW_NUMBER() OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query) ORDER BY SUM(clicks) DESC) AS rn,
-            COUNT(*) OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query)) AS variantCount
+            SUM(sum_position) AS sum_pos
           FROM read_parquet({{FILES}}, union_by_name = true)
+          WHERE query IS NOT NULL ${after}
           GROUP BY COALESCE(NULLIF(query_canonical, ''), query), query
-        )
-        SELECT
-          joinKey,
-          MAX(variantCount)::BIGINT AS variantCount,
-          MAX(CASE WHEN rn = 1 THEN query END) AS canonicalName,
-          GROUP_CONCAT(CASE WHEN rn <= 10 THEN query || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') AS variants
-        FROM per_variant
-        GROUP BY joinKey
-      `
-		})).map((r) => ({
-			joinKey: String(r.joinKey),
-			variantCount: BigInt(r.variantCount),
-			canonicalName: r.canonicalName == null ? null : String(r.canonicalName),
-			variants: r.variants == null ? null : String(r.variants)
-		}));
+          ORDER BY query
+          LIMIT ${ROLLUP_PAGE_ROWS_WIDE}
+        `
+			});
+			for (const r of rows) {
+				const joinKey = String(r.joinKey);
+				const list = byCanonical.get(joinKey);
+				const v = {
+					query: String(r.query),
+					clicks: Number(r.clicks),
+					impressions: Number(r.impressions),
+					sumPos: Number(r.sum_pos)
+				};
+				if (list) list.push(v);
+				else byCanonical.set(joinKey, [v]);
+			}
+			if (rows.length < 2e4) break;
+			cursor = String(rows[rows.length - 1].query);
+		}
+		const out = [];
+		for (const [joinKey, variants] of byCanonical) {
+			variants.sort((a, b) => b.clicks - a.clicks || a.query.localeCompare(b.query));
+			const canonicalName = variants[0]?.query ?? null;
+			const top = variants.slice(0, 10).filter((v) => v.impressions > 0);
+			const variantsStr = top.length === 0 ? null : top.map((v) => `${v.query}:::${v.clicks}:::${v.impressions}:::${(v.sumPos / v.impressions + 1).toFixed(1)}`).join("||");
+			out.push({
+				joinKey,
+				variantCount: BigInt(variants.length),
+				canonicalName,
+				variants: variantsStr
+			});
+		}
+		return out;
 	}
 };
 const queryCanonicalDailyRollup = {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.31.4",
+  "version": "0.31.6",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -171,33 +171,25 @@
     "node": ">=18"
   },
   "peerDependencies": {
-    "@duckdb/duckdb-wasm": "^1.32.0",
-    "hyparquet": "^1.26.1",
-    "hyparquet-writer": "^0.16.1"
+    "@duckdb/duckdb-wasm": "^1.32.0"
   },
   "peerDependenciesMeta": {
     "@duckdb/duckdb-wasm": {
       "optional": true
-    },
-    "hyparquet": {
-      "optional": true
-    },
-    "hyparquet-writer": {
-      "optional": true
     }
   },
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
-    "hyparquet": "^1.26.1",
-    "hyparquet-writer": "^0.16.1",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.31.4",
-    "@gscdump/contracts": "0.31.4"
+    "gscdump": "0.31.6",
+    "@gscdump/contracts": "0.31.6"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",
     "@types/proper-lockfile": "^4.1.4",
     "aws4fetch": "^1.0.20",
+    "hyparquet": "^1.26.1",
+    "hyparquet-writer": "^0.16.1",
     "icebird": "^0.8.10",
     "tsx": "^4.22.4",
     "unstorage": "^1.17.5",