@gscdump/engine 3.4.3 → 3.5.0

package/dist/resolver/compile.mjs

@@ -312,7 +312,7 @@ function buildExtrasQueries(state, options) {
 	const whereExpr = whereParts.length > 0 ? sql`WHERE ${joinAnd(whereParts)}` : sql``;
 	const outerQueryCol = sql.raw("query");
 	const canonKey = adapter.dimExprSql("queryCanonical", queriesKey);
-	const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${canonKey} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${canonKey} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${canonKey}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${canonKey}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, GROUP_CONCAT(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
+	const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${canonKey} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${canonKey} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${canonKey}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${canonKey}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, STRING_AGG(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
 	extras.push({
 		key: "canonicalExtras",
 		sql: compiled.sql,

package/dist/resolver/pg-adapter.d.mts

@@ -32,6 +32,7 @@ interface R2SqlResolverAdapterOptions extends ResolverAdapterOptions {
 declare function createParquetResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
 /**
  * Multi-tenant pg-flavored adapter for the Iceberg / R2 SQL read path.
+ * Set `dialect: 'r2sql'` to emit R2 SQL regex predicates. The default targets DuckDB.
  * Identical SQL output to `pgResolverAdapter` except WHERE clauses inject
  * `site_id = ?` AND `search_type = ?` automatically when those scopes are
  * passed to `resolveToSQL`. Required for the Iceberg fact tables which are
@@ -39,8 +40,15 @@ declare function createParquetResolverAdapter(options?: ResolverAdapterOptions):
  * cross-tenant data. Single-use: the adapter has no `tableRef` override,
  * so callers must rewrite bare table names to their qualified form (e.g.
  * `${namespace}.pages`) before sending to R2 SQL.
+ *
+ * The bare `site_id = ?` / `search_type = ?` partition predicates are only
+ * safe on int-partition catalogs. Legacy string-partition catalogs must pass
+ * `partitionKeyEncoding: 'string'` to emit `CONCAT(col, '') = ?` instead —
+ * bare equality undercounts (silently returns fewer rows) on those catalogs.
  */
-declare function createIcebergResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
+declare function createIcebergResolverAdapter(options?: R2SqlResolverAdapterOptions & {
+  dialect?: 'duckdb' | 'r2sql';
+}): ResolverAdapter<PgTableKey>;
 /**
  * R2 SQL adapter for the Iceberg fact tables.
  *

package/dist/resolver/pg-adapter.mjs

@@ -65,6 +65,14 @@ const pgResolverAdapter = createResolverAdapter({
 	...PG_BASE_CONFIG,
 	tableLabel: "pg-resolver-adapter"
 });
+function withPartitionKeyEncoding(adapter, encoding) {
+	if ((encoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "int") return adapter;
+	return {
+		...adapter,
+		siteIdColRef: (tk) => sql`CONCAT(${adapter.siteIdColRef(tk)}, '')`,
+		searchTypeColRef: (tk) => sql`CONCAT(${adapter.searchTypeColRef(tk)}, '')`
+	};
+}
 function createParquetResolverAdapter(options = {}) {
 	return createResolverAdapter({
 		...PG_BASE_CONFIG,
@@ -75,19 +83,20 @@ function createParquetResolverAdapter(options = {}) {
 	});
 }
 function createIcebergResolverAdapter(options = {}) {
-	return createResolverAdapter({
+	return withPartitionKeyEncoding(createResolverAdapter({
 		...PG_BASE_CONFIG,
 		schema: icebergSchema,
 		includeSiteId: true,
 		includeSearchType: true,
 		tableLabel: "iceberg-resolver-adapter",
+		regexPredicate: options.dialect === "r2sql" ? (expr, pattern, negate) => negate ? sql`NOT regexp_like(${expr}, ${pattern})` : sql`regexp_like(${expr}, ${pattern})` : PG_BASE_CONFIG.regexPredicate,
 		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
 		tableRef: (tk) => sql.raw(`"${tk}"`),
 		queryDimTableRef: () => sql.raw("\"query_dim\"")
-	});
+	}), options.partitionKeyEncoding);
 }
 function createR2SqlResolverAdapter(options = {}) {
-	const adapter = createResolverAdapter({
+	return withPartitionKeyEncoding(createResolverAdapter({
 		...PG_BASE_CONFIG,
 		schema: icebergSchema,
 		includeSiteId: true,
@@ -101,12 +110,6 @@ function createR2SqlResolverAdapter(options = {}) {
 		},
 		tableRef: (tk) => sql.raw(`"${tk}"`),
 		queryDimTableRef: () => sql.raw("\"query_dim\"")
-	});
-	if ((options.partitionKeyEncoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "int") return adapter;
-	return {
-		...adapter,
-		siteIdColRef: (tk) => sql`CONCAT(${adapter.siteIdColRef(tk)}, '')`,
-		searchTypeColRef: (tk) => sql`CONCAT(${adapter.searchTypeColRef(tk)}, '')`
-	};
+	}), options.partitionKeyEncoding);
 }
 export { createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "3.4.3",
+  "version": "3.5.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -189,10 +189,10 @@
     }
   },
   "dependencies": {
-    "@gscdump/contracts": "^3.4.3",
-    "@gscdump/lakehouse": "^3.4.3",
+    "@gscdump/contracts": "^3.5.0",
+    "@gscdump/lakehouse": "^3.5.0",
     "drizzle-orm": "1.0.0-rc.4",
-    "gscdump": "^3.4.3",
+    "gscdump": "^3.5.0",
     "proper-lockfile": "^4.1.2"
   },
   "devDependencies": {