@gscdump/engine 0.9.0 → 0.9.1

package/dist/_chunks/duckdb.d.mts

@@ -1,4 +1,4 @@
-import { N as TableName, S as QueryExecutor, h as ParquetCodec, w as Row } from "./storage.mjs";
+import { C as QueryExecutor, P as TableName, T as Row, g as ParquetCodec } from "./storage.mjs";
 interface DuckDBHandle {
   query: (sql: string, params?: unknown[]) => Promise<Row[]>;
   registerFileBuffer: (name: string, bytes: Uint8Array) => Promise<void>;

package/dist/_chunks/planner.d.mts

@@ -1,4 +1,4 @@
-import { N as TableName } from "./storage.mjs";
+import { P as TableName } from "./storage.mjs";
 import { LogicalQueryPlan } from "gscdump/query/plan";
 import { BuilderState } from "gscdump/query";
 interface ResolvedQuery {

package/dist/_chunks/registry.d.mts

@@ -1,4 +1,4 @@
-import { w as Row } from "./storage.mjs";
+import { T as Row } from "./storage.mjs";
 import { t as AnalysisParams } from "./analysis-types.mjs";
 import { r as FileSet } from "./source-types.mjs";
 import { BuilderState } from "gscdump/query";

package/dist/_chunks/storage.d.mts

@@ -307,6 +307,16 @@ interface ExtraResult {
   key: string;
   rows: Row[];
 }
+interface OptimizedQueryResult {
+  rows: Row[];
+  totalCount: number;
+  totals: {
+    clicks: number;
+    impressions: number;
+    ctr: number;
+    position: number;
+  };
+}
 interface QueryExecuteOptions {
   sql: string;
   params: unknown[];
@@ -403,6 +413,16 @@ interface StorageEngine {
    * when the state has no extras-eligible dimensions.
    */
   queryExtras: (ctx: QueryCtx, state: BuilderState) => Promise<ExtraResult[]>;
+  /**
+   * Single-scan variant of {@link query} that piggy-backs `totalCount` and
+   * unfiltered metric totals onto the dimensioned result via window functions.
+   * Replaces the host-side rows + totals + count fan-out with one DuckDB
+   * execution. Window-function output columns (`totalCount`, `totalClicks`,
+   * `totalImpressions`, `totalCtr`, `totalPosition`) are stripped from `rows`
+   * before return; missing per-metric totals (when the metric was not
+   * requested in `state.metrics`) default to 0.
+   */
+  queryOptimized: (ctx: QueryCtx, state: BuilderState) => Promise<OptimizedQueryResult>;
   /**
    * Run arbitrary SQL resolved against named partition sets. Composes
    * manifest lookup + object reads + placeholder substitution + execution
@@ -480,4 +500,4 @@ declare function mondayOfWeek(isoDate: string): string;
 /** YYYY-Qq for the quarter containing the given YYYY-MM month string. */
 declare function quarterOfMonth(month: string): string;
 declare function objectKey(ctx: TenantCtx, table: TableName, partition: string, version: number, searchType?: SearchType): string;
-export { SyncStateFilter as A, dayPartition as B, QueryResult as C, StorageEngine as D, SearchType$1 as E, Watermark as F, objectKey as G, inferSearchType as H, WatermarkFilter as I, weekPartition as J, quarterOfMonth as K, WatermarkScope as L, SyncStateScope as M, TableName$1 as N, SyncState as O, TenantCtx$1 as P, WriteCtx as R, QueryExecutor as S, RunSQLOptions as T, mondayOfWeek as U, inferLegacyTier as V, monthPartition as W, enumeratePartitions as X, CompactionThresholds as Y, PurgeResult as _, DataSource as a, QueryExecuteOptions as b, FileSetRef as c, LockScope as d, ManifestEntry as f, PurgeFilter as g, ParquetCodec as h, DEFAULT_SEARCH_TYPE as i, SyncStateKind as j, SyncStateDetail as k, GcCtx as l, ManifestStore as m, CompactionTier as n, EngineOptions as o, ManifestPurgeResult as p, quarterPartition as q, ComparisonResult as r, ExtraResult as s, CodecCtx as t, ListLiveFilter as u, PurgeUrlsResult as v, Row$1 as w, QueryExecuteResult as x, QueryCtx as y, WriteResult as z };
+export { SyncStateDetail as A, WriteResult as B, QueryExecutor as C, SearchType$1 as D, RunSQLOptions as E, TenantCtx$1 as F, monthPartition as G, inferLegacyTier as H, Watermark as I, quarterPartition as J, objectKey as K, WatermarkFilter as L, SyncStateKind as M, SyncStateScope as N, StorageEngine as O, TableName$1 as P, WatermarkScope as R, QueryExecuteResult as S, Row$1 as T, inferSearchType as U, dayPartition as V, mondayOfWeek as W, CompactionThresholds as X, weekPartition as Y, enumeratePartitions as Z, PurgeFilter as _, DataSource as a, QueryCtx as b, FileSetRef as c, LockScope as d, ManifestEntry as f, ParquetCodec as g, OptimizedQueryResult as h, DEFAULT_SEARCH_TYPE as i, SyncStateFilter as j, SyncState as k, GcCtx as l, ManifestStore as m, CompactionTier as n, EngineOptions as o, ManifestPurgeResult as p, quarterOfMonth as q, ComparisonResult as r, ExtraResult as s, CodecCtx as t, ListLiveFilter as u, PurgeResult as v, QueryResult as w, QueryExecuteOptions as x, PurgeUrlsResult as y, WriteCtx as z };

package/dist/adapters/hyparquet.d.mts

@@ -1,4 +1,4 @@
-import { N as TableName, a as DataSource, h as ParquetCodec, t as CodecCtx, w as Row } from "../_chunks/storage.mjs";
+import { P as TableName, T as Row, a as DataSource, g as ParquetCodec, t as CodecCtx } from "../_chunks/storage.mjs";
 import { t as ColumnDef } from "../_chunks/schema.mjs";
 declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
 interface EncodeFlexOptions {

package/dist/adapters/r2-manifest.d.mts

@@ -1,4 +1,4 @@
-import { N as TableName, m as ManifestStore } from "../_chunks/storage.mjs";
+import { P as TableName, m as ManifestStore } from "../_chunks/storage.mjs";
 interface R2ObjectMetadata {
   etag: string;
 }

package/dist/contracts.d.mts

@@ -1,2 +1,2 @@
-import { A as SyncStateFilter, C as QueryResult, D as StorageEngine, E as SearchType, F as Watermark, I as WatermarkFilter, L as WatermarkScope, M as SyncStateScope, N as TableName, O as SyncState, P as TenantCtx, R as WriteCtx, S as QueryExecutor, T as RunSQLOptions, a as DataSource, b as QueryExecuteOptions, c as FileSetRef, d as LockScope, f as ManifestEntry, h as ParquetCodec, j as SyncStateKind, k as SyncStateDetail, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, t as CodecCtx, u as ListLiveFilter, w as Row, x as QueryExecuteResult, y as QueryCtx, z as WriteResult } from "./_chunks/storage.mjs";
+import { A as SyncStateDetail, B as WriteResult, C as QueryExecutor, D as SearchType, E as RunSQLOptions, F as TenantCtx, I as Watermark, L as WatermarkFilter, M as SyncStateKind, N as SyncStateScope, O as StorageEngine, P as TableName, R as WatermarkScope, S as QueryExecuteResult, T as Row, a as DataSource, b as QueryCtx, c as FileSetRef, d as LockScope, f as ManifestEntry, g as ParquetCodec, j as SyncStateFilter, k as SyncState, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, t as CodecCtx, u as ListLiveFilter, w as QueryResult, x as QueryExecuteOptions, z as WriteCtx } from "./_chunks/storage.mjs";
 export { CodecCtx, CompactionTier, DataSource, EngineOptions, FileSetRef, GcCtx, ListLiveFilter, LockScope, ManifestEntry, ManifestStore, ParquetCodec, QueryCtx, QueryExecuteOptions, QueryExecuteResult, QueryExecutor, QueryResult, Row, RunSQLOptions, SearchType, StorageEngine, SyncState, SyncStateDetail, SyncStateFilter, SyncStateKind, SyncStateScope, TableName, TenantCtx, Watermark, WatermarkFilter, WatermarkScope, WriteCtx, WriteResult };

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { A as SyncStateFilter, B as dayPartition, C as QueryResult, D as StorageEngine, E as SearchType, F as Watermark, G as objectKey, H as inferSearchType, I as WatermarkFilter, J as weekPartition, K as quarterOfMonth, L as WatermarkScope, M as SyncStateScope, N as TableName, O as SyncState, P as TenantCtx, R as WriteCtx, S as QueryExecutor, T as RunSQLOptions, U as mondayOfWeek, V as inferLegacyTier, W as monthPartition, X as enumeratePartitions, Y as CompactionThresholds, _ as PurgeResult, a as DataSource, b as QueryExecuteOptions, c as FileSetRef, d as LockScope, f as ManifestEntry, g as PurgeFilter, h as ParquetCodec, i as DEFAULT_SEARCH_TYPE, j as SyncStateKind, k as SyncStateDetail, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, q as quarterPartition, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeUrlsResult, w as Row, x as QueryExecuteResult, y as QueryCtx, z as WriteResult } from "./_chunks/storage.mjs";
+import { A as SyncStateDetail, B as WriteResult, C as QueryExecutor, D as SearchType, E as RunSQLOptions, F as TenantCtx, G as monthPartition, H as inferLegacyTier, I as Watermark, J as quarterPartition, K as objectKey, L as WatermarkFilter, M as SyncStateKind, N as SyncStateScope, O as StorageEngine, P as TableName, R as WatermarkScope, S as QueryExecuteResult, T as Row, U as inferSearchType, V as dayPartition, W as mondayOfWeek, X as CompactionThresholds, Y as weekPartition, Z as enumeratePartitions, _ as PurgeFilter, a as DataSource, b as QueryCtx, c as FileSetRef, d as LockScope, f as ManifestEntry, g as ParquetCodec, h as OptimizedQueryResult, i as DEFAULT_SEARCH_TYPE, j as SyncStateFilter, k as SyncState, l as GcCtx, m as ManifestStore, n as CompactionTier, o as EngineOptions, p as ManifestPurgeResult, q as quarterOfMonth, r as ComparisonResult, s as ExtraResult, t as CodecCtx, u as ListLiveFilter, v as PurgeResult, w as QueryResult, x as QueryExecuteOptions, y as PurgeUrlsResult, z as WriteCtx } from "./_chunks/storage.mjs";
 import { a as createDuckDBExecutor, i as createDuckDBCodec, n as DuckDBHandle, r as canonicalEmptyParquetSchema, t as DuckDBFactory } from "./_chunks/duckdb.mjs";
 import { _ as pages, a as allTables, c as inferTable, d as TABLE_METADATA, f as countries, g as page_keywords, h as keywords, i as TableSchema, m as drizzleSchema, n as ColumnType, o as currentSchemaVersion, p as devices, r as SCHEMAS, s as dimensionToColumn, t as ColumnDef, u as DrizzleSchema } from "./_chunks/schema.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
@@ -9,4 +9,4 @@ declare function coerceRow(row: Row$1): Row$1;
 declare function coerceRows(rows: readonly Row$1[]): Row$1[];
 declare const MAX_DAY_BYTES: number;
 declare function createStorageEngine(opts: EngineOptions): StorageEngine;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type ComparisonResult, DEFAULT_SEARCH_TYPE, type DataSource, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, type EngineOptions, type ExtraResult, FILES_PLACEHOLDER, type FileSetRef, type GcCtx, type GscApiRow, type IngestOptions, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type OptimizedQueryResult, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryResult, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SearchType, type StorageEngine, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, TABLE_METADATA, type TableName, type TableSchema, type TenantCtx, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, countries, createDuckDBCodec, createDuckDBExecutor, createRowAccumulator, createStorageEngine, currentSchemaVersion, dayPartition, devices, dimensionToColumn, drizzleSchema, enumeratePartitions, formatLiteral, inferLegacyTier, inferSearchType, inferTable, keywords, mondayOfWeek, monthPartition, objectKey, page_keywords, pages, quarterOfMonth, quarterPartition, resolveToSQL, substituteNamedFiles, toPath, toSumPosition, transformGscRow, weekPartition };

package/dist/index.mjs

@@ -2,7 +2,7 @@ import { a as inferTable, c as countries, d as keywords, f as page_keywords, i a
 import { a as mondayOfWeek, c as quarterOfMonth, d as weekPartition, i as inferSearchType, l as quarterPartition, n as dayPartition, o as monthPartition, r as inferLegacyTier, s as objectKey, t as DEFAULT_SEARCH_TYPE, u as tenantPrefix } from "./_chunks/storage.mjs";
 import { a as compactTieredImpl, i as substituteNamedFiles, n as compileLogicalQueryPlan, o as enumeratePartitions, r as resolveToSQL, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
 import { bindLiterals, formatLiteral, sqlEscape } from "./sql-bind.mjs";
-import { g as resolveComparisonSQL, m as buildTotalsSql, p as buildExtrasQueries, t as createParquetResolverAdapter } from "./_chunks/pg-adapter.mjs";
+import { g as resolveComparisonSQL, m as buildTotalsSql, p as buildExtrasQueries, t as createParquetResolverAdapter, v as resolveToSQLOptimized } from "./_chunks/pg-adapter.mjs";
 import { createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
@@ -439,6 +439,46 @@ function createStorageEngine(opts) {
 			totals: totalsRow.rows[0] ?? {}
 		};
 	}
+	async function queryOptimized(ctx, state) {
+		const adapter = createParquetResolverAdapter();
+		const plan = buildLogicalPlan(state, adapter.capabilities);
+		const table = ctx.table ?? plan.dataset;
+		const partitions = enumeratePartitions(plan.dateRange.startDate, plan.dateRange.endDate);
+		const { sql, params } = resolveToSQLOptimized(state, {
+			adapter,
+			siteId: void 0
+		});
+		const result = await runSQL({
+			ctx: {
+				userId: ctx.userId,
+				siteId: ctx.siteId
+			},
+			table,
+			fileSets: { FILES: {
+				table,
+				partitions
+			} },
+			sql,
+			params,
+			signal: ctx.signal
+		});
+		const firstRow = result.rows[0];
+		const totalCount = Number(firstRow?.totalCount ?? 0);
+		const totals = {
+			clicks: Number(firstRow?.totalClicks ?? 0),
+			impressions: Number(firstRow?.totalImpressions ?? 0),
+			ctr: Number(firstRow?.totalCtr ?? 0),
+			position: Number(firstRow?.totalPosition ?? 0)
+		};
+		return {
+			rows: result.rows.map((r) => {
+				const { totalCount: _tc, totalClicks: _tcl, totalImpressions: _ti, totalCtr: _tr, totalPosition: _tp, ...rest } = r;
+				return rest;
+			}),
+			totalCount,
+			totals
+		};
+	}
 	async function queryExtras(ctx, state) {
 		const adapter = createParquetResolverAdapter();
 		const extras = buildExtrasQueries(state, {
@@ -579,6 +619,7 @@ function createStorageEngine(opts) {
 		query,
 		queryComparison,
 		queryExtras,
+		queryOptimized,
 		runSQL,
 		compactTiered,
 		gcOrphans,

package/dist/ingest.d.mts

@@ -1,4 +1,4 @@
-import { N as TableName, w as Row } from "./_chunks/storage.mjs";
+import { P as TableName, T as Row } from "./_chunks/storage.mjs";
 /**
  * Canonical GSC API dimension order per table. Consumers hitting the raw
  * `searchanalytics.query` endpoint must request dimensions in this order so

package/dist/planner.d.mts

@@ -1,3 +1,3 @@
-import { X as enumeratePartitions } from "./_chunks/storage.mjs";
+import { Z as enumeratePartitions } from "./_chunks/storage.mjs";
 import { a as substituteNamedFiles, i as resolveToSQL, n as ResolvedQuery, r as compileLogicalQueryPlan, t as FILES_PLACEHOLDER } from "./_chunks/planner.mjs";
 export { FILES_PLACEHOLDER, ResolvedQuery, compileLogicalQueryPlan, enumeratePartitions, resolveToSQL, substituteNamedFiles };

package/dist/source/index.d.mts

@@ -1,4 +1,4 @@
-import { D as StorageEngine, P as TenantCtx, w as Row } from "../_chunks/storage.mjs";
+import { F as TenantCtx, O as StorageEngine, T as Row } from "../_chunks/storage.mjs";
 import { n as AnalysisResult, t as AnalysisParams } from "../_chunks/analysis-types.mjs";
 import { i as QueryRow, r as FileSet, s as SqlQuerySource, t as AnalysisQuerySource } from "../_chunks/source-types.mjs";
 import { t as AnalyzerRegistry } from "../_chunks/registry.mjs";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -159,7 +159,7 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.9.0"
+    "gscdump": "0.9.1"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",