@vaultgradient/pipequery-cli 0.7.0 → 0.10.0

package/src/server/sources/mysql.ts

@@ -2,6 +2,8 @@ import mysql from 'mysql2/promise';
 import type { MysqlSourceConfig } from '../../config/schema.js';
 import { parseDuration } from '../../utils/parseDuration.js';
 import { expandEnv } from '../../utils/expandEnv.js';
+import { parseQuery } from '../../engine.js';
+import { compileMysqlPushdown } from './pushdown/mysql.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
 
 const DEFAULT_INTERVAL_MS = 30_000;
@@ -80,6 +82,48 @@ export class MysqlSourceAdapter implements SourceAdapter {
     await this.fetch();
   }
 
+  /**
+   * Adapter-native push-down. Compiles the pipe expression to MySQL SQL
+   * (see ./pushdown/mysql.ts) and runs it directly. SourceManager.runQuery
+   * dispatches here automatically when the pipeline shape is eligible —
+   * unsupported shapes return `{ ok: false, reason }` and the caller falls
+   * back to the in-process engine.
+   *
+   * Mirrors PostgresSourceAdapter.runPushdown — same shape, same result
+   * type. Both adapters route through the dialect-parametric compiler in
+   * compile.ts.
+   */
+  async runPushdown(expression: string): Promise<
+    | { ok: true; rows: unknown[]; sql: string; params: unknown[] }
+    | { ok: false; reason: string }
+  > {
+    if (!this.pool) {
+      return { ok: false, reason: 'MySQL pool not initialized — call start() first' };
+    }
+    let pipeline;
+    try {
+      pipeline = parseQuery(expression);
+    } catch (err) {
+      return { ok: false, reason: `parse error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const compiled = compileMysqlPushdown(pipeline, this.config.query);
+    if (!compiled.ok) {
+      return { ok: false, reason: compiled.reason };
+    }
+    try {
+      const [rows] = await this.pool.query(compiled.compiled.sql, compiled.compiled.params);
+      const asArray = Array.isArray(rows) ? (rows as unknown[]) : [];
+      return {
+        ok: true,
+        rows: asArray,
+        sql: compiled.compiled.sql,
+        params: compiled.compiled.params,
+      };
+    } catch (err) {
+      return { ok: false, reason: `mysql error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+
   private async fetch(): Promise<void> {
     if (!this.pool) {
       this.error = 'MySQL pool not initialized';

package/src/server/sources/postgres.ts

@@ -3,6 +3,8 @@ import type { PostgresSourceConfig } from '../../config/schema.js';
 import { parseDuration } from '../../utils/parseDuration.js';
 import { expandEnv } from '../../utils/expandEnv.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
+import { parseQuery } from '../../engine.js';
+import { compilePostgresPushdown } from './pushdown/postgres.js';
 
 const DEFAULT_INTERVAL_MS = 30_000;
 const DEFAULT_MAX_ROWS = 10_000;
@@ -86,6 +88,54 @@ export class PostgresSourceAdapter implements SourceAdapter {
     await this.fetch();
   }
 
+  /**
+   * Push-down execution prototype. Compiles a pipequery expression rooted at
+   * this source into a single Postgres query (where / sort / first only in
+   * v1) and runs it directly against the database — no in-memory
+   * materialization of the user's source query, no polling, no ring buffer.
+   *
+   * Returns:
+   *   - `{ ok: true, rows, sql, params, fullyPushed: true }` on a clean push-down
+   *   - `{ ok: false, reason }` when the AST contains operators we don't yet
+   *     translate. Caller should fall back to the in-process engine on the
+   *     ring-buffered data instead.
+   *
+   * This is an opt-in API; the default getData() / fetch() polling loop is
+   * unchanged so existing users see no behaviour change.
+   */
+  async runPushdown(expression: string): Promise<
+    | { ok: true; rows: unknown[]; sql: string; params: unknown[]; fullyPushed: true }
+    | { ok: false; reason: string }
+  > {
+    if (!this.pool) {
+      return { ok: false, reason: 'Postgres pool not initialized — call start() first' };
+    }
+    let pipeline;
+    try {
+      pipeline = parseQuery(expression);
+    } catch (err) {
+      return { ok: false, reason: `parse error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+
+    const compiled = compilePostgresPushdown(pipeline, this.config.query);
+    if (!compiled.ok) {
+      return { ok: false, reason: compiled.reason };
+    }
+
+    try {
+      const res = await this.pool.query(compiled.compiled.sql, compiled.compiled.params);
+      return {
+        ok: true,
+        rows: res.rows,
+        sql: compiled.compiled.sql,
+        params: compiled.compiled.params,
+        fullyPushed: true,
+      };
+    } catch (err) {
+      return { ok: false, reason: `postgres error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+
   private async fetch(): Promise<void> {
     if (!this.pool) {
       this.error = 'Postgres pool not initialized';

package/src/server/sources/pushdown/clickhouse.ts

@@ -0,0 +1,17 @@
+// Pipe-AST → ClickHouse SQL compiler.
+//
+// Thin shim over the dialect-parametric compileForDialect; same operator +
+// aggregate set as Postgres / MySQL / Snowflake. ClickHouse uses
+// backtick-quoted identifiers and `?` positional placeholders — both
+// captured in CLICKHOUSE_DIALECT.
+
+import type { Pipeline } from '../../../../../src/engine/types.js';
+import { compileForDialect, type CompileResult } from './compile.js';
+import { CLICKHOUSE_DIALECT } from './dialect.js';
+
+export function compileClickhousePushdown(
+  pipeline: Pipeline,
+  baseQuery: string,
+): CompileResult {
+  return compileForDialect(pipeline, baseQuery, CLICKHOUSE_DIALECT);
+}

package/src/server/sources/pushdown/compile.ts

@@ -0,0 +1,328 @@
+// Pipe-AST → SQL compiler (dialect-parametric).
+//
+// Translates a Pipeline AST whose source is a single SQL-backed source into
+// a parameterized SQL query. Every emission goes through a SqlDialect, so
+// adding a new engine reduces to writing a dialect (see dialect.ts).
+//
+// Supported operators:
+//   - WhereOp        with BinaryExpr / UnaryExpr / FieldAccess / literals
+//   - SortOp         with FieldAccess criteria
+//   - FirstOp        → LIMIT N
+//   - SelectOp       projection (FieldAccess / arithmetic / AliasExpr)
+//   - DistinctOp     full-row dedup (no DISTINCT ON; portable across dialects)
+//   - RollupOp       → GROUP BY + aggregates in projection
+//   - AggregateOp    pipeline-terminal aggregate (e.g. `orders | sum(total)`)
+//
+// Supported aggregate functions: sum, avg, min, max, count, distinct_count.
+// Anything else (median, stddev, percentile, vwap, sharpe, …) declines so
+// the in-process engine handles them — those have no clean portable SQL
+// translation across the dialects we care about.
+//
+// Operator-order rules:
+//   - WhereOp must precede grouping (RollupOp / AggregateOp). After
+//     grouping, "where" is HAVING — declined for now.
+//   - SortOp / FirstOp accepted at any position; the SQL clauses are always
+//     emitted at the end so they apply post-grouping when grouping exists.
+//   - SelectOp / DistinctOp must come before grouping; grouping owns its
+//     own projection.
+//   - At most one grouping op (RollupOp xor AggregateOp).
+
+import type {
+  AggregateFnName,
+  Expression,
+  Operation,
+  Pipeline,
+} from '../../../../../src/engine/types.js';
+import type { SqlDialect } from './dialect.js';
+
+export interface CompiledPushdown {
+  /** Parameterized SQL using the dialect's placeholder syntax. */
+  sql: string;
+  /** Values to bind, in order. */
+  params: unknown[];
+}
+
+export type CompileResult =
+  | { ok: true; compiled: CompiledPushdown }
+  | { ok: false; reason: string };
+
+interface CompilerCtx {
+  /** Subquery wrapping the user-provided source query. */
+  baseSelect: string;
+  /** Running parameter array; mutated as we walk. */
+  params: unknown[];
+  /** Dialect for quoting + placeholder rendering. */
+  dialect: SqlDialect;
+}
+
+const BINARY_OP_SQL: Record<string, string | undefined> = {
+  '>': '>',
+  '>=': '>=',
+  '<': '<',
+  '<=': '<=',
+  '==': '=',
+  '!=': '<>',
+  '&&': 'AND',
+  '||': 'OR',
+  '+': '+',
+  '-': '-',
+  '*': '*',
+  '/': '/',
+  '%': '%',
+};
+
+/**
+ * Aggregate functions we know how to translate to portable SQL. Both
+ * Postgres and MySQL accept these identically — no per-dialect overrides
+ * needed yet. The function takes the rendered argument SQL (or `null` for
+ * the no-arg `count()` shape) and produces the call site.
+ */
+const SQL_AGGREGATE_FN: Partial<Record<AggregateFnName, (arg: string | null) => string>> = {
+  sum: (arg) => `SUM(${arg ?? '*'})`,
+  avg: (arg) => `AVG(${arg ?? '*'})`,
+  min: (arg) => `MIN(${arg ?? '*'})`,
+  max: (arg) => `MAX(${arg ?? '*'})`,
+  count: (arg) => `COUNT(${arg ?? '*'})`,
+  distinct_count: (arg) => `COUNT(DISTINCT ${arg ?? '*'})`,
+};
+
+export function compileForDialect(
+  pipeline: Pipeline,
+  baseQuery: string,
+  dialect: SqlDialect,
+): CompileResult {
+  const ctx: CompilerCtx = {
+    baseSelect: stripTrailingSemicolon(baseQuery),
+    params: [],
+    dialect,
+  };
+
+  let where: string | null = null;
+  let orderBy: string | null = null;
+  let limit: number | null = null;
+  let selectClause: string | null = null;
+  let distinctClause = false;
+  let groupByClause: string | null = null;
+  let grouped = false;
+
+  for (const op of pipeline.operations) {
+    switch (op.kind) {
+      case 'WhereOp': {
+        if (grouped) {
+          return { ok: false, reason: 'where after grouping (HAVING) is not yet pushable' };
+        }
+        const compiled = compileExpr(op.condition, ctx);
+        if (compiled === null) {
+          return { ok: false, reason: 'where(): unsupported expression node' };
+        }
+        where = where ? `(${where}) AND (${compiled})` : compiled;
+        break;
+      }
+
+      case 'SortOp': {
+        const parts: string[] = [];
+        for (const c of op.criteria) {
+          if (c.expression.kind !== 'FieldAccess' || c.expression.path.length !== 1) {
+            return { ok: false, reason: 'sort(): only single-field paths supported' };
+          }
+          const ident = ctx.dialect.quoteIdent(c.expression.path[0]);
+          parts.push(`${ident} ${c.direction === 'desc' ? 'DESC' : 'ASC'}`);
+        }
+        orderBy = parts.join(', ');
+        break;
+      }
+
+      case 'FirstOp': {
+        limit = op.count;
+        break;
+      }
+
+      case 'SelectOp': {
+        if (grouped) {
+          return { ok: false, reason: 'select after rollup is not yet pushable' };
+        }
+        if (selectClause !== null) {
+          return { ok: false, reason: 'multiple select() ops in one pipeline' };
+        }
+        const parts: string[] = [];
+        for (const f of op.fields) {
+          const compiled = compileSelectField(f, ctx, false);
+          if (compiled === null) {
+            return { ok: false, reason: 'select(): unsupported field expression' };
+          }
+          parts.push(compiled);
+        }
+        selectClause = parts.join(', ');
+        break;
+      }
+
+      case 'DistinctOp': {
+        if (op.fields !== undefined && op.fields.length > 0) {
+          return { ok: false, reason: 'distinct(field) is not yet pushable; full-row distinct works' };
+        }
+        if (grouped) {
+          return { ok: false, reason: 'distinct after grouping is not yet pushable' };
+        }
+        distinctClause = true;
+        break;
+      }
+
+      case 'RollupOp': {
+        if (grouped) {
+          return { ok: false, reason: 'multiple grouping ops (rollup/aggregate) in one pipeline' };
+        }
+        if (selectClause !== null) {
+          return { ok: false, reason: 'rollup after select is not yet pushable' };
+        }
+        const keyParts: string[] = [];
+        const groupByParts: string[] = [];
+        for (const k of op.keys) {
+          if (k.kind !== 'FieldAccess' || k.path.length !== 1) {
+            return { ok: false, reason: 'rollup(): keys must be single-field paths' };
+          }
+          const ident = ctx.dialect.quoteIdent(k.path[0]);
+          keyParts.push(ident);
+          groupByParts.push(ident);
+        }
+        const aggParts: string[] = [];
+        for (const a of op.aggregates) {
+          const compiled = compileSelectField(a, ctx, true);
+          if (compiled === null) {
+            return { ok: false, reason: 'rollup(): unsupported aggregate expression' };
+          }
+          aggParts.push(compiled);
+        }
+        selectClause = [...keyParts, ...aggParts].join(', ');
+        groupByClause = groupByParts.length > 0 ? groupByParts.join(', ') : null;
+        grouped = true;
+        break;
+      }
+
+      case 'AggregateOp': {
+        if (grouped) {
+          return { ok: false, reason: 'multiple grouping ops (rollup/aggregate) in one pipeline' };
+        }
+        if (selectClause !== null) {
+          return { ok: false, reason: 'aggregate after select is not yet pushable' };
+        }
+        const fn = SQL_AGGREGATE_FN[op.function];
+        if (!fn) {
+          return { ok: false, reason: `aggregate "${op.function}" has no SQL push-down` };
+        }
+        if (op.args && op.args.length > 0) {
+          return { ok: false, reason: `aggregate "${op.function}" with extra args is not yet pushable` };
+        }
+        const argSql = op.field === undefined ? null : compileExpr(op.field, ctx);
+        if (op.field !== undefined && argSql === null) {
+          return { ok: false, reason: `aggregate "${op.function}": unsupported argument expression` };
+        }
+        selectClause = `${fn(argSql)} AS ${ctx.dialect.quoteIdent(op.function)}`;
+        grouped = true;
+        break;
+      }
+
+      default:
+        return { ok: false, reason: `${(op as Operation).kind}: not yet pushable` };
+    }
+  }
+
+  let sql = `SELECT ${distinctClause ? 'DISTINCT ' : ''}${selectClause ?? '*'} FROM (${ctx.baseSelect}) AS pq_src`;
+  if (where) sql += ` WHERE ${where}`;
+  if (groupByClause) sql += ` GROUP BY ${groupByClause}`;
+  if (orderBy) sql += ` ORDER BY ${orderBy}`;
+  if (limit !== null) sql += ` LIMIT ${limit}`;
+
+  return { ok: true, compiled: { sql, params: ctx.params } };
+}
+
+// ─── Expression compilers ───────────────────────────────────────────────────
+
+function compileSelectField(expr: Expression, ctx: CompilerCtx, aggregateContext: boolean): string | null {
+  if (expr.kind === 'AliasExpr') {
+    const inner = compileSelectField(expr.expression, ctx, aggregateContext);
+    if (inner === null) return null;
+    return `${inner} AS ${ctx.dialect.quoteIdent(expr.alias)}`;
+  }
+  if (expr.kind === 'FunctionCall') {
+    if (!aggregateContext) return null;
+    return compileAggregateCall(expr.name, expr.args, ctx);
+  }
+  return compileExpr(expr, ctx);
+}
+
+function compileAggregateCall(name: string, args: Expression[], ctx: CompilerCtx): string | null {
+  const fn = SQL_AGGREGATE_FN[name as AggregateFnName];
+  if (!fn) return null;
+  if (args.length === 0) return fn(null);
+  if (args.length > 1) return null;
+  const argSql = compileExpr(args[0], ctx);
+  if (argSql === null) return null;
+  return fn(argSql);
+}
+
+function compileExpr(expr: Expression, ctx: CompilerCtx): string | null {
+  switch (expr.kind) {
+    case 'NumberLiteral':
+    case 'StringLiteral':
+    case 'BooleanLiteral':
+      return ctx.dialect.bindLiteral(expr.value, ctx.params);
+    case 'NullLiteral':
+      return 'NULL';
+    case 'FieldAccess': {
+      if (expr.path.length !== 1) return null;
+      return ctx.dialect.quoteIdent(expr.path[0]);
+    }
+    case 'BinaryExpr': {
+      const opSql = BINARY_OP_SQL[expr.operator];
+      if (!opSql) return null;
+      const left = compileExpr(expr.left, ctx);
+      const right = compileExpr(expr.right, ctx);
+      if (left === null || right === null) return null;
+
+      // x = NULL never matches in either Postgres or MySQL; need
+      // IS NULL / IS NOT NULL.
+      if (expr.right.kind === 'NullLiteral') {
+        if (expr.operator === '==') return `${left} IS NULL`;
+        if (expr.operator === '!=') return `${left} IS NOT NULL`;
+      }
+      if (expr.left.kind === 'NullLiteral') {
+        if (expr.operator === '==') return `${right} IS NULL`;
+        if (expr.operator === '!=') return `${right} IS NOT NULL`;
+      }
+
+      return `(${left} ${opSql} ${right})`;
+    }
+    case 'UnaryExpr': {
+      const operand = compileExpr(expr.operand, ctx);
+      if (operand === null) return null;
+      if (expr.operator === '!') return `(NOT ${operand})`;
+      if (expr.operator === '-') return `(-${operand})`;
+      return null;
+    }
+    case 'FunctionCall':
+    case 'AliasExpr':
+      return null;
+  }
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}
+
+const PUSHABLE_OPS: ReadonlySet<Operation['kind']> = new Set([
+  'WhereOp',
+  'SortOp',
+  'FirstOp',
+  'SelectOp',
+  'DistinctOp',
+  'RollupOp',
+  'AggregateOp',
+]);
+
+/** Convenience helper: does the pipeline only contain operators we *might*
+ *  push down? Doesn't validate expression nodes. */
+export function isPushableShape(pipeline: Pipeline): boolean {
+  return pipeline.operations.every((op) => PUSHABLE_OPS.has(op.kind));
+}

package/src/server/sources/pushdown/dialect.ts

@@ -0,0 +1,88 @@
+// SQL dialect abstraction for the pipe-AST → SQL compilers.
+//
+// The compiler in compile.ts is dialect-agnostic — every place that emits
+// SQL goes through one of the methods here. Adding a new engine (Snowflake,
+// ClickHouse, BigQuery, etc.) reduces to: write a dialect, plug it in.
+
+export interface SqlDialect {
+  /**
+   * Quote an identifier (column / alias). Embedded special characters
+   * must be escaped per dialect rules so the result can never break out
+   * of the quoted span.
+   *   - Postgres / Snowflake: `"name"`, internal `"` doubled
+   *   - MySQL / ClickHouse:   `` `name` ``, internal `` ` `` doubled
+   */
+  quoteIdent(name: string): string;
+
+  /**
+   * Embed a literal value in the emitted SQL. Dialects that use parameter
+   * binding push the value into `params` and return a placeholder
+   * (`$N`, `?`); dialects that don't bind (ClickHouse over HTTP) inline
+   * the value as a properly-escaped literal and leave `params` untouched.
+   *
+   * The compiler doesn't care which path the dialect takes — it just
+   * gets back a string to splice into the SQL.
+   */
+  bindLiteral(value: unknown, params: unknown[]): string;
+}
+
+export const POSTGRES_DIALECT: SqlDialect = {
+  quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`,
+  bindLiteral: (value, params) => {
+    params.push(value);
+    return `$${params.length}`;
+  },
+};
+
+export const MYSQL_DIALECT: SqlDialect = {
+  quoteIdent: (name) => `\`${name.replace(/`/g, '``')}\``,
+  bindLiteral: (value, params) => {
+    params.push(value);
+    return '?';
+  },
+};
+
+/**
+ * Snowflake speaks ANSI SQL with double-quoted identifiers (case-sensitive)
+ * and accepts `?` positional binds.
+ */
+export const SNOWFLAKE_DIALECT: SqlDialect = {
+  quoteIdent: (name) => `"${name.replace(/"/g, '""')}"`,
+  bindLiteral: (value, params) => {
+    params.push(value);
+    return '?';
+  },
+};
+
+/**
+ * ClickHouse accepts both backticks and double quotes for identifiers; we
+ * use backticks for parity with MySQL conventions. The HTTP client uses
+ * named `{name:Type}` binds (which require per-param type tagging the
+ * dialect interface doesn't carry), so we inline literals with proper
+ * escape instead. Safe: literal values come from typed AST nodes
+ * (NumberLiteral / StringLiteral / BooleanLiteral / NullLiteral) — never
+ * from raw user strings.
+ */
+export const CLICKHOUSE_DIALECT: SqlDialect = {
+  quoteIdent: (name) => `\`${name.replace(/`/g, '``')}\``,
+  bindLiteral: (value) => clickhouseLiteral(value),
+};
+
+function clickhouseLiteral(value: unknown): string {
+  if (value === null || value === undefined) return 'NULL';
+  if (typeof value === 'boolean') return value ? '1' : '0';
+  if (typeof value === 'number') {
+    if (!Number.isFinite(value)) return 'NULL';
+    return String(value);
+  }
+  if (typeof value === 'string') {
+    // Order matters: backslash MUST be doubled first so subsequent
+    // inserted backslashes (from quote escaping) don't get re-doubled.
+    // ClickHouse accepts literal control chars inside string literals;
+    // only backslash and single quote require escape to prevent injection.
+    const escaped = value.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
+    return `'${escaped}'`;
+  }
+  // Fallback: stringify and escape — covers Date, BigInt, etc. as best-effort.
+  return clickhouseLiteral(String(value));
+}