@vaultgradient/pipequery-cli 0.6.1 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vaultgradient/pipequery-cli",
-  "version": "0.6.1",
+  "version": "0.10.0",
   "description": "CLI for PipeQuery — data sources, aggregations, APIs, and terminal dashboards",
   "type": "module",
   "bin": {
@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.91.1",
+    "@clickhouse/client": "^1.18.3",
     "@fastify/cors": "^11.2.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "better-sqlite3": "^12.9.0",
@@ -25,10 +26,12 @@
     "grammy": "^1.42.0",
     "ink": "^7.0.1",
     "kafkajs": "^2.2.4",
+    "mongodb": "^7.2.0",
     "mysql2": "^3.22.2",
     "ora": "^9.3.0",
     "pg": "^8.20.0",
     "react": "^19.2.5",
+    "snowflake-sdk": "^2.4.0",
     "ws": "^8.20.0",
     "yaml": "^2.8.3",
     "zod": "^4.3.6"
@@ -43,11 +46,17 @@
     "typescript": "~6.0.0",
     "vitest": "^4.1.4"
   },
+  "overrides": {
+    "gcp-metadata": "^8.0.0"
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/andreadito/pipequery.git",
     "directory": "cli"
   },
   "author": "andreadito",
-  "license": "MIT"
+  "license": "MIT",
+  "engines": {
+    "node": ">=22"
+  }
 }

package/src/commands/telegram.ts

@@ -1,14 +1,17 @@
+import { resolve } from 'node:path';
 import { loadConfig } from '../config/loader.js';
 import { log } from '../utils/logger.js';
 import { LocalProvider, AttachedProvider, type Provider } from '../mcp/provider.js';
 import { buildBot } from '../telegram/bot.js';
 import { createNLTranslator } from '../telegram/nl.js';
+import { createEventLogger } from '../telegram/event-log.js';
 
 interface TelegramServeOptions {
   botToken?: string;
   attach?: string;
   allowUser?: string[];
   anthropicKey?: string;
+  logFile?: string;
 }
 
 export async function telegramServeCommand(opts: TelegramServeOptions): Promise<void> {
@@ -26,9 +29,13 @@ export async function telegramServeCommand(opts: TelegramServeOptions): Promise<
     ? createNLTranslator(provider, { apiKey: anthropicKey })
     : undefined;
 
+  const logFilePath = opts.logFile ? resolve(process.cwd(), opts.logFile) : undefined;
+  const logger = createEventLogger({ filePath: logFilePath });
+
   const bot = buildBot(token, provider, {
     allowUsers: opts.allowUser ?? [],
     nl,
+    logger,
   });
 
   const me = await bot.api.getMe();
@@ -38,6 +45,9 @@ export async function telegramServeCommand(opts: TelegramServeOptions): Promise<
   } else {
     log.dim('Natural-language translation disabled. Pass --anthropic-key or set ANTHROPIC_API_KEY to enable.');
   }
+  if (logFilePath) {
+    log.info(`Event log → ${logFilePath} (JSONL)`);
+  }
   if (!opts.allowUser || opts.allowUser.length === 0) {
     log.warn('No --allow-user set: anyone with the bot username can query. Add allowlist for production.');
   }

package/src/config/schema.ts

@@ -26,7 +26,10 @@ export type SourceConfig =
   | PostgresSourceConfig
   | MysqlSourceConfig
   | SqliteSourceConfig
-  | KafkaSourceConfig;
+  | KafkaSourceConfig
+  | SnowflakeSourceConfig
+  | ClickhouseSourceConfig
+  | MongoSourceConfig;
 
 export interface RestSourceConfig {
   type: 'rest';
@@ -35,12 +38,34 @@ export interface RestSourceConfig {
   headers?: Record<string, string>;
   dataPath?: string;
   interval?: string;
+  /** Optional auth helper. Currently supports `bearer`; more kinds (basic,
+   *  apiKeyHeader, hmac-*) tracked in issue #37. The token is interpolated
+   *  through ${ENV_VAR} like every other string field. */
+  auth?: BearerAuth;
+}
+
+export interface BearerAuth {
+  kind: 'bearer';
+  token: string;
 }
 
 export interface WebSocketSourceConfig {
   type: 'websocket';
   url: string;
   maxBuffer?: number;
+  /** Payload(s) to send to the server immediately after the WebSocket
+   *  opens. Most exchange feeds (Binance, Coinbase, Kraken, OKX, Bybit, …)
+   *  require this — without it, no data is delivered. Each payload is
+   *  JSON-stringified and sent in order. Re-sent on every reconnect. */
+  subscribe?: unknown | unknown[];
+  /** Optional keepalive: sends `payload` to the server every `interval`
+   *  while the socket is open. Most exchanges close idle sockets within
+   *  a few minutes; a periodic ping keeps the stream alive. */
+  heartbeat?: {
+    payload: unknown;
+    /** Duration like "30s", "1m". */
+    interval: string;
+  };
 }
 
 export interface FileSourceConfig {
@@ -223,3 +248,67 @@ export interface TelegramNotifyConfig {
    */
   message?: string;
 }
+
+// ─── Snowflake / ClickHouse / MongoDB ───────────────────────────────────────
+
+export interface SnowflakeSourceConfig {
+  type: 'snowflake';
+  /** Snowflake account locator, e.g. `myorg-myaccount`. Supports `${ENV_VAR}`. */
+  account: string;
+  /** Username. Supports `${ENV_VAR}`. */
+  username: string;
+  /** Password. Supports `${ENV_VAR}`. (Key-pair / OAuth out of scope for v1.) */
+  password: string;
+  /** Database to USE. Supports `${ENV_VAR}`. */
+  database?: string;
+  /** Schema to USE. Supports `${ENV_VAR}`. */
+  schema?: string;
+  /** Compute warehouse to USE. Supports `${ENV_VAR}`. */
+  warehouse?: string;
+  /** Optional role. Supports `${ENV_VAR}`. */
+  role?: string;
+  /** SELECT query polled on every interval. */
+  query: string;
+  /** Poll interval, e.g. "60s". Default: 60s. */
+  interval?: string;
+  /** Hard cap on rows per fetch (default 10000). */
+  maxRows?: number;
+}
+
+export interface ClickhouseSourceConfig {
+  type: 'clickhouse';
+  /** ClickHouse HTTP/HTTPS endpoint, e.g. `https://host:8443`. Supports `${ENV_VAR}`. */
+  url: string;
+  /** Username. Supports `${ENV_VAR}`. */
+  username?: string;
+  /** Password. Supports `${ENV_VAR}`. */
+  password?: string;
+  /** Database to query. Defaults to `default`. */
+  database?: string;
+  /** SELECT query polled on every interval. */
+  query: string;
+  /** Poll interval, e.g. "30s". Default: 30s. */
+  interval?: string;
+  /** Hard cap on rows per fetch (default 10000). */
+  maxRows?: number;
+}
+
+export interface MongoSourceConfig {
+  type: 'mongodb';
+  /** MongoDB connection URI. Supports `${ENV_VAR}`. */
+  url: string;
+  /** Database name. Supports `${ENV_VAR}`. */
+  database: string;
+  /** Collection to read from. */
+  collection: string;
+  /**
+   * Optional default filter applied on every poll (Mongo `$match` stage).
+   * Plain JSON object — `{ active: true }` etc. The filter is `AND`-merged
+   * with any push-down `where()` predicates the engine compiles in.
+   */
+  filter?: Record<string, unknown>;
+  /** Poll interval, e.g. "30s". Default: 30s. */
+  interval?: string;
+  /** Hard cap on documents per fetch (default 10000). */
+  maxRows?: number;
+}

package/src/index.ts

@@ -71,7 +71,7 @@ source
 source
   .command('add <name>')
   .description('Add a data source')
-  .requiredOption('-t, --type <type>', 'Source type: rest, websocket, file, postgres, mysql, sqlite, kafka')
+  .requiredOption('-t, --type <type>', 'Source type: rest, websocket, file, postgres, mysql, sqlite, kafka, snowflake, clickhouse, mongodb')
   .option('-u, --url <url>', 'Connection URL for REST / WebSocket / Postgres / MySQL sources')
   .option('-i, --interval <interval>', 'Poll interval (e.g., 30s, 5m)', '30s')
   .option('-d, --data-path <path>', 'Dot-path to extract data from REST response')
@@ -185,6 +185,7 @@ telegram
   .option('--attach <url>', 'Attach to a running `pq serve` instance instead of loading pipequery.yaml locally')
   .option('--allow-user <handle>', 'Allowlist a user (numeric id or @username); repeatable. With no allowlist, anyone with the bot username can query.', collectRepeated, [])
   .option('--anthropic-key <key>', 'Anthropic API key (or set $ANTHROPIC_API_KEY) — enables natural-language → pipequery translation on plain-text messages via claude-haiku-4-5.')
+  .option('--log-file <path>', 'Append every bot event (commands, NL queries, unauthorized attempts) as JSONL to <path>. Stderr always carries a colored, human-readable copy.')
   .action(telegramServeCommand);
 
 // ─── pq watch ────────────────────────────────────────────────────────────────

package/src/mcp/provider.ts

@@ -2,7 +2,6 @@ import type { DataContext } from '../engine.js';
 import type { EndpointConfig, PipeQueryConfig } from '../config/schema.js';
 import type { SourceStatus } from '../server/sources/types.js';
 import { SourceManager } from '../server/sources/manager.js';
-import { query } from '../engine.js';
 
 export interface ProviderSourceInfo {
   name: string;
@@ -80,14 +79,12 @@ export class LocalProvider implements Provider {
     if (!endpoint) {
       throw new Error(`Endpoint "${path}" not found`);
     }
-    const context = this.sourceManager.getContext();
-    return query(context, endpoint.query);
+    return this.sourceManager.runQuery(endpoint.query);
   }
 
   async runQuery(expression: string): Promise<unknown> {
     await this.ready;
-    const context = this.sourceManager.getContext();
-    return query(context, expression);
+    return this.sourceManager.runQuery(expression);
   }
 
   async dispose(): Promise<void> {
@@ -123,11 +120,11 @@ class AttachedLocalProvider implements Provider {
   async callEndpoint(path: string): Promise<unknown> {
     const endpoint = this.endpoints.get(path);
     if (!endpoint) throw new Error(`Endpoint "${path}" not found`);
-    return query(this.sourceManager.getContext(), endpoint.query);
+    return this.sourceManager.runQuery(endpoint.query);
   }
 
   async runQuery(expression: string): Promise<unknown> {
-    return query(this.sourceManager.getContext(), expression);
+    return this.sourceManager.runQuery(expression);
   }
 
   async dispose(): Promise<void> {

package/src/server/routes/control.ts

@@ -1,5 +1,4 @@
 import type { FastifyInstance } from 'fastify';
-import { query } from '../../engine.js';
 import type { SourceConfig, EndpointConfig } from '../../config/schema.js';
 import type { SourceManager } from '../sources/manager.js';
 
@@ -89,9 +88,8 @@ export function registerControlRoutes(
 
   app.post<{ Body: { query: string } }>('/api/_control/query', async (req) => {
     const { query: expr } = req.body;
-    const context = sourceManager.getContext();
     try {
-      const result = query(context, expr);
+      const result = await sourceManager.runQuery(expr);
       return { ok: true, result };
     } catch (err) {
       return { ok: false, error: err instanceof Error ? err.message : String(err) };

package/src/server/routes/dynamic.ts

@@ -1,5 +1,4 @@
 import type { FastifyInstance } from 'fastify';
-import { query } from '../../engine.js';
 import type { EndpointConfig } from '../../config/schema.js';
 import type { SourceManager } from '../sources/manager.js';
 import { ResponseCache } from '../cache.js';
@@ -32,10 +31,9 @@ export function registerDynamicRoutes(
       }
     }
 
-    // Execute query
-    const context = sourceManager.getContext();
+    // Execute query (push-down preferred when the source supports it)
     try {
-      const result = query(context, config.query);
+      const result = await sourceManager.runQuery(config.query);
 
       if (config.cache) {
         cache.set(path, result, parseDuration(config.cache));

package/src/server/sources/clickhouse.ts

@@ -0,0 +1,153 @@
+import { createClient, type ClickHouseClient } from '@clickhouse/client';
+import type { ClickhouseSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import { expandEnv } from '../../utils/expandEnv.js';
+import { parseQuery } from '../../engine.js';
+import { compileClickhousePushdown } from './pushdown/clickhouse.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+
+export class ClickhouseSourceAdapter implements SourceAdapter {
+  private client: ClickHouseClient | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+
+  constructor(private config: ClickhouseSourceConfig) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+  }
+
+  async start(): Promise<void> {
+    this.client = createClient({
+      url: expandEnv(this.config.url, 'clickhouse source URL'),
+      ...(this.config.username
+        ? { username: expandEnv(this.config.username, 'clickhouse source username') }
+        : {}),
+      ...(this.config.password
+        ? { password: expandEnv(this.config.password, 'clickhouse source password') }
+        : {}),
+      ...(this.config.database ? { database: this.config.database } : {}),
+    });
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.client) {
+      void this.client.close();
+      this.client = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  /**
+   * Adapter-native push-down. ClickHouse dialect inlines literals (vs.
+   * Postgres / MySQL / Snowflake which bind via placeholders) so the
+   * `params` array returned in the success result is always empty;
+   * the SQL is fully self-contained.
+   */
+  async runPushdown(expression: string): Promise<
+    | { ok: true; rows: unknown[]; sql: string; params: unknown[] }
+    | { ok: false; reason: string }
+  > {
+    if (!this.client) {
+      return { ok: false, reason: 'ClickHouse client not initialized — call start() first' };
+    }
+    let pipeline;
+    try {
+      pipeline = parseQuery(expression);
+    } catch (err) {
+      return { ok: false, reason: `parse error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const compiled = compileClickhousePushdown(pipeline, this.config.query);
+    if (!compiled.ok) {
+      return { ok: false, reason: compiled.reason };
+    }
+    try {
+      const rows = await this.executeSql(compiled.compiled.sql);
+      return {
+        ok: true,
+        rows,
+        sql: compiled.compiled.sql,
+        params: compiled.compiled.params,
+      };
+    } catch (err) {
+      return { ok: false, reason: `clickhouse error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.client) {
+      this.error = 'ClickHouse client not initialized';
+      return;
+    }
+    try {
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const rows = await this.executeSql(capQuery);
+      if (rows.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      this.data = rows;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private async executeSql(sql: string): Promise<unknown[]> {
+    const result = await this.client!.query({
+      query: sql,
+      format: 'JSONEachRow',
+    });
+    const rows = await result.json();
+    return Array.isArray(rows) ? rows : [];
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}

package/src/server/sources/kafka.ts

@@ -1,26 +1,14 @@
 import { randomBytes } from 'node:crypto';
 import { Kafka, logLevel, type Consumer, type SASLOptions } from 'kafkajs';
 import type { KafkaSourceConfig } from '../../config/schema.js';
+import { expandEnv } from '../../utils/expandEnv.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
 
 const DEFAULT_MAX_BUFFER = 1000;
 
-function expandEnv(input: string): string {
-  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
-    const value = process.env[name];
-    if (value === undefined) {
-      process.stderr.write(
-        `[pipequery] env var "${name}" referenced in kafka source is not set\n`,
-      );
-      return '';
-    }
-    return value;
-  });
-}
-
 function normalizeBrokers(brokers: string | string[]): string[] {
   const list = Array.isArray(brokers) ? brokers : brokers.split(',');
-  return list.map((b) => expandEnv(b.trim())).filter(Boolean);
+  return list.map((b) => expandEnv(b.trim(), 'kafka source brokers')).filter(Boolean);
 }
 
 export class KafkaSourceAdapter implements SourceAdapter {
@@ -55,8 +43,8 @@ export class KafkaSourceAdapter implements SourceAdapter {
       sasl: this.config.sasl
         ? ({
             mechanism: this.config.sasl.mechanism,
-            username: expandEnv(this.config.sasl.username),
-            password: expandEnv(this.config.sasl.password),
+            username: expandEnv(this.config.sasl.username, 'kafka source SASL username'),
+            password: expandEnv(this.config.sasl.password, 'kafka source SASL password'),
           } as SASLOptions)
         : undefined,
       // Silence kafkajs's own logger — we surface errors via SourceStatus.

package/src/server/sources/manager.ts

@@ -1,5 +1,5 @@
 import type { SourceConfig } from '../../config/schema.js';
-import type { SourceAdapter, SourceStatus } from './types.js';
+import type { PushdownResult, SourceAdapter, SourceStatus } from './types.js';
 import { RestSourceAdapter } from './rest.js';
 import { WebSocketSourceAdapter } from './websocket.js';
 import { FileSourceAdapter } from './file.js';
@@ -8,6 +8,10 @@ import { PostgresSourceAdapter } from './postgres.js';
 import { MysqlSourceAdapter } from './mysql.js';
 import { SqliteSourceAdapter } from './sqlite.js';
 import { KafkaSourceAdapter } from './kafka.js';
+import { SnowflakeSourceAdapter } from './snowflake.js';
+import { ClickhouseSourceAdapter } from './clickhouse.js';
+import { MongoSourceAdapter } from './mongodb.js';
+import { parseQuery, query } from '../../engine.js';
 
 export type DataContext = Record<string, unknown[]>;
 
@@ -98,6 +102,66 @@ export class SourceManager {
     );
   }
 
+  /**
+   * Run a pipe expression, preferring adapter-native push-down when the
+   * source supports it.
+   *
+   * Decision tree:
+   *   1. Parse the expression. If it can't parse, fall straight through to
+   *      the in-process engine, which will throw the same parse error a
+   *      caller would have seen before push-down existed (no behaviour
+   *      change for malformed input).
+   *   2. Look up the source named in the pipeline. If it exists and its
+   *      adapter exposes `runPushdown`, try it. On `{ ok: true }` we're
+   *      done — return the rows. On `{ ok: false }` (decline) or any
+   *      thrown error, fall back to in-process.
+   *   3. In-process: query(getContext(), expression) — the existing path.
+   *
+   * The push-down attempt is best-effort and *invisible to callers*: same
+   * return type as `query()`, no new failure modes. Telemetry (which
+   * branch ran, latency, source-engine SQL) belongs in the audit log when
+   * Phase 5 lands; not threaded through here.
+   */
+  async runQuery(expression: string): Promise<unknown> {
+    const pushed = await this.tryPushdown(expression);
+    if (pushed && pushed.ok) return pushed.rows;
+    return query(this.getContext(), expression);
+  }
+
+  /**
+   * Inspect the pipeline AST and dispatch to an adapter's runPushdown when
+   * eligible. Returns `null` when push-down isn't even attempted (no parse,
+   * no source, no capability). Returns a PushdownResult when it was tried —
+   * `{ ok: true }` for success, `{ ok: false, reason }` for decline.
+   *
+   * Exposed on the manager (rather than tucked inside runQuery) so the
+   * future `event.audit` channel can record `pushed_down: bool` and the
+   * `source_engine_query` SQL string for governance. Today only Postgres
+   * implements runPushdown; MySQL / SQLite / ClickHouse will follow the
+   * same interface.
+   */
+  async tryPushdown(expression: string): Promise<PushdownResult | null> {
+    let pipeline;
+    try {
+      pipeline = parseQuery(expression);
+    } catch {
+      // Bad expression — let the in-process engine raise the canonical error.
+      return null;
+    }
+    const entry = this.sources.get(pipeline.source);
+    if (!entry) return null;
+    if (!entry.adapter.runPushdown) return null;
+    try {
+      return await entry.adapter.runPushdown(expression);
+    } catch (err) {
+      // A throw from runPushdown is treated as a decline: same fallback
+      // path as `{ ok: false }`. We don't surface the thrown error because
+      // the in-process retry will produce a more useful one (or succeed).
+      const reason = err instanceof Error ? err.message : String(err);
+      return { ok: false, reason: `pushdown threw: ${reason}` };
+    }
+  }
+
   async dispose(): Promise<void> {
     this.globalListeners.clear();
     for (const entry of this.sources.values()) {
@@ -125,6 +189,12 @@ export class SourceManager {
         return new SqliteSourceAdapter(config, this.cwd);
       case 'kafka':
         return new KafkaSourceAdapter(config);
+      case 'snowflake':
+        return new SnowflakeSourceAdapter(config);
+      case 'clickhouse':
+        return new ClickhouseSourceAdapter(config);
+      case 'mongodb':
+        return new MongoSourceAdapter(config);
       default:
         throw new Error(`Unknown source type: ${(config as { type: string }).type}`);
     }