@vaultgradient/pipequery-cli 0.3.3 → 0.6.0

package/src/server/sources/kafka.ts

@@ -0,0 +1,166 @@
+import { randomBytes } from 'node:crypto';
+import { Kafka, logLevel, type Consumer, type SASLOptions } from 'kafkajs';
+import type { KafkaSourceConfig } from '../../config/schema.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_MAX_BUFFER = 1000;
+
+function expandEnv(input: string): string {
+  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
+    const value = process.env[name];
+    if (value === undefined) {
+      process.stderr.write(
+        `[pipequery] env var "${name}" referenced in kafka source is not set\n`,
+      );
+      return '';
+    }
+    return value;
+  });
+}
+
+function normalizeBrokers(brokers: string | string[]): string[] {
+  const list = Array.isArray(brokers) ? brokers : brokers.split(',');
+  return list.map((b) => expandEnv(b.trim())).filter(Boolean);
+}
+
+export class KafkaSourceAdapter implements SourceAdapter {
+  private data: unknown[] = [];
+  private kafka: Kafka | null = null;
+  private consumer: Consumer | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private connected = false;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly maxBuffer: number;
+  private readonly groupId: string;
+
+  constructor(private config: KafkaSourceConfig) {
+    this.maxBuffer = config.maxBuffer ?? DEFAULT_MAX_BUFFER;
+    // Auto-generate a per-process consumer group by default so each pipequery
+    // instance sees the full firehose. Users who want load-balanced consumption
+    // across replicas can set groupId explicitly in the yaml.
+    this.groupId = config.groupId ?? `pipequery-${randomBytes(6).toString('hex')}`;
+  }
+
+  async start(): Promise<void> {
+    const brokers = normalizeBrokers(this.config.brokers);
+    if (brokers.length === 0) {
+      throw new Error('Kafka source has no brokers configured');
+    }
+
+    this.kafka = new Kafka({
+      clientId: `pipequery-${this.config.topic}`,
+      brokers,
+      ssl: this.config.ssl ?? false,
+      sasl: this.config.sasl
+        ? ({
+            mechanism: this.config.sasl.mechanism,
+            username: expandEnv(this.config.sasl.username),
+            password: expandEnv(this.config.sasl.password),
+          } as SASLOptions)
+        : undefined,
+      // Silence kafkajs's own logger — we surface errors via SourceStatus.
+      logLevel: logLevel.ERROR,
+    });
+
+    this.consumer = this.kafka.consumer({ groupId: this.groupId });
+    await this.consumer.connect();
+    await this.consumer.subscribe({
+      topic: this.config.topic,
+      fromBeginning: this.config.fromBeginning ?? false,
+    });
+    this.connected = true;
+    this.error = undefined;
+
+    // Fire-and-forget; kafkajs's run() resolves immediately and keeps running
+    // in the background until disconnect().
+    void this.consumer.run({
+      eachMessage: async ({ message, partition }) => {
+        try {
+          const row = this.decode(message.value, message.key, message.timestamp, partition, message.offset);
+          this.data.push(row);
+          if (this.data.length > this.maxBuffer) {
+            this.data = this.data.slice(-this.maxBuffer);
+          }
+          this.lastFetch = new Date();
+          this.error = undefined;
+          this.notify();
+        } catch {
+          // Skip malformed messages rather than crashing the consumer.
+        }
+      },
+    });
+  }
+
+  stop(): void {
+    this.connected = false;
+    if (this.consumer) {
+      const consumer = this.consumer;
+      this.consumer = null;
+      // Fire-and-forget disconnect; SourceAdapter.stop() is sync, matching
+      // the rest of the adapters.
+      void consumer.disconnect().catch(() => {
+        // Swallow — we're shutting down anyway.
+      });
+    }
+    this.kafka = null;
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: this.connected && !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  private decode(
+    value: Buffer | null,
+    key: Buffer | null,
+    timestamp: string,
+    partition: number,
+    offset: string,
+  ): Record<string, unknown> {
+    const base: Record<string, unknown> = {
+      _kafka_topic: this.config.topic,
+      _kafka_partition: partition,
+      _kafka_offset: offset,
+      _kafka_timestamp: Number.parseInt(timestamp, 10),
+      _kafka_key: key ? key.toString('utf8') : null,
+    };
+
+    if (!value) {
+      return { ...base, value: null };
+    }
+
+    const format = this.config.valueFormat ?? 'json';
+    if (format === 'string') {
+      return { ...base, value: value.toString('utf8') };
+    }
+    if (format === 'raw') {
+      return { ...base, value: value.toString('base64') };
+    }
+    // json
+    const decoded = JSON.parse(value.toString('utf8')) as unknown;
+    if (decoded !== null && typeof decoded === 'object' && !Array.isArray(decoded)) {
+      // Spread the message body into the top-level row so queries can reference
+      // fields directly (e.g. `events | where(event_type == 'order.paid')`).
+      return { ...base, ...(decoded as Record<string, unknown>) };
+    }
+    return { ...base, value: decoded };
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}

package/src/server/sources/manager.ts

@@ -4,6 +4,10 @@ import { RestSourceAdapter } from './rest.js';
 import { WebSocketSourceAdapter } from './websocket.js';
 import { FileSourceAdapter } from './file.js';
 import { StaticSourceAdapter } from './static.js';
+import { PostgresSourceAdapter } from './postgres.js';
+import { MysqlSourceAdapter } from './mysql.js';
+import { SqliteSourceAdapter } from './sqlite.js';
+import { KafkaSourceAdapter } from './kafka.js';
 
 export type DataContext = Record<string, unknown[]>;
 
@@ -113,6 +117,14 @@ export class SourceManager {
         return new FileSourceAdapter(config, this.cwd);
       case 'static':
         return new StaticSourceAdapter(config);
+      case 'postgres':
+        return new PostgresSourceAdapter(config);
+      case 'mysql':
+        return new MysqlSourceAdapter(config);
+      case 'sqlite':
+        return new SqliteSourceAdapter(config, this.cwd);
+      case 'kafka':
+        return new KafkaSourceAdapter(config);
       default:
         throw new Error(`Unknown source type: ${(config as { type: string }).type}`);
     }

package/src/server/sources/mysql.ts

@@ -0,0 +1,128 @@
+import mysql from 'mysql2/promise';
+import type { MysqlSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+
+function expandEnv(input: string): string {
+  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
+    const value = process.env[name];
+    if (value === undefined) {
+      process.stderr.write(
+        `[pipequery] env var "${name}" referenced in mysql source URL is not set\n`,
+      );
+      return '';
+    }
+    return value;
+  });
+}
+
+function resolveSsl(ssl: MysqlSourceConfig['ssl']): mysql.PoolOptions['ssl'] {
+  if (ssl === false) return undefined;
+  if (ssl === 'no-verify') return { rejectUnauthorized: false };
+  return { rejectUnauthorized: true };
+}
+
+export class MysqlSourceAdapter implements SourceAdapter {
+  private pool: mysql.Pool | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+
+  constructor(private config: MysqlSourceConfig) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+  }
+
+  async start(): Promise<void> {
+    const uri = expandEnv(this.config.url);
+    this.pool = mysql.createPool({
+      uri,
+      ssl: resolveSsl(this.config.ssl),
+      connectionLimit: 2,
+      waitForConnections: true,
+      // mysql2 returns BigInt for large integers by default — stringify so the
+      // data serializes cleanly to JSON (which can't encode BigInt).
+      supportBigNumbers: true,
+      bigNumberStrings: true,
+    });
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.pool) {
+      void this.pool.end();
+      this.pool = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.pool) {
+      this.error = 'MySQL pool not initialized';
+      return;
+    }
+    try {
+      // Same safety-cap pattern as Postgres: wrap the user's query and LIMIT
+      // to maxRows+1 so we can detect overflow without reading unbounded rows.
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const [rows] = await this.pool.query(capQuery);
+      const asArray = Array.isArray(rows) ? (rows as unknown[]) : [];
+      if (asArray.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      this.data = asArray;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}

package/src/server/sources/postgres.ts

@@ -0,0 +1,139 @@
+import pg from 'pg';
+import type { PostgresSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+
+/**
+ * Resolve ${ENV_VAR} placeholders in a string from process.env.
+ * Missing vars expand to empty string (with a stderr warning) so a
+ * mis-typed env doesn't silently embed literal "${FOO}" into a
+ * Postgres connection string.
+ */
+function expandEnv(input: string): string {
+  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
+    const value = process.env[name];
+    if (value === undefined) {
+      process.stderr.write(
+        `[pipequery] env var "${name}" referenced in postgres source URL is not set\n`,
+      );
+      return '';
+    }
+    return value;
+  });
+}
+
+function resolveSsl(ssl: PostgresSourceConfig['ssl']): pg.ClientConfig['ssl'] {
+  if (ssl === false) return false;
+  if (ssl === 'no-verify') return { rejectUnauthorized: false };
+  // 'require' (default) — verify certificates.
+  return { rejectUnauthorized: true };
+}
+
+export class PostgresSourceAdapter implements SourceAdapter {
+  private pool: pg.Pool | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+
+  constructor(private config: PostgresSourceConfig) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+  }
+
+  async start(): Promise<void> {
+    const connectionString = expandEnv(this.config.url);
+    this.pool = new pg.Pool({
+      connectionString,
+      ssl: resolveSsl(this.config.ssl),
+      // Small pool — we're read-only polling, not a transactional app.
+      min: 0,
+      max: 2,
+      idleTimeoutMillis: 10_000,
+    });
+
+    // Proactively surface connection errors (e.g. bad creds) on stderr
+    // so the user doesn't hit silent `healthy: false` with no explanation.
+    this.pool.on('error', (err) => {
+      this.error = err.message;
+    });
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    // Fire-and-forget pool shutdown; callers don't await stop().
+    if (this.pool) {
+      void this.pool.end();
+      this.pool = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.pool) {
+      this.error = 'Postgres pool not initialized';
+      return;
+    }
+    try {
+      // Safety cap: wrap the user's query so a runaway SELECT doesn't
+      // eat the process. We add LIMIT maxRows+1 to detect overflow.
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const res = await this.pool.query(capQuery);
+      if (res.rows.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      this.data = res.rows;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}

package/src/server/sources/sqlite.ts

@@ -0,0 +1,114 @@
+import { resolve } from 'node:path';
+import Database from 'better-sqlite3';
+import type { SqliteSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+const IN_MEMORY = ':memory:';
+
+export class SqliteSourceAdapter implements SourceAdapter {
+  private db: Database.Database | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+  private readonly resolvedPath: string;
+  private readonly readonly: boolean;
+
+  constructor(private config: SqliteSourceConfig, cwd: string) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+    this.readonly = config.readonly ?? true;
+    this.resolvedPath = config.path === IN_MEMORY ? IN_MEMORY : resolve(cwd, config.path);
+  }
+
+  async start(): Promise<void> {
+    // In-memory databases can't be read-only and don't share state across
+    // connections, so the readonly flag is silently ignored for :memory:.
+    const opts: Database.Options =
+      this.resolvedPath === IN_MEMORY ? {} : { readonly: this.readonly, fileMustExist: this.readonly };
+    this.db = new Database(this.resolvedPath, opts);
+    // WAL makes concurrent readers/writers safer when the DB is another
+    // process's production file — no effect for :memory:.
+    if (this.resolvedPath !== IN_MEMORY && !this.readonly) {
+      this.db.pragma('journal_mode = WAL');
+    }
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.db) {
+      this.error = 'SQLite database not initialized';
+      return;
+    }
+    try {
+      // Same safety-cap pattern as the other DB adapters.
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const rows = this.db.prepare(capQuery).all();
+      if (rows.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      // better-sqlite3 returns bigints only if defaultSafeIntegers is on;
+      // default mapping is already JSON-serializable (numbers, strings, Buffer).
+      this.data = rows;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}