@vaultgradient/pipequery-cli 0.3.3 → 0.4.0

package/src/server/sources/mysql.ts

@@ -0,0 +1,128 @@
+import mysql from 'mysql2/promise';
+import type { MysqlSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+
+function expandEnv(input: string): string {
+  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
+    const value = process.env[name];
+    if (value === undefined) {
+      process.stderr.write(
+        `[pipequery] env var "${name}" referenced in mysql source URL is not set\n`,
+      );
+      return '';
+    }
+    return value;
+  });
+}
+
+function resolveSsl(ssl: MysqlSourceConfig['ssl']): mysql.PoolOptions['ssl'] {
+  if (ssl === false) return undefined;
+  if (ssl === 'no-verify') return { rejectUnauthorized: false };
+  return { rejectUnauthorized: true };
+}
+
+export class MysqlSourceAdapter implements SourceAdapter {
+  private pool: mysql.Pool | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+
+  constructor(private config: MysqlSourceConfig) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+  }
+
+  async start(): Promise<void> {
+    const uri = expandEnv(this.config.url);
+    this.pool = mysql.createPool({
+      uri,
+      ssl: resolveSsl(this.config.ssl),
+      connectionLimit: 2,
+      waitForConnections: true,
+      // mysql2 returns BigInt for large integers by default — stringify so the
+      // data serializes cleanly to JSON (which can't encode BigInt).
+      supportBigNumbers: true,
+      bigNumberStrings: true,
+    });
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.pool) {
+      void this.pool.end();
+      this.pool = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.pool) {
+      this.error = 'MySQL pool not initialized';
+      return;
+    }
+    try {
+      // Same safety-cap pattern as Postgres: wrap the user's query and LIMIT
+      // to maxRows+1 so we can detect overflow without reading unbounded rows.
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const [rows] = await this.pool.query(capQuery);
+      const asArray = Array.isArray(rows) ? (rows as unknown[]) : [];
+      if (asArray.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      this.data = asArray;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}

package/src/server/sources/postgres.ts

@@ -0,0 +1,139 @@
+import pg from 'pg';
+import type { PostgresSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+
+/**
+ * Resolve ${ENV_VAR} placeholders in a string from process.env.
+ * Missing vars expand to empty string (with a stderr warning) so a
+ * mis-typed env doesn't silently embed literal "${FOO}" into a
+ * Postgres connection string.
+ */
+function expandEnv(input: string): string {
+  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
+    const value = process.env[name];
+    if (value === undefined) {
+      process.stderr.write(
+        `[pipequery] env var "${name}" referenced in postgres source URL is not set\n`,
+      );
+      return '';
+    }
+    return value;
+  });
+}
+
+function resolveSsl(ssl: PostgresSourceConfig['ssl']): pg.ClientConfig['ssl'] {
+  if (ssl === false) return false;
+  if (ssl === 'no-verify') return { rejectUnauthorized: false };
+  // 'require' (default) — verify certificates.
+  return { rejectUnauthorized: true };
+}
+
+export class PostgresSourceAdapter implements SourceAdapter {
+  private pool: pg.Pool | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+
+  constructor(private config: PostgresSourceConfig) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+  }
+
+  async start(): Promise<void> {
+    const connectionString = expandEnv(this.config.url);
+    this.pool = new pg.Pool({
+      connectionString,
+      ssl: resolveSsl(this.config.ssl),
+      // Small pool — we're read-only polling, not a transactional app.
+      min: 0,
+      max: 2,
+      idleTimeoutMillis: 10_000,
+    });
+
+    // Proactively surface connection errors (e.g. bad creds) on stderr
+    // so the user doesn't hit silent `healthy: false` with no explanation.
+    this.pool.on('error', (err) => {
+      this.error = err.message;
+    });
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    // Fire-and-forget pool shutdown; callers don't await stop().
+    if (this.pool) {
+      void this.pool.end();
+      this.pool = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.pool) {
+      this.error = 'Postgres pool not initialized';
+      return;
+    }
+    try {
+      // Safety cap: wrap the user's query so a runaway SELECT doesn't
+      // eat the process. We add LIMIT maxRows+1 to detect overflow.
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const res = await this.pool.query(capQuery);
+      if (res.rows.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      this.data = res.rows;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}

package/src/server/sources/sqlite.ts

@@ -0,0 +1,114 @@
+import { resolve } from 'node:path';
+import Database from 'better-sqlite3';
+import type { SqliteSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+const IN_MEMORY = ':memory:';
+
+export class SqliteSourceAdapter implements SourceAdapter {
+  private db: Database.Database | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+  private readonly resolvedPath: string;
+  private readonly readonly: boolean;
+
+  constructor(private config: SqliteSourceConfig, cwd: string) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+    this.readonly = config.readonly ?? true;
+    this.resolvedPath = config.path === IN_MEMORY ? IN_MEMORY : resolve(cwd, config.path);
+  }
+
+  async start(): Promise<void> {
+    // In-memory databases can't be read-only and don't share state across
+    // connections, so the readonly flag is silently ignored for :memory:.
+    const opts: Database.Options =
+      this.resolvedPath === IN_MEMORY ? {} : { readonly: this.readonly, fileMustExist: this.readonly };
+    this.db = new Database(this.resolvedPath, opts);
+    // WAL makes concurrent readers/writers safer when the DB is another
+    // process's production file — no effect for :memory:.
+    if (this.resolvedPath !== IN_MEMORY && !this.readonly) {
+      this.db.pragma('journal_mode = WAL');
+    }
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.db) {
+      this.db.close();
+      this.db = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.db) {
+      this.error = 'SQLite database not initialized';
+      return;
+    }
+    try {
+      // Same safety-cap pattern as the other DB adapters.
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const rows = this.db.prepare(capQuery).all();
+      if (rows.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      // better-sqlite3 returns bigints only if defaultSafeIntegers is on;
+      // default mapping is already JSON-serializable (numbers, strings, Buffer).
+      this.data = rows;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}