@vaultgradient/pipequery-cli 0.7.0 → 0.10.0

package/src/server/sources/types.ts

@@ -5,6 +5,26 @@ export interface SourceStatus {
   error?: string;
 }
 
+/**
+ * Result of an adapter-native push-down attempt.
+ *
+ * `ok: true` means the adapter executed the pipe expression natively (e.g.
+ * Postgres compiled it to SQL and ran it on the database) and returned rows
+ * without materializing the source query in memory. `sql` / `params` are
+ * optional telemetry — useful for audit logs and debugging, not required
+ * for callers to act on.
+ *
+ * `ok: false` is the explicit decline channel: the adapter recognised the
+ * shape but declined (unsupported op, multi-source pipeline, etc.). The
+ * caller should fall back to in-process execution against `getContext()`.
+ *
+ * Any thrown error means *neither* path succeeded — let it propagate; the
+ * caller has no useful fallback.
+ */
+export type PushdownResult =
+  | { ok: true; rows: unknown[]; sql?: string; params?: unknown[] }
+  | { ok: false; reason: string };
+
 export interface SourceAdapter {
   start(): Promise<void>;
   stop(): void;
@@ -12,4 +32,11 @@ export interface SourceAdapter {
   getStatus(): SourceStatus;
   onUpdate(callback: (data: unknown[]) => void): () => void;
   refresh?(): Promise<void>;
+  /**
+   * Optional: compile + execute the pipe expression natively against the
+   * underlying engine instead of materializing into memory. When absent
+   * (or when the adapter declines), SourceManager falls back to in-process
+   * execution. Today only PostgresSourceAdapter implements this.
+   */
+  runPushdown?(expression: string): Promise<PushdownResult>;
 }

package/src/telegram/bot.ts

@@ -10,6 +10,7 @@ import { Bot, type Context, GrammyError, HttpError } from 'grammy';
 import type { Provider } from '../mcp/provider.js';
 import { formatError, formatResult, escapeHtml } from './format.js';
 import type { NLTranslator } from './nl.js';
+import type { EventLogger, UserRef } from './event-log.js';
 
 export interface BotOptions {
   /** Whitelist of Telegram user IDs (numeric) or @usernames allowed to query.
@@ -18,12 +19,16 @@ export interface BotOptions {
   /** Optional natural-language translator. If present, plain-text messages
    *  (no leading slash) are routed through it. */
   nl?: NLTranslator;
+  /** Optional structured event logger. When present, every command,
+   *  NL query, and unauthorized attempt is recorded. */
+  logger?: EventLogger;
 }
 
 export function buildBot(token: string, provider: Provider, opts: BotOptions = {}): Bot {
   const bot = new Bot(token);
   const allowList = normalizeAllowList(opts.allowUsers ?? []);
   const nl = opts.nl;
+  const logger = opts.logger;
 
   // Auth middleware — runs before any handler.
   bot.use(async (ctx, next) => {
@@ -36,6 +41,7 @@ export function buildBot(token: string, provider: Provider, opts: BotOptions = {
       return;
     }
     if (!isAllowed(ctx, allowList)) {
+      logger?.log({ kind: 'unauthorized', user: userOf(ctx) });
       await ctx.reply('🔒 You are not authorized to use this bot.');
       return;
     }
@@ -43,6 +49,7 @@ export function buildBot(token: string, provider: Provider, opts: BotOptions = {
   });
 
   bot.command('start', async (ctx) => {
+    const start = Date.now();
     await ctx.reply(
       [
         '👋 <b>pipequery bot</b>',
@@ -59,9 +66,11 @@ export function buildBot(token: string, provider: Provider, opts: BotOptions = {
       ].join('\n'),
       { parse_mode: 'HTML' },
     );
+    logger?.log({ kind: 'command', user: userOf(ctx), cmd: '/start', outcome: 'success', latencyMs: Date.now() - start });
   });
 
   bot.command('help', async (ctx) => {
+    const start = Date.now();
     const lines = [
       '<b>Examples</b>',
       '<code>/sources</code>',
@@ -86,37 +95,73 @@ export function buildBot(token: string, provider: Provider, opts: BotOptions = {
       '• Results &gt; 30 rows are truncated with a footer; tighten your query to see all of them.',
     );
     await ctx.reply(lines.join('\n'), { parse_mode: 'HTML' });
+    logger?.log({ kind: 'command', user: userOf(ctx), cmd: '/help', outcome: 'success', latencyMs: Date.now() - start });
   });
 
   bot.command('sources', async (ctx) => {
+    const start = Date.now();
     try {
       const sources = await provider.listSources();
       if (sources.length === 0) {
         await ctx.reply('<i>No sources configured.</i>', { parse_mode: 'HTML' });
-        return;
+      } else {
+        const lines = sources.map((s) => {
+          const dot = s.status.healthy ? '🟢' : '🔴';
+          const rowCount = s.status.rowCount.toLocaleString();
+          const err = s.status.error ? ` <i>(${escapeHtml(s.status.error)})</i>` : '';
+          return `${dot} <b>${escapeHtml(s.name)}</b> — ${rowCount} rows${err}`;
+        });
+        await ctx.reply(lines.join('\n'), { parse_mode: 'HTML' });
       }
-      const lines = sources.map((s) => {
-        const dot = s.status.healthy ? '🟢' : '🔴';
-        const rowCount = s.status.rowCount.toLocaleString();
-        const err = s.status.error ? ` <i>(${escapeHtml(s.status.error)})</i>` : '';
-        return `${dot} <b>${escapeHtml(s.name)}</b> — ${rowCount} rows${err}`;
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/sources',
+        outcome: 'success',
+        rowCount: sources.length,
+        latencyMs: Date.now() - start,
       });
-      await ctx.reply(lines.join('\n'), { parse_mode: 'HTML' });
     } catch (err) {
       await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/sources',
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: errMessage(err),
+      });
     }
   });
 
   bot.command('describe', async (ctx) => {
+    const start = Date.now();
     const name = ctx.match.trim();
     if (!name) {
       await ctx.reply('Usage: <code>/describe &lt;source-name&gt;</code>', { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/describe',
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: 'missing source name',
+      });
       return;
     }
     try {
       const desc = await provider.describeSource(name, 5);
       if (!desc) {
         await ctx.reply(`Source <code>${escapeHtml(name)}</code> not found.`, { parse_mode: 'HTML' });
+        logger?.log({
+          kind: 'command',
+          user: userOf(ctx),
+          cmd: '/describe',
+          args: name,
+          outcome: 'error',
+          latencyMs: Date.now() - start,
+          error: 'source not found',
+        });
         return;
       }
       const fields = desc.fields.length ? desc.fields.map((f) => `<code>${escapeHtml(f)}</code>`).join(', ') : '<i>(none inferred)</i>';
@@ -125,52 +170,141 @@ export function buildBot(token: string, provider: Provider, opts: BotOptions = {
         `<b>${escapeHtml(name)}</b> — ${desc.status.rowCount.toLocaleString()} rows\n<b>Fields:</b> ${fields}\n\n<b>Sample:</b>\n${sample}`,
         { parse_mode: 'HTML' },
       );
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/describe',
+        args: name,
+        outcome: 'success',
+        rowCount: desc.sample.length,
+        latencyMs: Date.now() - start,
+      });
     } catch (err) {
       await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/describe',
+        args: name,
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: errMessage(err),
+      });
     }
   });
 
   bot.command('endpoints', async (ctx) => {
+    const start = Date.now();
     try {
       const endpoints = await provider.listEndpoints();
       if (endpoints.length === 0) {
         await ctx.reply('<i>No endpoints configured.</i>', { parse_mode: 'HTML' });
-        return;
+      } else {
+        const lines = endpoints.map(
+          (e) => `<b>${escapeHtml(e.path)}</b>\n<code>${escapeHtml(e.config.query)}</code>`,
+        );
+        await ctx.reply(lines.join('\n\n'), { parse_mode: 'HTML' });
       }
-      const lines = endpoints.map(
-        (e) => `<b>${escapeHtml(e.path)}</b>\n<code>${escapeHtml(e.config.query)}</code>`,
-      );
-      await ctx.reply(lines.join('\n\n'), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/endpoints',
+        outcome: 'success',
+        rowCount: endpoints.length,
+        latencyMs: Date.now() - start,
+      });
     } catch (err) {
       await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/endpoints',
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: errMessage(err),
+      });
     }
   });
 
   bot.command('call', async (ctx) => {
+    const start = Date.now();
     const path = ctx.match.trim();
     if (!path) {
       await ctx.reply('Usage: <code>/call &lt;endpoint-path&gt;</code>', { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/call',
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: 'missing path',
+      });
       return;
     }
     try {
       const result = await provider.callEndpoint(path);
       await ctx.reply(formatResult(result), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/call',
+        args: path,
+        outcome: 'success',
+        rowCount: rowCountOf(result),
+        latencyMs: Date.now() - start,
+      });
     } catch (err) {
       await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/call',
+        args: path,
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: errMessage(err),
+      });
     }
   });
 
   bot.command('query', async (ctx) => {
+    const start = Date.now();
     const expr = ctx.match.trim();
     if (!expr) {
       await ctx.reply('Usage: <code>/query &lt;pipequery expression&gt;</code>', { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/query',
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: 'missing expression',
+      });
       return;
     }
     try {
       const result = await provider.runQuery(expr);
       await ctx.reply(formatResult(result), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/query',
+        args: expr,
+        outcome: 'success',
+        rowCount: rowCountOf(result),
+        latencyMs: Date.now() - start,
+      });
     } catch (err) {
       await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'command',
+        user: userOf(ctx),
+        cmd: '/query',
+        args: expr,
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: errMessage(err),
+      });
     }
   });
 
@@ -180,12 +314,21 @@ export function buildBot(token: string, provider: Provider, opts: BotOptions = {
   bot.on('message:text', async (ctx) => {
     const text = ctx.message.text.trim();
     if (!text || text.startsWith('/')) return;
+    const start = Date.now();
     if (!nl) {
       await ctx.reply(
         'I don\'t recognise that. Use <code>/help</code> to see the available commands. ' +
           'Natural-language queries are disabled — start the bot with <code>--anthropic-key</code> to enable them.',
         { parse_mode: 'HTML' },
       );
+      logger?.log({
+        kind: 'nl',
+        user: userOf(ctx),
+        text,
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: 'NL translation disabled',
+      });
       return;
     }
     try {
@@ -194,13 +337,39 @@ export function buildBot(token: string, provider: Provider, opts: BotOptions = {
       if (!expression) {
         const reason = explanation || 'I couldn\'t translate that into a pipequery expression.';
         await ctx.reply(`🤔 ${escapeHtml(reason)}`, { parse_mode: 'HTML' });
+        logger?.log({
+          kind: 'nl',
+          user: userOf(ctx),
+          text,
+          explanation,
+          outcome: 'not_answerable',
+          latencyMs: Date.now() - start,
+        });
         return;
       }
       const result = await provider.runQuery(expression);
       const header = `<i>${escapeHtml(explanation || 'Translated query')}</i>\n<code>${escapeHtml(expression)}</code>\n\n`;
       await ctx.reply(header + formatResult(result), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'nl',
+        user: userOf(ctx),
+        text,
+        expression,
+        explanation,
+        outcome: 'success',
+        rowCount: rowCountOf(result),
+        latencyMs: Date.now() - start,
+      });
     } catch (err) {
       await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+      logger?.log({
+        kind: 'nl',
+        user: userOf(ctx),
+        text,
+        outcome: 'error',
+        latencyMs: Date.now() - start,
+        error: errMessage(err),
+      });
     }
   });
 
@@ -245,3 +414,20 @@ function trackUnauthSeen(id: number, username?: string): void {
       `Restart with --allow-user @yourname to lock it down.\n`,
   );
 }
+
+// ─── Logging helpers ────────────────────────────────────────────────────────
+
+function userOf(ctx: Context): UserRef {
+  return {
+    id: ctx.from?.id,
+    username: ctx.from?.username,
+  };
+}
+
+function rowCountOf(value: unknown): number | undefined {
+  return Array.isArray(value) ? value.length : undefined;
+}
+
+function errMessage(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}

package/src/telegram/event-log.ts

@@ -0,0 +1,234 @@
+/**
+ * Per-message event logging for the Telegram bot.
+ *
+ * Two outputs:
+ *   1. Pretty-printed line on stderr — colored, scannable, always on. This
+ *      is what you see when running `pq telegram serve` in the foreground.
+ *   2. Optional JSONL file — one JSON object per line, machine-readable.
+ *      Opted in via `--log-file <path>` on `pq telegram serve`. Useful for
+ *      `jq`/`grep`/audit-log workflows.
+ *
+ * Pretty-print example:
+ *
+ *   21:15:03 ✓ @andreadito  /sources                          8 rows  12ms
+ *   21:15:24 ✓ @andreadito  "top 5 paid orders"               5 rows  847ms
+ *                           → orders | where(status == "paid") | sort(total desc) | first(5)
+ *   21:15:47 🔒 @bob         unauthorized
+ *   21:16:02 ✗ @andreadito  /query foo                        parse error
+ *   21:17:00 ⚠ @andreadito  "drop the orders table"           not answerable
+ *                           → PipeQuery is read-only and cannot drop tables.
+ *
+ * Format design notes:
+ *   - HH:MM:SS local time (dim) so the eye anchors on the action, not the timestamp.
+ *   - One outcome glyph: ✓ success, ✗ error, 🔒 unauthorized, ⚠ not-answerable.
+ *   - Username column padded so things line up at typical widths.
+ *   - User text/command in white-ish so it's the most-readable element.
+ *   - Stats (rows + ms) right-aligned in dim — present when relevant.
+ *   - Translator output (NL → expression OR not-answerable explanation) goes
+ *     on a continuation line so the primary line stays scannable.
+ */
+import { appendFile } from 'node:fs/promises';
+import chalk from 'chalk';
+
+export type BotEvent =
+  | {
+      kind: 'command';
+      user: UserRef;
+      cmd: string;
+      args?: string;
+      outcome: 'success' | 'error';
+      rowCount?: number;
+      latencyMs: number;
+      error?: string;
+    }
+  | {
+      kind: 'nl';
+      user: UserRef;
+      text: string;
+      expression?: string;
+      explanation?: string;
+      outcome: 'success' | 'error' | 'not_answerable';
+      rowCount?: number;
+      latencyMs: number;
+      error?: string;
+    }
+  | { kind: 'unauthorized'; user: UserRef };
+
+export interface UserRef {
+  id?: number;
+  username?: string;
+}
+
+export interface EventLogger {
+  log(event: BotEvent): void;
+  /** Best-effort flush for the JSONL file. Currently a no-op since
+   *  appendFile awaits its own write; exposed for future buffering. */
+  flush(): Promise<void>;
+}
+
+export interface EventLoggerOptions {
+  /** Optional JSONL file path. When set, every event is also appended
+   *  as one JSON object per line. */
+  filePath?: string;
+  /** Override the stderr writer (for tests). */
+  writeStderr?: (line: string) => void;
+  /** Override the wall clock (for tests). */
+  now?: () => Date;
+}
+
+const USER_WIDTH = 14;
+const ACTION_WIDTH = 36;
+
+export function createEventLogger(opts: EventLoggerOptions = {}): EventLogger {
+  const writeStderr = opts.writeStderr ?? ((line) => process.stderr.write(line));
+  const now = opts.now ?? (() => new Date());
+  const filePath = opts.filePath;
+
+  return {
+    log(event: BotEvent): void {
+      const ts = now();
+      writeStderr(formatPretty(event, ts) + '\n');
+      if (filePath) {
+        const json = JSON.stringify({ ts: ts.toISOString(), ...flattenForJson(event) });
+        // Fire-and-forget write; the bot doesn't await this so a slow disk
+        // doesn't back-pressure incoming Telegram messages. Errors surface
+        // on stderr so a misconfigured path is visible.
+        void appendFile(filePath, json + '\n').catch((err) => {
+          writeStderr(
+            chalk.hex('#ef4444')('  ✗') +
+              ' [pipequery-telegram] log-file write failed: ' +
+              (err instanceof Error ? err.message : String(err)) +
+              '\n',
+          );
+        });
+      }
+    },
+    async flush(): Promise<void> {
+      // Reserved for future buffering. appendFile already awaits internally.
+    },
+  };
+}
+
+// ─── Pretty formatter ───────────────────────────────────────────────────────
+
+export function formatPretty(event: BotEvent, ts: Date): string {
+  const time = chalk.dim(formatTime(ts));
+  const user = chalk.hex('#06b6d4')(padRight(formatUser(event.user), USER_WIDTH));
+
+  if (event.kind === 'unauthorized') {
+    const glyph = chalk.hex('#f59e0b')('🔒');
+    return `  ${time} ${glyph} ${user}  ${chalk.hex('#f59e0b')('unauthorized')}`;
+  }
+
+  if (event.kind === 'command') {
+    const glyph = outcomeGlyph(event.outcome);
+    const action = padRight(`${event.cmd}${event.args ? ' ' + truncate(event.args, 24) : ''}`, ACTION_WIDTH);
+    const stats = formatStats(event.rowCount, event.latencyMs);
+    if (event.outcome === 'error' && event.error) {
+      return `  ${time} ${glyph} ${user}  ${action}  ${chalk.hex('#ef4444')(truncate(event.error, 60))}`;
+    }
+    return `  ${time} ${glyph} ${user}  ${action}  ${stats}`;
+  }
+
+  // NL message
+  const glyph = nlOutcomeGlyph(event.outcome);
+  const action = padRight(`"${truncate(event.text, ACTION_WIDTH - 4)}"`, ACTION_WIDTH);
+  let line: string;
+  if (event.outcome === 'error' && event.error) {
+    line = `  ${time} ${glyph} ${user}  ${action}  ${chalk.hex('#ef4444')(truncate(event.error, 60))}`;
+  } else if (event.outcome === 'not_answerable') {
+    line = `  ${time} ${glyph} ${user}  ${action}  ${chalk.hex('#f59e0b')('not answerable')}`;
+  } else {
+    line = `  ${time} ${glyph} ${user}  ${action}  ${formatStats(event.rowCount, event.latencyMs)}`;
+  }
+  // Continuation line: translated expression (success) or explanation (not_answerable).
+  const continuation =
+    event.outcome === 'success' && event.expression
+      ? chalk.dim('→ ') + chalk.hex('#82aaff')(event.expression)
+      : event.outcome === 'not_answerable' && event.explanation
+        ? chalk.dim('→ ') + chalk.hex('#f59e0b')(event.explanation)
+        : null;
+  if (continuation) {
+    // Indent continuation under the action column for visual alignment.
+    const pad = ' '.repeat(2 + 8 + 1 + 2 + 1 + USER_WIDTH + 2);
+    return `${line}\n${pad}${continuation}`;
+  }
+  return line;
+}
+
+function outcomeGlyph(outcome: 'success' | 'error'): string {
+  return outcome === 'success' ? chalk.hex('#10b981')('✓') : chalk.hex('#ef4444')('✗');
+}
+
+function nlOutcomeGlyph(outcome: 'success' | 'error' | 'not_answerable'): string {
+  if (outcome === 'success') return chalk.hex('#10b981')('✓');
+  if (outcome === 'error') return chalk.hex('#ef4444')('✗');
+  return chalk.hex('#f59e0b')('⚠');
+}
+
+function formatStats(rowCount: number | undefined, latencyMs: number): string {
+  const parts: string[] = [];
+  if (typeof rowCount === 'number') {
+    parts.push(`${rowCount.toLocaleString()} ${rowCount === 1 ? 'row' : 'rows'}`);
+  }
+  parts.push(`${latencyMs}ms`);
+  return chalk.dim(parts.join('  '));
+}
+
+function formatTime(d: Date): string {
+  const hh = String(d.getHours()).padStart(2, '0');
+  const mm = String(d.getMinutes()).padStart(2, '0');
+  const ss = String(d.getSeconds()).padStart(2, '0');
+  return `${hh}:${mm}:${ss}`;
+}
+
+function formatUser(u: UserRef): string {
+  if (u.username) return `@${u.username}`;
+  if (u.id !== undefined) return `id:${u.id}`;
+  return '<unknown>';
+}
+
+function padRight(s: string, width: number): string {
+  // padEnd uses code-unit length, which is fine for ASCII labels we control.
+  // Truncate first so we never overflow the column.
+  const t = truncate(s, width);
+  return t.padEnd(width);
+}
+
+function truncate(s: string, max: number): string {
+  if (s.length <= max) return s;
+  if (max <= 1) return s.slice(0, max);
+  return s.slice(0, max - 1) + '…';
+}
+
+// ─── JSONL serialization ────────────────────────────────────────────────────
+
+function flattenForJson(event: BotEvent): Record<string, unknown> {
+  const userOut: Record<string, unknown> = {};
+  if (event.user.username) userOut.username = event.user.username;
+  if (event.user.id !== undefined) userOut.id = event.user.id;
+  const base = { kind: event.kind, user: userOut };
+  if (event.kind === 'unauthorized') return base;
+  if (event.kind === 'command') {
+    return {
+      ...base,
+      cmd: event.cmd,
+      ...(event.args ? { args: event.args } : {}),
+      outcome: event.outcome,
+      ...(event.rowCount !== undefined ? { rowCount: event.rowCount } : {}),
+      latencyMs: event.latencyMs,
+      ...(event.error ? { error: event.error } : {}),
+    };
+  }
+  // nl
+  return {
+    ...base,
+    text: event.text,
+    ...(event.expression ? { expression: event.expression } : {}),
+    ...(event.explanation ? { explanation: event.explanation } : {}),
+    outcome: event.outcome,
+    ...(event.rowCount !== undefined ? { rowCount: event.rowCount } : {}),
+    latencyMs: event.latencyMs,
+    ...(event.error ? { error: event.error } : {}),
+  };
+}