@vaultgradient/pipequery-cli 0.3.3 → 0.6.0

package/src/server/watches/manager.ts

@@ -0,0 +1,177 @@
+/**
+ * Watch manager — polls each configured watch on its interval, evaluates
+ * the pipequery expression against the live source data, and dispatches
+ * notifications when the watch's fire condition is met.
+ *
+ * Lives alongside the existing endpoint registry in createServer(). Same
+ * lifecycle: start when the server boots, stop when it disposes.
+ *
+ * Idempotency model:
+ *   - For `when_non_empty`: track per-watch `lastWasEmpty` boolean. Fire
+ *     only when the previous tick was empty (or first run) and the current
+ *     tick is non-empty. Re-fires after the result goes empty and back.
+ *   - For `when_empty`: the inverse.
+ *   - For `on_change`: track a hash of the last result; fire on hash change.
+ *
+ * All firing is best-effort. If a notifier throws, the error is logged on
+ * stderr and the watch state advances anyway — we don't want one stuck
+ * notifier to wedge an alert pipeline.
+ */
+import { createHash } from 'node:crypto';
+import { query } from '../../engine.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import * as activity from '../activity-log.js';
+import type { WatchConfig, WatchNotify } from '../../config/schema.js';
+import type { SourceManager } from '../sources/manager.js';
+import type { Notifier } from '../../telegram/notifier.js';
+import { TelegramNotifier } from '../../telegram/notifier.js';
+
+const DEFAULT_INTERVAL_MS = 60_000;
+
+interface WatchState {
+  config: WatchConfig;
+  notifiers: Notifier[];
+  intervalMs: number;
+  timer: ReturnType<typeof setInterval> | null;
+  lastWasEmpty: boolean | null;
+  lastHash: string | null;
+}
+
+export class WatchManager {
+  private watches = new Map<string, WatchState>();
+  private disposed = false;
+
+  constructor(private sourceManager: SourceManager) {}
+
+  start(configs: Record<string, WatchConfig>): void {
+    for (const [name, cfg] of Object.entries(configs)) {
+      try {
+        this.startOne(name, cfg);
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        activity.error(`Watch "${name}" failed to start: ${msg}`);
+        process.stderr.write(`[pipequery-watch] "${name}" failed to start: ${msg}\n`);
+      }
+    }
+  }
+
+  private startOne(name: string, cfg: WatchConfig): void {
+    const notifiers = buildNotifiers(cfg.notify);
+    if (notifiers.length === 0) {
+      throw new Error(`watch has no notification channels configured`);
+    }
+    const intervalMs = cfg.interval ? parseDuration(cfg.interval) : DEFAULT_INTERVAL_MS;
+    const state: WatchState = {
+      config: cfg,
+      notifiers,
+      intervalMs,
+      timer: null,
+      lastWasEmpty: null,
+      lastHash: null,
+    };
+    this.watches.set(name, state);
+    activity.info(`Watch "${name}" registered (every ${cfg.interval ?? '60s'})`);
+
+    // Run once immediately to seed `lastWasEmpty`/`lastHash` — we don't fire
+    // on the very first tick, since "transitioned" requires a previous state.
+    void this.tick(name);
+    state.timer = setInterval(() => void this.tick(name), intervalMs);
+  }
+
+  private async tick(name: string): Promise<void> {
+    if (this.disposed) return;
+    const state = this.watches.get(name);
+    if (!state) return;
+
+    let result: unknown;
+    try {
+      const ctx = this.sourceManager.getContext();
+      result = query(ctx, state.config.query);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      activity.error(`Watch "${name}" query failed: ${msg}`);
+      // Don't advance state on query failure — wait for the next tick.
+      return;
+    }
+
+    const fireWhen = state.config.fireWhen ?? 'when_non_empty';
+    const isEmpty = isResultEmpty(result);
+    const hash = hashResult(result);
+
+    let shouldFire = false;
+    let reason = '';
+    if (fireWhen === 'when_non_empty') {
+      if (state.lastWasEmpty !== false && !isEmpty) {
+        shouldFire = true;
+        reason = 'result transitioned to non-empty';
+      }
+    } else if (fireWhen === 'when_empty') {
+      if (state.lastWasEmpty !== true && isEmpty) {
+        shouldFire = true;
+        reason = 'result transitioned to empty';
+      }
+    } else if (fireWhen === 'on_change') {
+      if (state.lastHash !== null && state.lastHash !== hash) {
+        shouldFire = true;
+        reason = 'result content changed';
+      }
+    }
+
+    state.lastWasEmpty = isEmpty;
+    state.lastHash = hash;
+
+    if (!shouldFire) return;
+
+    activity.info(`Watch "${name}" fired (${reason})`);
+    await Promise.all(
+      state.notifiers.map((n) =>
+        n.notify({ watchName: name, result, reason }).catch((err) => {
+          const msg = err instanceof Error ? err.message : String(err);
+          activity.error(`Watch "${name}" notify failed: ${msg}`);
+          process.stderr.write(`[pipequery-watch] "${name}" notify failed: ${msg}\n`);
+        }),
+      ),
+    );
+  }
+
+  stop(name: string): boolean {
+    const state = this.watches.get(name);
+    if (!state) return false;
+    if (state.timer) clearInterval(state.timer);
+    this.watches.delete(name);
+    return true;
+  }
+
+  list(): string[] {
+    return [...this.watches.keys()];
+  }
+
+  dispose(): void {
+    this.disposed = true;
+    for (const state of this.watches.values()) {
+      if (state.timer) clearInterval(state.timer);
+    }
+    this.watches.clear();
+  }
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function buildNotifiers(notify: WatchNotify): Notifier[] {
+  const out: Notifier[] = [];
+  if (notify.telegram) out.push(new TelegramNotifier(notify.telegram));
+  return out;
+}
+
+function isResultEmpty(result: unknown): boolean {
+  if (result === null || result === undefined) return true;
+  if (Array.isArray(result)) return result.length === 0;
+  if (typeof result === 'object') return Object.keys(result as object).length === 0;
+  return false;
+}
+
+function hashResult(result: unknown): string {
+  // Stable JSON for hashing — skip undefined / sort keys.
+  const json = JSON.stringify(result, (_, v) => (v === undefined ? null : v));
+  return createHash('sha256').update(json).digest('hex');
+}

package/src/telegram/bot.ts

@@ -0,0 +1,247 @@
+/**
+ * Telegram bot transport for pipequery.
+ *
+ * Same shape as the MCP server: takes a Provider (Local or Attached) and
+ * exposes the same five verbs as bot commands. No state in the bot itself —
+ * everything routes through the provider, so behaviour matches the MCP path
+ * exactly (including governance, when that lands).
+ */
+import { Bot, type Context, GrammyError, HttpError } from 'grammy';
+import type { Provider } from '../mcp/provider.js';
+import { formatError, formatResult, escapeHtml } from './format.js';
+import type { NLTranslator } from './nl.js';
+
+export interface BotOptions {
+  /** Whitelist of Telegram user IDs (numeric) or @usernames allowed to query.
+   *  Empty array = anyone with the bot token can query (NOT recommended). */
+  allowUsers?: string[];
+  /** Optional natural-language translator. If present, plain-text messages
+   *  (no leading slash) are routed through it. */
+  nl?: NLTranslator;
+}
+
+export function buildBot(token: string, provider: Provider, opts: BotOptions = {}): Bot {
+  const bot = new Bot(token);
+  const allowList = normalizeAllowList(opts.allowUsers ?? []);
+  const nl = opts.nl;
+
+  // Auth middleware — runs before any handler.
+  bot.use(async (ctx, next) => {
+    if (allowList.size === 0) {
+      // No allowlist configured. Print a warning once per unique user so
+      // operators notice they're running an unauthenticated bot.
+      const id = ctx.from?.id;
+      if (id) trackUnauthSeen(id, ctx.from?.username);
+      await next();
+      return;
+    }
+    if (!isAllowed(ctx, allowList)) {
+      await ctx.reply('🔒 You are not authorized to use this bot.');
+      return;
+    }
+    await next();
+  });
+
+  bot.command('start', async (ctx) => {
+    await ctx.reply(
+      [
+        '👋 <b>pipequery bot</b>',
+        '',
+        'Run pipequery expressions against your configured data sources from chat.',
+        '',
+        '<b>Commands</b>',
+        '/sources — list configured sources and their health',
+        '/describe &lt;name&gt; — sample rows + field names',
+        '/endpoints — list pre-configured endpoints',
+        '/call &lt;path&gt; — execute a pre-configured endpoint',
+        '/query &lt;expression&gt; — run an arbitrary pipequery expression',
+        '/help — this message',
+      ].join('\n'),
+      { parse_mode: 'HTML' },
+    );
+  });
+
+  bot.command('help', async (ctx) => {
+    const lines = [
+      '<b>Examples</b>',
+      '<code>/sources</code>',
+      '<code>/describe products</code>',
+      '<code>/query products | sort(price desc) | first(5)</code>',
+      '<code>/query crypto | where(price &gt; 100) | rollup(avg(price) as avg)</code>',
+      '<code>/call /api/top-coins</code>',
+    ];
+    if (nl) {
+      lines.push(
+        '',
+        '<b>Natural language</b>',
+        'Send any plain-text message (no leading <code>/</code>) and I\'ll translate it into a pipequery expression and run it.',
+        'Example: <i>"top 5 most expensive paid orders"</i>',
+      );
+    }
+    lines.push(
+      '',
+      '<b>Tips</b>',
+      '• Wrap quoted strings carefully — Telegram\'s autocorrect can mangle them. Send long expressions as a separate message.',
+      '• Use <code>/describe &lt;source&gt;</code> to discover field names before constructing a where clause.',
+      '• Results &gt; 30 rows are truncated with a footer; tighten your query to see all of them.',
+    );
+    await ctx.reply(lines.join('\n'), { parse_mode: 'HTML' });
+  });
+
+  bot.command('sources', async (ctx) => {
+    try {
+      const sources = await provider.listSources();
+      if (sources.length === 0) {
+        await ctx.reply('<i>No sources configured.</i>', { parse_mode: 'HTML' });
+        return;
+      }
+      const lines = sources.map((s) => {
+        const dot = s.status.healthy ? '🟢' : '🔴';
+        const rowCount = s.status.rowCount.toLocaleString();
+        const err = s.status.error ? ` <i>(${escapeHtml(s.status.error)})</i>` : '';
+        return `${dot} <b>${escapeHtml(s.name)}</b> — ${rowCount} rows${err}`;
+      });
+      await ctx.reply(lines.join('\n'), { parse_mode: 'HTML' });
+    } catch (err) {
+      await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+    }
+  });
+
+  bot.command('describe', async (ctx) => {
+    const name = ctx.match.trim();
+    if (!name) {
+      await ctx.reply('Usage: <code>/describe &lt;source-name&gt;</code>', { parse_mode: 'HTML' });
+      return;
+    }
+    try {
+      const desc = await provider.describeSource(name, 5);
+      if (!desc) {
+        await ctx.reply(`Source <code>${escapeHtml(name)}</code> not found.`, { parse_mode: 'HTML' });
+        return;
+      }
+      const fields = desc.fields.length ? desc.fields.map((f) => `<code>${escapeHtml(f)}</code>`).join(', ') : '<i>(none inferred)</i>';
+      const sample = formatResult(desc.sample);
+      await ctx.reply(
+        `<b>${escapeHtml(name)}</b> — ${desc.status.rowCount.toLocaleString()} rows\n<b>Fields:</b> ${fields}\n\n<b>Sample:</b>\n${sample}`,
+        { parse_mode: 'HTML' },
+      );
+    } catch (err) {
+      await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+    }
+  });
+
+  bot.command('endpoints', async (ctx) => {
+    try {
+      const endpoints = await provider.listEndpoints();
+      if (endpoints.length === 0) {
+        await ctx.reply('<i>No endpoints configured.</i>', { parse_mode: 'HTML' });
+        return;
+      }
+      const lines = endpoints.map(
+        (e) => `<b>${escapeHtml(e.path)}</b>\n<code>${escapeHtml(e.config.query)}</code>`,
+      );
+      await ctx.reply(lines.join('\n\n'), { parse_mode: 'HTML' });
+    } catch (err) {
+      await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+    }
+  });
+
+  bot.command('call', async (ctx) => {
+    const path = ctx.match.trim();
+    if (!path) {
+      await ctx.reply('Usage: <code>/call &lt;endpoint-path&gt;</code>', { parse_mode: 'HTML' });
+      return;
+    }
+    try {
+      const result = await provider.callEndpoint(path);
+      await ctx.reply(formatResult(result), { parse_mode: 'HTML' });
+    } catch (err) {
+      await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+    }
+  });
+
+  bot.command('query', async (ctx) => {
+    const expr = ctx.match.trim();
+    if (!expr) {
+      await ctx.reply('Usage: <code>/query &lt;pipequery expression&gt;</code>', { parse_mode: 'HTML' });
+      return;
+    }
+    try {
+      const result = await provider.runQuery(expr);
+      await ctx.reply(formatResult(result), { parse_mode: 'HTML' });
+    } catch (err) {
+      await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+    }
+  });
+
+  // Plain-text fallback — only fires if NL translation is enabled. Anything
+  // starting with `/` is already routed by grammy's command matchers above;
+  // bot.on('message:text') still receives them, so we filter explicitly.
+  bot.on('message:text', async (ctx) => {
+    const text = ctx.message.text.trim();
+    if (!text || text.startsWith('/')) return;
+    if (!nl) {
+      await ctx.reply(
+        'I don\'t recognise that. Use <code>/help</code> to see the available commands. ' +
+          'Natural-language queries are disabled — start the bot with <code>--anthropic-key</code> to enable them.',
+        { parse_mode: 'HTML' },
+      );
+      return;
+    }
+    try {
+      await ctx.replyWithChatAction('typing');
+      const { expression, explanation } = await nl.translate(text);
+      if (!expression) {
+        const reason = explanation || 'I couldn\'t translate that into a pipequery expression.';
+        await ctx.reply(`🤔 ${escapeHtml(reason)}`, { parse_mode: 'HTML' });
+        return;
+      }
+      const result = await provider.runQuery(expression);
+      const header = `<i>${escapeHtml(explanation || 'Translated query')}</i>\n<code>${escapeHtml(expression)}</code>\n\n`;
+      await ctx.reply(header + formatResult(result), { parse_mode: 'HTML' });
+    } catch (err) {
+      await ctx.reply(formatError(err), { parse_mode: 'HTML' });
+    }
+  });
+
+  bot.catch((err) => {
+    if (err.error instanceof GrammyError) {
+      process.stderr.write(`[pipequery-telegram] grammy error: ${err.error.description}\n`);
+    } else if (err.error instanceof HttpError) {
+      process.stderr.write(`[pipequery-telegram] http error: ${err.error.message}\n`);
+    } else {
+      process.stderr.write(`[pipequery-telegram] unhandled: ${err.error}\n`);
+    }
+  });
+
+  return bot;
+}
+
+// ─── Auth helpers ───────────────────────────────────────────────────────────
+
+function normalizeAllowList(entries: string[]): Set<string> {
+  const set = new Set<string>();
+  for (const raw of entries) {
+    const trimmed = raw.trim().replace(/^@/, '').toLowerCase();
+    if (trimmed) set.add(trimmed);
+  }
+  return set;
+}
+
+function isAllowed(ctx: Context, allowList: Set<string>): boolean {
+  const from = ctx.from;
+  if (!from) return false;
+  if (allowList.has(String(from.id))) return true;
+  if (from.username && allowList.has(from.username.toLowerCase())) return true;
+  return false;
+}
+
+const seenUnauth = new Set<number>();
+function trackUnauthSeen(id: number, username?: string): void {
+  if (seenUnauth.has(id)) return;
+  seenUnauth.add(id);
+  process.stderr.write(
+    `[pipequery-telegram] WARNING: bot has no allowlist; user "${username ?? id}" can query freely. ` +
+      `Restart with --allow-user @yourname to lock it down.\n`,
+  );
+}

package/src/telegram/format.ts

@@ -0,0 +1,158 @@
+/**
+ * Telegram message formatting for pipequery results.
+ *
+ * Telegram has hard limits we have to respect:
+ *  - Plain message text: 4096 characters
+ *  - MarkdownV2 / HTML escaping rules (we use HTML mode — friendlier to
+ *    pipe expressions and SQL-ish content with underscores)
+ *  - Single message must be self-contained (no streaming/long-polling tricks)
+ *
+ * For result sets that don't fit, we return a truncated rendering with a
+ * row-count footer ("showing 20 of 1247 rows — narrow your query") rather
+ * than dumping the full payload as a file. Files are heavier UX than most
+ * Telegram users want for a quick chat query.
+ */
+
+const TELEGRAM_TEXT_LIMIT = 4096;
+const FENCE_OVERHEAD = 32; // <pre>...</pre>\n\nfooter
+const TRUNCATE_BUDGET = TELEGRAM_TEXT_LIMIT - FENCE_OVERHEAD;
+const MAX_ROWS_INLINE = 30;
+const MAX_COL_WIDTH = 24;
+
+export interface FormatOptions {
+  /** Max rows to render before truncating with a footer. Default 30. */
+  maxRows?: number;
+  /** Max width per column character before ellipsis. Default 24. */
+  maxColWidth?: number;
+}
+
+/**
+ * Render a query result as a Telegram-safe HTML message.
+ *
+ * - Object arrays → fixed-width table inside <pre>
+ * - Scalar arrays → bullet list
+ * - Single objects / scalars → JSON inside <pre>
+ * - Empty results → "(no rows)"
+ *
+ * Always returns a string ≤ 4096 chars.
+ */
+export function formatResult(result: unknown, opts: FormatOptions = {}): string {
+  const maxRows = opts.maxRows ?? MAX_ROWS_INLINE;
+
+  if (Array.isArray(result)) {
+    if (result.length === 0) return '<i>(no rows)</i>';
+
+    const allObjects = result.every(
+      (r) => r !== null && typeof r === 'object' && !Array.isArray(r),
+    );
+
+    if (allObjects) {
+      return renderTable(result as Record<string, unknown>[], result.length, maxRows, opts);
+    }
+    // Mixed / scalar array — render as JSON list.
+    return renderJson(result.slice(0, maxRows), result.length, maxRows);
+  }
+
+  return renderJson(result, 1, 1);
+}
+
+export function formatError(err: unknown): string {
+  const msg = err instanceof Error ? err.message : String(err);
+  return `❌ <b>Error</b>\n<pre>${escapeHtml(msg)}</pre>`;
+}
+
+// ─── Table rendering ────────────────────────────────────────────────────────
+
+function renderTable(
+  rows: Record<string, unknown>[],
+  totalCount: number,
+  maxRows: number,
+  opts: FormatOptions,
+): string {
+  const maxColWidth = opts.maxColWidth ?? MAX_COL_WIDTH;
+  const sample = rows.slice(0, maxRows);
+
+  // Column set = union of all keys in the rendered sample, in first-seen
+  // order. Mirrors the engine's row-shape semantics where rows aren't
+  // required to be schema-uniform.
+  const columns: string[] = [];
+  const seen = new Set<string>();
+  for (const r of sample) {
+    for (const k of Object.keys(r)) {
+      if (!seen.has(k)) {
+        seen.add(k);
+        columns.push(k);
+      }
+    }
+  }
+
+  const widths = columns.map((c) =>
+    Math.min(
+      maxColWidth,
+      Math.max(c.length, ...sample.map((r) => stringifyCell(r[c]).length)),
+    ),
+  );
+
+  const headerRow = columns.map((c, i) => padCell(c, widths[i])).join('  ');
+  const sep = widths.map((w) => '─'.repeat(w)).join('  ');
+  const dataRows = sample.map((r) =>
+    columns.map((c, i) => padCell(stringifyCell(r[c]), widths[i])).join('  '),
+  );
+
+  const lines = [headerRow, sep, ...dataRows];
+  let body = lines.join('\n');
+
+  // Hard cap on body to leave room for footer + fence
+  if (body.length > TRUNCATE_BUDGET) {
+    body = body.slice(0, TRUNCATE_BUDGET - 30) + '\n... (truncated)';
+  }
+
+  let out = `<pre>${escapeHtml(body)}</pre>`;
+  if (totalCount > maxRows) {
+    out += `\n<i>showing ${maxRows} of ${totalCount} rows — narrow your query</i>`;
+  } else {
+    out += `\n<i>${totalCount} row${totalCount === 1 ? '' : 's'}</i>`;
+  }
+  return out;
+}
+
+function stringifyCell(value: unknown): string {
+  if (value === null || value === undefined) return '';
+  if (typeof value === 'string') return value;
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value);
+  // Compact JSON for nested objects / arrays.
+  return JSON.stringify(value);
+}
+
+function padCell(s: string, width: number): string {
+  // Truncate with ellipsis if too wide
+  if (s.length > width) return s.slice(0, width - 1) + '…';
+  return s.padEnd(width, ' ');
+}
+
+// ─── JSON fallback ──────────────────────────────────────────────────────────
+
+function renderJson(data: unknown, totalCount: number, maxRows: number): string {
+  const text = JSON.stringify(data, null, 2);
+  let body = text;
+  if (body.length > TRUNCATE_BUDGET) {
+    body = body.slice(0, TRUNCATE_BUDGET - 30) + '\n... (truncated)';
+  }
+  let out = `<pre>${escapeHtml(body)}</pre>`;
+  if (Array.isArray(data) && totalCount > maxRows) {
+    out += `\n<i>showing ${maxRows} of ${totalCount} rows — narrow your query</i>`;
+  }
+  return out;
+}
+
+// ─── HTML escaping (Telegram's HTML mode) ───────────────────────────────────
+
+const HTML_ENTITIES: Record<string, string> = {
+  '&': '&amp;',
+  '<': '&lt;',
+  '>': '&gt;',
+};
+
+export function escapeHtml(s: string): string {
+  return s.replace(/[&<>]/g, (c) => HTML_ENTITIES[c] ?? c);
+}