@vaultgradient/pipequery-cli 0.4.0 → 0.6.0

package/README.md

@@ -8,6 +8,8 @@ Connect to any data source (REST APIs, WebSockets, files, Postgres, MySQL, SQLit
 
 **Give your AI agent live data** — `pq mcp serve` exposes every configured source to any Model Context Protocol client (Claude Desktop, Claude Code, Cursor, Copilot). Your AI can now query your REST APIs, files, Postgres / MySQL / SQLite databases, Kafka streams, and WebSocket feeds directly.
 
+**Use it from Telegram** — `pq telegram serve` runs a Telegram bot that maps the same MCP commands to chat (`/query`, `/sources`, `/describe`, `/call`). Pair with `pq watch add` to push alerts to a Telegram channel when a query result transitions (`when_non_empty`, `when_empty`, or `on_change`).
+
 ## Installation
 
 ```bash
@@ -267,6 +269,68 @@ Restart Claude Desktop; ask "what pipequery sources are configured?" to verify.
 
 Same idea — consult each tool's MCP config section. The stdio command is `pq mcp serve` with `cwd` pointing at a directory containing `pipequery.yaml`.
 
+## Use from Telegram
+
+`pq telegram serve` runs a Telegram bot that exposes the same five verbs as MCP, mapped to slash commands.
+
+```bash
+PIPEQUERY_TG_BOT_TOKEN=<your-bot-token> pq telegram serve --allow-user @yourname
+
+# Or attach to a running pq serve instance
+pq telegram serve --bot-token <token> --attach http://localhost:3000 --allow-user @yourname
+```
+
+Bot commands:
+
+| Command | What it does |
+|---|---|
+| `/sources` | List configured sources and their health |
+| `/describe <name>` | Sample rows + inferred field names |
+| `/endpoints` | List pre-configured endpoints |
+| `/call <path>` | Execute a pre-configured endpoint by path |
+| `/query <expression>` | Run an arbitrary pipequery expression |
+| `/help` | Show usage |
+
+Without `--allow-user`, the bot replies to anyone who finds it (a stderr warning prints on first message). For anything reachable from the public internet, set the allowlist.
+
+### Natural-language queries
+
+Pass `--anthropic-key` (or set `ANTHROPIC_API_KEY`) and the bot accepts plain-English questions in addition to slash commands. Anything that doesn't start with `/` is translated into a pipequery expression by `claude-haiku-4-5` and executed against the configured sources.
+
+```bash
+ANTHROPIC_API_KEY=sk-ant-... pq telegram serve --allow-user @yourname
+```
+
+Then in Telegram: `top 5 most expensive paid orders` → bot replies with the translated expression and the result.
+
+The translator uses prompt caching for both the system prompt (always-stable DSL grammar) and the per-tenant schema preamble (source list + inferred fields), so repeated queries are cheap. The schema preamble refreshes every minute.
+
+## Watch — alerts to Telegram
+
+`pq watch add` registers a query that runs on every interval; when its result transitions (becomes non-empty, becomes empty, or changes content), pipequery posts a notification to a Telegram chat / channel.
+
+```bash
+# Set up the bot token once
+export PIPEQUERY_TG_BOT_TOKEN=<your-bot-token>
+
+# BTC dropped under $80k? Tell me.
+pq watch add btc_dip \
+  -q "crypto | where(symbol == 'BTC' && price < 80000) | first(1)" \
+  -i 30s \
+  --telegram-chat-id -1001234567890 \
+  --telegram-message "🚨 BTC dipped: \${{ .price }}"
+
+pq watch list
+pq watch remove btc_dip
+```
+
+`fireWhen` modes:
+- **`when_non_empty`** (default) — fire on the empty → non-empty transition. Standard alerting shape; no spam while still triggered, re-fires after going empty and back.
+- **`when_empty`** — inverse, e.g. liveness checks that fire when nothing matches.
+- **`on_change`** — fire whenever the result content changes. Use sparingly; can be chatty.
+
+Message templates support `{{ .field }}` (first-row field), `{{ .count }}` (total row count), `{{ .watchName }}`, and `{{ .reason }}`. If you omit the template, the bot posts a default summary plus the rendered result.
+
 ## Docker & Remote Deployment
 
 The `pq` server and CLI are fully decoupled. You can run the server in Docker (locally or on a remote machine) and control it from your local terminal.

package/__tests__/nl-translator.test.ts

@@ -0,0 +1,228 @@
+/**
+ * NL translator tests.
+ *
+ * The Anthropic SDK is injected (not mocked at the network layer), so these
+ * tests exercise the real shape of the request we send (system blocks with
+ * cache_control breakpoints, single user message) and the parsing of the
+ * response. No live API calls.
+ */
+import { describe, expect, it, beforeEach } from 'vitest';
+import type Anthropic from '@anthropic-ai/sdk';
+import { createNLTranslator } from '../src/telegram/nl.js';
+import type { Provider, ProviderSourceInfo, ProviderSourceDescription } from '../src/mcp/provider.js';
+
+interface CapturedCall {
+  model: string;
+  system: unknown;
+  messages: unknown;
+  max_tokens: number;
+}
+
+function fakeProvider(): Provider {
+  return {
+    async listSources(): Promise<ProviderSourceInfo[]> {
+      return [
+        { name: 'orders', status: { healthy: true, rowCount: 1234, lastFetch: null } },
+        { name: 'users', status: { healthy: true, rowCount: 50, lastFetch: null } },
+      ];
+    },
+    async describeSource(name: string): Promise<ProviderSourceDescription | undefined> {
+      if (name === 'orders') {
+        return {
+          name,
+          status: { healthy: true, rowCount: 1234, lastFetch: null },
+          sample: [{ id: 1, total: 100, status: 'paid' }],
+          fields: ['id', 'total', 'status', 'country'],
+        };
+      }
+      if (name === 'users') {
+        return {
+          name,
+          status: { healthy: true, rowCount: 50, lastFetch: null },
+          sample: [{ id: 1, email: 'a@b.com' }],
+          fields: ['id', 'email', 'banned'],
+        };
+      }
+      return undefined;
+    },
+    async listEndpoints() {
+      return [];
+    },
+    async callEndpoint() {
+      return [];
+    },
+    async runQuery() {
+      return [];
+    },
+    async dispose() {},
+  };
+}
+
+function fakeClient(replyText: string, captured: CapturedCall[]): Anthropic {
+  return {
+    messages: {
+      async create(params: {
+        model: string;
+        system: unknown;
+        messages: unknown;
+        max_tokens: number;
+      }) {
+        captured.push({
+          model: params.model,
+          system: params.system,
+          messages: params.messages,
+          max_tokens: params.max_tokens,
+        });
+        return {
+          id: 'msg_test',
+          type: 'message',
+          role: 'assistant',
+          content: [{ type: 'text', text: replyText }],
+          model: params.model,
+          stop_reason: 'end_turn',
+          stop_sequence: null,
+          usage: { input_tokens: 0, output_tokens: 0 },
+        };
+      },
+    },
+  } as unknown as Anthropic;
+}
+
+describe('nl translator', () => {
+  let captured: CapturedCall[];
+  beforeEach(() => {
+    captured = [];
+  });
+
+  it('parses a clean JSON reply', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient(
+        '{"expression":"orders | sort(total desc) | first(5)","explanation":"Top 5 orders."}',
+        captured,
+      ),
+    });
+    const r = await t.translate('top 5 orders by total');
+    expect(r.expression).toBe('orders | sort(total desc) | first(5)');
+    expect(r.explanation).toBe('Top 5 orders.');
+  });
+
+  it('strips ```json fences before parsing', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient(
+        '```json\n{"expression":"orders | rollup(count() as n)","explanation":"Row count."}\n```',
+        captured,
+      ),
+    });
+    const r = await t.translate('how many orders?');
+    expect(r.expression).toBe('orders | rollup(count() as n)');
+  });
+
+  it('extracts the first JSON object even with leading prose', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient(
+        'Here you go!\n{"expression":"users | first(10)","explanation":"First 10 users."}',
+        captured,
+      ),
+    });
+    const r = await t.translate('show me users');
+    expect(r.expression).toBe('users | first(10)');
+  });
+
+  it('rejects responses without an expression field', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient('{"foo":"bar"}', captured),
+    });
+    await expect(t.translate('whatever')).rejects.toThrow(/expression/);
+  });
+
+  it('rejects non-JSON replies', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient('I cannot help with that.', captured),
+    });
+    await expect(t.translate('whatever')).rejects.toThrow(/non-JSON/);
+  });
+
+  it('sends two cache breakpoints — system prompt and schema preamble', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient(
+        '{"expression":"orders | first(5)","explanation":"x"}',
+        captured,
+      ),
+    });
+    await t.translate('q');
+    expect(captured).toHaveLength(1);
+    const system = captured[0].system as Array<{ type: string; text: string; cache_control?: unknown }>;
+    expect(system).toHaveLength(2);
+    // Both system blocks must carry an ephemeral cache_control breakpoint —
+    // that's the whole point of putting the prompt and schema separately.
+    expect(system[0].cache_control).toEqual({ type: 'ephemeral' });
+    expect(system[1].cache_control).toEqual({ type: 'ephemeral' });
+    // First block is the static prompt, second is the per-tenant schema.
+    expect(system[0].text).toContain('PipeQuery');
+    expect(system[1].text).toContain('orders');
+    expect(system[1].text).toContain('users');
+    expect(system[1].text).toContain('fields:');
+  });
+
+  it('uses claude-haiku-4-5 by default', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient(
+        '{"expression":"orders | first(1)","explanation":"x"}',
+        captured,
+      ),
+    });
+    await t.translate('q');
+    expect(captured[0].model).toBe('claude-haiku-4-5');
+  });
+
+  it('caches the schema preamble across calls within TTL', async () => {
+    let describeCalls = 0;
+    const provider: Provider = {
+      ...fakeProvider(),
+      async describeSource(name: string) {
+        describeCalls++;
+        return {
+          name,
+          status: { healthy: true, rowCount: 1, lastFetch: null },
+          sample: [],
+          fields: ['x'],
+        };
+      },
+    };
+    const t = createNLTranslator(provider, {
+      apiKey: 'sk-test',
+      client: fakeClient(
+        '{"expression":"orders | first(1)","explanation":"x"}',
+        captured,
+      ),
+    });
+    await t.translate('q1');
+    const first = describeCalls;
+    await t.translate('q2');
+    expect(describeCalls).toBe(first); // cached, no second describe round-trip
+
+    t.invalidateSchema();
+    await t.translate('q3');
+    expect(describeCalls).toBeGreaterThan(first);
+  });
+
+  it('allows the model to signal "not answerable" via empty expression', async () => {
+    const t = createNLTranslator(fakeProvider(), {
+      apiKey: 'sk-test',
+      client: fakeClient(
+        '{"expression":"","explanation":"PipeQuery is read-only and cannot drop tables."}',
+        captured,
+      ),
+    });
+    const r = await t.translate('drop the orders table');
+    expect(r.expression).toBe('');
+    expect(r.explanation).toMatch(/read-only/);
+  });
+});

package/__tests__/telegram-format.test.ts

@@ -0,0 +1,62 @@
+import { describe, expect, it } from 'vitest';
+import { escapeHtml, formatError, formatResult } from '../src/telegram/format.js';
+
+describe('telegram format', () => {
+  it('renders an empty array as a soft "(no rows)" message', () => {
+    expect(formatResult([])).toBe('<i>(no rows)</i>');
+  });
+
+  it('renders an object array as a Markdown table inside a <pre>', () => {
+    const out = formatResult([
+      { name: 'Alice', amount: 120.5, status: 'paid' },
+      { name: 'Bob', amount: 45.0, status: 'pending' },
+    ]);
+    expect(out).toContain('<pre>');
+    expect(out).toContain('name');
+    expect(out).toContain('Alice');
+    expect(out).toContain('paid');
+    expect(out).toContain('2 rows');
+  });
+
+  it('truncates with footer when more than maxRows', () => {
+    const rows = Array.from({ length: 100 }, (_, i) => ({ id: i, name: `row${i}` }));
+    const out = formatResult(rows);
+    expect(out).toContain('showing 30 of 100 rows');
+  });
+
+  it('handles scalar arrays via JSON fallback', () => {
+    const out = formatResult([1, 2, 3, 'foo']);
+    expect(out).toContain('<pre>');
+    expect(out).toContain('foo');
+  });
+
+  it('handles single objects via JSON fallback', () => {
+    const out = formatResult({ total: 1377, n: 3 });
+    expect(out).toContain('1377');
+    expect(out).toContain('"n": 3');
+  });
+
+  it('escapes HTML in cell values so user data cannot inject markup', () => {
+    const out = formatResult([{ payload: '<script>alert(1)</script>' }]);
+    expect(out).not.toContain('<script>');
+    expect(out).toContain('&lt;script&gt;');
+  });
+
+  it('formatError wraps in <pre> with the ❌ prefix', () => {
+    const out = formatError(new Error('boom'));
+    expect(out).toMatch(/^❌/);
+    expect(out).toContain('<pre>');
+    expect(out).toContain('boom');
+  });
+
+  it('escapeHtml handles ampersand, lt, gt', () => {
+    expect(escapeHtml('a & b < c > d')).toBe('a &amp; b &lt; c &gt; d');
+  });
+
+  it('truncates very wide cells with an ellipsis', () => {
+    const long = 'x'.repeat(100);
+    const out = formatResult([{ huge: long }]);
+    // Default max col width is 24; the rendered cell should not contain the full 100 chars.
+    expect(out).toContain('xxxxxxxxxxxxxxxxxxxxxxx…');
+  });
+});

package/__tests__/watch-manager.test.ts

@@ -0,0 +1,202 @@
+/**
+ * Watch manager fire-condition tests.
+ *
+ * Doesn't touch Telegram — we replace the notifier with an in-memory recorder
+ * and drive the manager's tick() directly via a tiny private-call escape
+ * hatch. The point is to verify the *transition* logic (when_non_empty,
+ * when_empty, on_change) and the idempotency model.
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { SourceManager } from '../src/server/sources/manager.js';
+import { WatchManager } from '../src/server/watches/manager.js';
+import type { Notifier, NotifyPayload } from '../src/telegram/notifier.js';
+
+class RecordingNotifier implements Notifier {
+  calls: NotifyPayload[] = [];
+  async notify(payload: NotifyPayload): Promise<void> {
+    this.calls.push(payload);
+  }
+}
+
+/**
+ * Build a SourceManager seeded with a single static source whose data we can
+ * mutate between ticks to drive transitions.
+ */
+async function buildManagerWithSource(initial: unknown[]): Promise<{
+  sm: SourceManager;
+  setRows: (rows: unknown[]) => void;
+}> {
+  const sm = new SourceManager(process.cwd());
+  let current = initial;
+  // Inject a fake adapter directly via the public addSource path. The static
+  // adapter would also work but it's easier to introspect a custom one.
+  await sm.addSource('events', { type: 'static', data: initial });
+
+  // Static adapter snapshots data at construction; for these tests we want
+  // mutability, so reach in. This is test-only and lets us simulate "the
+  // source data changed between ticks".
+  const setRows = (rows: unknown[]) => {
+    current = rows;
+    // SourceManager.getContext() reads from adapter.getData() each call —
+    // patch the adapter to return the current array each time.
+    const adapter = (sm as unknown as { sources: Map<string, { adapter: { getData: () => unknown[] } }> }).sources.get('events')!.adapter;
+    adapter.getData = () => current;
+  };
+  setRows(initial);
+  return { sm, setRows };
+}
+
+describe('WatchManager', () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it('when_non_empty fires once on the empty → non-empty transition', async () => {
+    const { sm, setRows } = await buildManagerWithSource([]);
+    const wm = new WatchManager(sm);
+    const recorder = new RecordingNotifier();
+    // Seed via internal start() mirror — register a watch by reaching into
+    // private state so we can attach our recorder. (In production code this
+    // happens through buildNotifiers + WatchConfig.notify; bypassing here.)
+    (wm as unknown as {
+      watches: Map<string, unknown>;
+      startOne: (n: string, c: unknown) => void;
+    }).watches.set('w', {
+      config: { query: 'events | where(severity == "high")', fireWhen: 'when_non_empty' },
+      notifiers: [recorder],
+      intervalMs: 100,
+      timer: null,
+      lastWasEmpty: null,
+      lastHash: null,
+    });
+
+    const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
+
+    // First tick: result is empty (no high-severity events). Should NOT fire
+    // (transitions require a previous state).
+    await tick('w');
+    expect(recorder.calls).toHaveLength(0);
+
+    // Second tick still empty — no fire.
+    await tick('w');
+    expect(recorder.calls).toHaveLength(0);
+
+    // Mutate the source so the query now returns rows.
+    setRows([{ severity: 'high', msg: 'disk full' }]);
+    await tick('w');
+    expect(recorder.calls).toHaveLength(1);
+    expect(recorder.calls[0].reason).toMatch(/non-empty/);
+
+    // Subsequent ticks while still non-empty: no re-fire.
+    await tick('w');
+    await tick('w');
+    expect(recorder.calls).toHaveLength(1);
+
+    // Goes empty again — no fire on this transition (not the configured shape).
+    setRows([]);
+    await tick('w');
+    expect(recorder.calls).toHaveLength(1);
+
+    // And back to non-empty: re-fires.
+    setRows([{ severity: 'high', msg: 'oom' }]);
+    await tick('w');
+    expect(recorder.calls).toHaveLength(2);
+
+    wm.dispose();
+  });
+
+  it('when_empty fires on the inverse transition', async () => {
+    const { sm, setRows } = await buildManagerWithSource([{ severity: 'high', msg: 'oom' }]);
+    const wm = new WatchManager(sm);
+    const recorder = new RecordingNotifier();
+    (wm as unknown as { watches: Map<string, unknown> }).watches.set('w', {
+      config: { query: 'events | where(severity == "high")', fireWhen: 'when_empty' },
+      notifiers: [recorder],
+      intervalMs: 100,
+      timer: null,
+      lastWasEmpty: null,
+      lastHash: null,
+    });
+    const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
+
+    await tick('w');           // first tick, non-empty, no fire
+    expect(recorder.calls).toHaveLength(0);
+
+    setRows([]);
+    await tick('w');           // transition to empty — fires
+    expect(recorder.calls).toHaveLength(1);
+    expect(recorder.calls[0].reason).toMatch(/empty/);
+
+    await tick('w');           // still empty — no re-fire
+    expect(recorder.calls).toHaveLength(1);
+
+    wm.dispose();
+  });
+
+  it('on_change fires whenever the result hash differs', async () => {
+    const { sm, setRows } = await buildManagerWithSource([{ id: 1 }]);
+    const wm = new WatchManager(sm);
+    const recorder = new RecordingNotifier();
+    (wm as unknown as { watches: Map<string, unknown> }).watches.set('w', {
+      config: { query: 'events | first(10)', fireWhen: 'on_change' },
+      notifiers: [recorder],
+      intervalMs: 100,
+      timer: null,
+      lastWasEmpty: null,
+      lastHash: null,
+    });
+    const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
+
+    await tick('w');           // first tick — no fire (no previous hash)
+    expect(recorder.calls).toHaveLength(0);
+
+    await tick('w');           // same data — no fire
+    expect(recorder.calls).toHaveLength(0);
+
+    setRows([{ id: 2 }]);
+    await tick('w');           // different — fires
+    expect(recorder.calls).toHaveLength(1);
+
+    setRows([{ id: 2 }, { id: 3 }]);
+    await tick('w');           // different again — fires again
+    expect(recorder.calls).toHaveLength(2);
+
+    setRows([{ id: 2 }, { id: 3 }]);
+    await tick('w');           // identical to last — no fire
+    expect(recorder.calls).toHaveLength(2);
+
+    wm.dispose();
+  });
+
+  it('a notifier that throws does not stop the watch advancing state', async () => {
+    const { sm, setRows } = await buildManagerWithSource([]);
+    const wm = new WatchManager(sm);
+    const throwing: Notifier = { notify: async () => { throw new Error('telegram down'); } };
+    (wm as unknown as { watches: Map<string, unknown> }).watches.set('w', {
+      config: { query: 'events | first(10)', fireWhen: 'when_non_empty' },
+      notifiers: [throwing],
+      intervalMs: 100,
+      timer: null,
+      lastWasEmpty: null,
+      lastHash: null,
+    });
+    const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
+
+    await tick('w');           // empty
+    setRows([{ id: 1 }]);
+    // tick() awaits the notifier; since the notifier swallows the throw via
+    // the Promise.all(...catch) in WatchManager, the test should resolve.
+    await expect(tick('w')).resolves.toBeUndefined();
+
+    // State did advance (lastWasEmpty is now false), so a subsequent
+    // non-empty tick must NOT re-fire.
+    setRows([{ id: 1 }, { id: 2 }]);
+    await tick('w');
+    // Nothing to assert about the throwing notifier; the success criterion
+    // is simply that we got here.
+    wm.dispose();
+  });
+});