@vaultgradient/pipequery-cli 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -1
- package/__tests__/nl-translator.test.ts +228 -0
- package/__tests__/telegram-format.test.ts +62 -0
- package/__tests__/watch-manager.test.ts +202 -0
- package/dist/index.js +1938 -80
- package/package.json +12 -2
- package/src/commands/mcp.ts +87 -0
- package/src/commands/source.ts +63 -1
- package/src/commands/telegram.ts +65 -0
- package/src/commands/watch.ts +84 -0
- package/src/config/schema.ts +153 -1
- package/src/index.ts +80 -6
- package/src/mcp/http.ts +191 -0
- package/src/mcp/provider.ts +223 -0
- package/src/mcp/server.ts +138 -0
- package/src/mcp/stdio.ts +26 -0
- package/src/server/app.ts +10 -1
- package/src/server/sources/kafka.ts +166 -0
- package/src/server/sources/manager.ts +12 -0
- package/src/server/sources/mysql.ts +128 -0
- package/src/server/sources/postgres.ts +139 -0
- package/src/server/sources/sqlite.ts +114 -0
- package/src/server/watches/manager.ts +177 -0
- package/src/telegram/bot.ts +247 -0
- package/src/telegram/format.ts +158 -0
- package/src/telegram/nl.ts +318 -0
- package/src/telegram/notifier.ts +110 -0
package/README.md
CHANGED
|
@@ -2,10 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
A command-line tool for building data pipelines, REST APIs, and terminal dashboards using PipeQuery expressions.
|
|
4
4
|
|
|
5
|
-
Connect to any data source (REST APIs, WebSockets, files), run pipe-based queries to transform and aggregate data, then expose results as API endpoints or visualize them as rich terminal charts.
|
|
5
|
+
Connect to any data source (REST APIs, WebSockets, files, Postgres, MySQL, SQLite, Kafka), run pipe-based queries to transform and aggregate data, then expose results as API endpoints or visualize them as rich terminal charts.
|
|
6
6
|
|
|
7
7
|
**Create API endpoints on the fly** — run `pq endpoint add /api/prices -q "crypto | sort(price desc)"` and instantly get a live JSON endpoint, no config file needed.
|
|
8
8
|
|
|
9
|
+
**Give your AI agent live data** — `pq mcp serve` exposes every configured source to any Model Context Protocol client (Claude Desktop, Claude Code, Cursor, Copilot). Your AI can now query your REST APIs, files, Postgres / MySQL / SQLite databases, Kafka streams, and WebSocket feeds directly.
|
|
10
|
+
|
|
11
|
+
**Use it from Telegram** — `pq telegram serve` runs a Telegram bot that maps the same MCP commands to chat (`/query`, `/sources`, `/describe`, `/call`). Pair with `pq watch add` to push alerts to a Telegram channel when a query result transitions (`when_non_empty`, `when_empty`, or `on_change`).
|
|
12
|
+
|
|
9
13
|
## Installation
|
|
10
14
|
|
|
11
15
|
```bash
|
|
@@ -80,6 +84,29 @@ pq source add coins -t rest -u "https://api.example.com/coins" -i 30s
|
|
|
80
84
|
# Add a file source with live watching
|
|
81
85
|
pq source add orders -t file -p ./data/orders.csv -w
|
|
82
86
|
|
|
87
|
+
# Add a Postgres source (credentials interpolated from env at runtime)
|
|
88
|
+
pq source add users -t postgres \
|
|
89
|
+
-u "postgres://\${DB_USER}:\${DB_PASS}@db.internal:5432/app" \
|
|
90
|
+
-q "SELECT id, email, created_at FROM users WHERE created_at > NOW() - INTERVAL '7 days'" \
|
|
91
|
+
-i 60s
|
|
92
|
+
|
|
93
|
+
# Add a MySQL source
|
|
94
|
+
pq source add orders -t mysql \
|
|
95
|
+
-u "mysql://\${DB_USER}:\${DB_PASS}@db.internal:3306/shop" \
|
|
96
|
+
-q "SELECT id, total, status FROM orders WHERE created_at > NOW() - INTERVAL 1 DAY" \
|
|
97
|
+
-i 60s
|
|
98
|
+
|
|
99
|
+
# Add a SQLite source (path is relative to pipequery.yaml; opens read-only by default)
|
|
100
|
+
pq source add events -t sqlite -p ./events.db \
|
|
101
|
+
-q "SELECT * FROM events WHERE ts > strftime('%s', 'now', '-1 hour')" \
|
|
102
|
+
-i 30s
|
|
103
|
+
|
|
104
|
+
# Add a Kafka / Redpanda source (streaming — last 1000 messages kept in a ring buffer)
|
|
105
|
+
pq source add live_orders -t kafka \
|
|
106
|
+
--brokers "broker1:9092,broker2:9092" \
|
|
107
|
+
--topic order-events \
|
|
108
|
+
--value-format json
|
|
109
|
+
|
|
83
110
|
# Test a source (fetch sample data)
|
|
84
111
|
pq source test coins
|
|
85
112
|
|
|
@@ -92,6 +119,10 @@ Source types:
|
|
|
92
119
|
- **websocket** — Streams data from a WebSocket connection
|
|
93
120
|
- **file** — Reads JSON or CSV files, optionally watches for changes
|
|
94
121
|
- **static** — Inline JSON data defined in config
|
|
122
|
+
- **postgres** — Polls a Postgres query. Supports `${ENV_VAR}` interpolation in the connection URL so credentials stay out of `pipequery.yaml`. Options: `-u` URL, `-q` query, `-i` interval, `--ssl <require\|no-verify\|false>`, `--max-rows <n>` (default 10000).
|
|
123
|
+
- **mysql** — Polls a MySQL (or MariaDB) query. Same env-var interpolation and `--ssl` / `--max-rows` options as `postgres`.
|
|
124
|
+
- **sqlite** — Polls a query against a local SQLite file (or `:memory:`). Opens read-only by default (pass `--no-readonly` to open read-write). Options: `-p` path, `-q` query, `-i` interval, `--max-rows <n>`, `--no-readonly`.
|
|
125
|
+
- **kafka** — Streams messages from a Kafka (or Kafka-compatible, e.g. Redpanda) topic into a bounded ring buffer. Each message is decoded per `valueFormat` and spread into a row alongside `_kafka_topic`/`_kafka_partition`/`_kafka_offset`/`_kafka_timestamp`/`_kafka_key` metadata. Auto-generates a per-process consumer group so each pipequery instance sees the full firehose; set `groupId` explicitly to load-balance across replicas. Options: `--brokers` (comma list, supports `${ENV_VAR}`), `--topic`, `--group-id`, `--from-beginning`, `--value-format <json\|string\|raw>`, `--max-buffer <n>` (default 1000), `--ssl true`. SASL is config-only (not CLI-exposed yet).
|
|
95
126
|
|
|
96
127
|
### `pq endpoint`
|
|
97
128
|
|
|
@@ -181,6 +212,125 @@ pq remote connect https://my-server.example.com
|
|
|
181
212
|
pq remote status
|
|
182
213
|
```
|
|
183
214
|
|
|
215
|
+
## Use with AI (MCP)
|
|
216
|
+
|
|
217
|
+
PipeQuery ships a [Model Context Protocol](https://modelcontextprotocol.io) server so any MCP-aware AI client — Claude Desktop, Claude Code, Cursor, Copilot, custom agents — can query your configured sources directly.
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
# stdio mode — what Claude Desktop / Cursor want
|
|
221
|
+
pq mcp serve
|
|
222
|
+
|
|
223
|
+
# HTTP/SSE mode — for remote clients or future hosted deployments
|
|
224
|
+
pq mcp serve --http --port 3001
|
|
225
|
+
|
|
226
|
+
# HTTP with bearer-token auth (recommended for anything non-localhost)
|
|
227
|
+
PIPEQUERY_MCP_TOKEN=$(openssl rand -hex 32) pq mcp serve --http --port 3001
|
|
228
|
+
# or inline:
|
|
229
|
+
pq mcp serve --http --port 3001 --auth-token "my-secret"
|
|
230
|
+
|
|
231
|
+
# Attach to a running `pq serve` instance instead of loading pipequery.yaml locally
|
|
232
|
+
pq mcp serve --attach http://localhost:3000
|
|
233
|
+
|
|
234
|
+
# Inspect the tool schemas (useful for directory submission / debugging)
|
|
235
|
+
pq mcp inspect
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
**HTTP authentication** — when a token is set (`PIPEQUERY_MCP_TOKEN` env var or `--auth-token` flag), clients must send `Authorization: Bearer <token>` on every request; unauthenticated requests are rejected with `401`. If no token is set, the endpoint is open and the server prints a loud startup warning — fine for localhost, never OK for a publicly reachable deployment.
|
|
239
|
+
|
|
240
|
+
**Tools exposed** to the AI:
|
|
241
|
+
|
|
242
|
+
| Tool | What it does |
|
|
243
|
+
|------|-------------|
|
|
244
|
+
| `query` | Run any PipeQuery expression against live sources |
|
|
245
|
+
| `list_sources` | List configured sources and their health |
|
|
246
|
+
| `describe_source` | Return a sample + inferred field names for one source |
|
|
247
|
+
| `list_endpoints` | List pre-configured endpoints and their stored queries |
|
|
248
|
+
| `call_endpoint` | Execute a pre-configured endpoint by path |
|
|
249
|
+
|
|
250
|
+
### Claude Desktop setup
|
|
251
|
+
|
|
252
|
+
Add to `~/Library/Application Support/Claude/claude_desktop_config.json` (macOS) or `%APPDATA%\Claude\claude_desktop_config.json` (Windows):
|
|
253
|
+
|
|
254
|
+
```json
|
|
255
|
+
{
|
|
256
|
+
"mcpServers": {
|
|
257
|
+
"pipequery": {
|
|
258
|
+
"command": "pq",
|
|
259
|
+
"args": ["mcp", "serve"],
|
|
260
|
+
"cwd": "/path/to/your/project-with-pipequery.yaml"
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Restart Claude Desktop; ask "what pipequery sources are configured?" to verify.
|
|
267
|
+
|
|
268
|
+
### Claude Code / Cursor setup
|
|
269
|
+
|
|
270
|
+
Same idea — consult each tool's MCP config section. The stdio command is `pq mcp serve` with `cwd` pointing at a directory containing `pipequery.yaml`.
|
|
271
|
+
|
|
272
|
+
## Use from Telegram
|
|
273
|
+
|
|
274
|
+
`pq telegram serve` runs a Telegram bot that exposes the same five verbs as MCP, mapped to slash commands.
|
|
275
|
+
|
|
276
|
+
```bash
|
|
277
|
+
PIPEQUERY_TG_BOT_TOKEN=<your-bot-token> pq telegram serve --allow-user @yourname
|
|
278
|
+
|
|
279
|
+
# Or attach to a running pq serve instance
|
|
280
|
+
pq telegram serve --bot-token <token> --attach http://localhost:3000 --allow-user @yourname
|
|
281
|
+
```
|
|
282
|
+
|
|
283
|
+
Bot commands:
|
|
284
|
+
|
|
285
|
+
| Command | What it does |
|
|
286
|
+
|---|---|
|
|
287
|
+
| `/sources` | List configured sources and their health |
|
|
288
|
+
| `/describe <name>` | Sample rows + inferred field names |
|
|
289
|
+
| `/endpoints` | List pre-configured endpoints |
|
|
290
|
+
| `/call <path>` | Execute a pre-configured endpoint by path |
|
|
291
|
+
| `/query <expression>` | Run an arbitrary pipequery expression |
|
|
292
|
+
| `/help` | Show usage |
|
|
293
|
+
|
|
294
|
+
Without `--allow-user`, the bot replies to anyone who finds it (a stderr warning prints on first message). For anything reachable from the public internet, set the allowlist.
|
|
295
|
+
|
|
296
|
+
### Natural-language queries
|
|
297
|
+
|
|
298
|
+
Pass `--anthropic-key` (or set `ANTHROPIC_API_KEY`) and the bot accepts plain-English questions in addition to slash commands. Anything that doesn't start with `/` is translated into a pipequery expression by `claude-haiku-4-5` and executed against the configured sources.
|
|
299
|
+
|
|
300
|
+
```bash
|
|
301
|
+
ANTHROPIC_API_KEY=sk-ant-... pq telegram serve --allow-user @yourname
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
Then in Telegram: `top 5 most expensive paid orders` → bot replies with the translated expression and the result.
|
|
305
|
+
|
|
306
|
+
The translator uses prompt caching for both the system prompt (always-stable DSL grammar) and the per-tenant schema preamble (source list + inferred fields), so repeated queries are cheap. The schema preamble refreshes every minute.
|
|
307
|
+
|
|
308
|
+
## Watch — alerts to Telegram
|
|
309
|
+
|
|
310
|
+
`pq watch add` registers a query that runs on every interval; when its result transitions (becomes non-empty, becomes empty, or changes content), pipequery posts a notification to a Telegram chat / channel.
|
|
311
|
+
|
|
312
|
+
```bash
|
|
313
|
+
# Set up the bot token once
|
|
314
|
+
export PIPEQUERY_TG_BOT_TOKEN=<your-bot-token>
|
|
315
|
+
|
|
316
|
+
# BTC dropped under $80k? Tell me.
|
|
317
|
+
pq watch add btc_dip \
|
|
318
|
+
-q "crypto | where(symbol == 'BTC' && price < 80000) | first(1)" \
|
|
319
|
+
-i 30s \
|
|
320
|
+
--telegram-chat-id -1001234567890 \
|
|
321
|
+
--telegram-message "🚨 BTC dipped: \${{ .price }}"
|
|
322
|
+
|
|
323
|
+
pq watch list
|
|
324
|
+
pq watch remove btc_dip
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
`fireWhen` modes:
|
|
328
|
+
- **`when_non_empty`** (default) — fire on the empty → non-empty transition. Standard alerting shape; no spam while still triggered, re-fires after going empty and back.
|
|
329
|
+
- **`when_empty`** — inverse, e.g. liveness checks that fire when nothing matches.
|
|
330
|
+
- **`on_change`** — fire whenever the result content changes. Use sparingly; can be chatty.
|
|
331
|
+
|
|
332
|
+
Message templates support `{{ .field }}` (first-row field), `{{ .count }}` (total row count), `{{ .watchName }}`, and `{{ .reason }}`. If you omit the template, the bot posts a default summary plus the rendered result.
|
|
333
|
+
|
|
184
334
|
## Docker & Remote Deployment
|
|
185
335
|
|
|
186
336
|
The `pq` server and CLI are fully decoupled. You can run the server in Docker (locally or on a remote machine) and control it from your local terminal.
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NL translator tests.
|
|
3
|
+
*
|
|
4
|
+
* The Anthropic SDK is injected (not mocked at the network layer), so these
|
|
5
|
+
* tests exercise the real shape of the request we send (system blocks with
|
|
6
|
+
* cache_control breakpoints, single user message) and the parsing of the
|
|
7
|
+
* response. No live API calls.
|
|
8
|
+
*/
|
|
9
|
+
import { describe, expect, it, beforeEach } from 'vitest';
|
|
10
|
+
import type Anthropic from '@anthropic-ai/sdk';
|
|
11
|
+
import { createNLTranslator } from '../src/telegram/nl.js';
|
|
12
|
+
import type { Provider, ProviderSourceInfo, ProviderSourceDescription } from '../src/mcp/provider.js';
|
|
13
|
+
|
|
14
|
+
interface CapturedCall {
|
|
15
|
+
model: string;
|
|
16
|
+
system: unknown;
|
|
17
|
+
messages: unknown;
|
|
18
|
+
max_tokens: number;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function fakeProvider(): Provider {
|
|
22
|
+
return {
|
|
23
|
+
async listSources(): Promise<ProviderSourceInfo[]> {
|
|
24
|
+
return [
|
|
25
|
+
{ name: 'orders', status: { healthy: true, rowCount: 1234, lastFetch: null } },
|
|
26
|
+
{ name: 'users', status: { healthy: true, rowCount: 50, lastFetch: null } },
|
|
27
|
+
];
|
|
28
|
+
},
|
|
29
|
+
async describeSource(name: string): Promise<ProviderSourceDescription | undefined> {
|
|
30
|
+
if (name === 'orders') {
|
|
31
|
+
return {
|
|
32
|
+
name,
|
|
33
|
+
status: { healthy: true, rowCount: 1234, lastFetch: null },
|
|
34
|
+
sample: [{ id: 1, total: 100, status: 'paid' }],
|
|
35
|
+
fields: ['id', 'total', 'status', 'country'],
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
if (name === 'users') {
|
|
39
|
+
return {
|
|
40
|
+
name,
|
|
41
|
+
status: { healthy: true, rowCount: 50, lastFetch: null },
|
|
42
|
+
sample: [{ id: 1, email: 'a@b.com' }],
|
|
43
|
+
fields: ['id', 'email', 'banned'],
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
return undefined;
|
|
47
|
+
},
|
|
48
|
+
async listEndpoints() {
|
|
49
|
+
return [];
|
|
50
|
+
},
|
|
51
|
+
async callEndpoint() {
|
|
52
|
+
return [];
|
|
53
|
+
},
|
|
54
|
+
async runQuery() {
|
|
55
|
+
return [];
|
|
56
|
+
},
|
|
57
|
+
async dispose() {},
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function fakeClient(replyText: string, captured: CapturedCall[]): Anthropic {
|
|
62
|
+
return {
|
|
63
|
+
messages: {
|
|
64
|
+
async create(params: {
|
|
65
|
+
model: string;
|
|
66
|
+
system: unknown;
|
|
67
|
+
messages: unknown;
|
|
68
|
+
max_tokens: number;
|
|
69
|
+
}) {
|
|
70
|
+
captured.push({
|
|
71
|
+
model: params.model,
|
|
72
|
+
system: params.system,
|
|
73
|
+
messages: params.messages,
|
|
74
|
+
max_tokens: params.max_tokens,
|
|
75
|
+
});
|
|
76
|
+
return {
|
|
77
|
+
id: 'msg_test',
|
|
78
|
+
type: 'message',
|
|
79
|
+
role: 'assistant',
|
|
80
|
+
content: [{ type: 'text', text: replyText }],
|
|
81
|
+
model: params.model,
|
|
82
|
+
stop_reason: 'end_turn',
|
|
83
|
+
stop_sequence: null,
|
|
84
|
+
usage: { input_tokens: 0, output_tokens: 0 },
|
|
85
|
+
};
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
} as unknown as Anthropic;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
describe('nl translator', () => {
|
|
92
|
+
let captured: CapturedCall[];
|
|
93
|
+
beforeEach(() => {
|
|
94
|
+
captured = [];
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it('parses a clean JSON reply', async () => {
|
|
98
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
99
|
+
apiKey: 'sk-test',
|
|
100
|
+
client: fakeClient(
|
|
101
|
+
'{"expression":"orders | sort(total desc) | first(5)","explanation":"Top 5 orders."}',
|
|
102
|
+
captured,
|
|
103
|
+
),
|
|
104
|
+
});
|
|
105
|
+
const r = await t.translate('top 5 orders by total');
|
|
106
|
+
expect(r.expression).toBe('orders | sort(total desc) | first(5)');
|
|
107
|
+
expect(r.explanation).toBe('Top 5 orders.');
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it('strips ```json fences before parsing', async () => {
|
|
111
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
112
|
+
apiKey: 'sk-test',
|
|
113
|
+
client: fakeClient(
|
|
114
|
+
'```json\n{"expression":"orders | rollup(count() as n)","explanation":"Row count."}\n```',
|
|
115
|
+
captured,
|
|
116
|
+
),
|
|
117
|
+
});
|
|
118
|
+
const r = await t.translate('how many orders?');
|
|
119
|
+
expect(r.expression).toBe('orders | rollup(count() as n)');
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
it('extracts the first JSON object even with leading prose', async () => {
|
|
123
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
124
|
+
apiKey: 'sk-test',
|
|
125
|
+
client: fakeClient(
|
|
126
|
+
'Here you go!\n{"expression":"users | first(10)","explanation":"First 10 users."}',
|
|
127
|
+
captured,
|
|
128
|
+
),
|
|
129
|
+
});
|
|
130
|
+
const r = await t.translate('show me users');
|
|
131
|
+
expect(r.expression).toBe('users | first(10)');
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
it('rejects responses without an expression field', async () => {
|
|
135
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
136
|
+
apiKey: 'sk-test',
|
|
137
|
+
client: fakeClient('{"foo":"bar"}', captured),
|
|
138
|
+
});
|
|
139
|
+
await expect(t.translate('whatever')).rejects.toThrow(/expression/);
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
it('rejects non-JSON replies', async () => {
|
|
143
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
144
|
+
apiKey: 'sk-test',
|
|
145
|
+
client: fakeClient('I cannot help with that.', captured),
|
|
146
|
+
});
|
|
147
|
+
await expect(t.translate('whatever')).rejects.toThrow(/non-JSON/);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
it('sends two cache breakpoints — system prompt and schema preamble', async () => {
|
|
151
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
152
|
+
apiKey: 'sk-test',
|
|
153
|
+
client: fakeClient(
|
|
154
|
+
'{"expression":"orders | first(5)","explanation":"x"}',
|
|
155
|
+
captured,
|
|
156
|
+
),
|
|
157
|
+
});
|
|
158
|
+
await t.translate('q');
|
|
159
|
+
expect(captured).toHaveLength(1);
|
|
160
|
+
const system = captured[0].system as Array<{ type: string; text: string; cache_control?: unknown }>;
|
|
161
|
+
expect(system).toHaveLength(2);
|
|
162
|
+
// Both system blocks must carry an ephemeral cache_control breakpoint —
|
|
163
|
+
// that's the whole point of putting the prompt and schema separately.
|
|
164
|
+
expect(system[0].cache_control).toEqual({ type: 'ephemeral' });
|
|
165
|
+
expect(system[1].cache_control).toEqual({ type: 'ephemeral' });
|
|
166
|
+
// First block is the static prompt, second is the per-tenant schema.
|
|
167
|
+
expect(system[0].text).toContain('PipeQuery');
|
|
168
|
+
expect(system[1].text).toContain('orders');
|
|
169
|
+
expect(system[1].text).toContain('users');
|
|
170
|
+
expect(system[1].text).toContain('fields:');
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it('uses claude-haiku-4-5 by default', async () => {
|
|
174
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
175
|
+
apiKey: 'sk-test',
|
|
176
|
+
client: fakeClient(
|
|
177
|
+
'{"expression":"orders | first(1)","explanation":"x"}',
|
|
178
|
+
captured,
|
|
179
|
+
),
|
|
180
|
+
});
|
|
181
|
+
await t.translate('q');
|
|
182
|
+
expect(captured[0].model).toBe('claude-haiku-4-5');
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it('caches the schema preamble across calls within TTL', async () => {
|
|
186
|
+
let describeCalls = 0;
|
|
187
|
+
const provider: Provider = {
|
|
188
|
+
...fakeProvider(),
|
|
189
|
+
async describeSource(name: string) {
|
|
190
|
+
describeCalls++;
|
|
191
|
+
return {
|
|
192
|
+
name,
|
|
193
|
+
status: { healthy: true, rowCount: 1, lastFetch: null },
|
|
194
|
+
sample: [],
|
|
195
|
+
fields: ['x'],
|
|
196
|
+
};
|
|
197
|
+
},
|
|
198
|
+
};
|
|
199
|
+
const t = createNLTranslator(provider, {
|
|
200
|
+
apiKey: 'sk-test',
|
|
201
|
+
client: fakeClient(
|
|
202
|
+
'{"expression":"orders | first(1)","explanation":"x"}',
|
|
203
|
+
captured,
|
|
204
|
+
),
|
|
205
|
+
});
|
|
206
|
+
await t.translate('q1');
|
|
207
|
+
const first = describeCalls;
|
|
208
|
+
await t.translate('q2');
|
|
209
|
+
expect(describeCalls).toBe(first); // cached, no second describe round-trip
|
|
210
|
+
|
|
211
|
+
t.invalidateSchema();
|
|
212
|
+
await t.translate('q3');
|
|
213
|
+
expect(describeCalls).toBeGreaterThan(first);
|
|
214
|
+
});
|
|
215
|
+
|
|
216
|
+
it('allows the model to signal "not answerable" via empty expression', async () => {
|
|
217
|
+
const t = createNLTranslator(fakeProvider(), {
|
|
218
|
+
apiKey: 'sk-test',
|
|
219
|
+
client: fakeClient(
|
|
220
|
+
'{"expression":"","explanation":"PipeQuery is read-only and cannot drop tables."}',
|
|
221
|
+
captured,
|
|
222
|
+
),
|
|
223
|
+
});
|
|
224
|
+
const r = await t.translate('drop the orders table');
|
|
225
|
+
expect(r.expression).toBe('');
|
|
226
|
+
expect(r.explanation).toMatch(/read-only/);
|
|
227
|
+
});
|
|
228
|
+
});
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { escapeHtml, formatError, formatResult } from '../src/telegram/format.js';
|
|
3
|
+
|
|
4
|
+
describe('telegram format', () => {
|
|
5
|
+
it('renders an empty array as a soft "(no rows)" message', () => {
|
|
6
|
+
expect(formatResult([])).toBe('<i>(no rows)</i>');
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
it('renders an object array as a Markdown table inside a <pre>', () => {
|
|
10
|
+
const out = formatResult([
|
|
11
|
+
{ name: 'Alice', amount: 120.5, status: 'paid' },
|
|
12
|
+
{ name: 'Bob', amount: 45.0, status: 'pending' },
|
|
13
|
+
]);
|
|
14
|
+
expect(out).toContain('<pre>');
|
|
15
|
+
expect(out).toContain('name');
|
|
16
|
+
expect(out).toContain('Alice');
|
|
17
|
+
expect(out).toContain('paid');
|
|
18
|
+
expect(out).toContain('2 rows');
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('truncates with footer when more than maxRows', () => {
|
|
22
|
+
const rows = Array.from({ length: 100 }, (_, i) => ({ id: i, name: `row${i}` }));
|
|
23
|
+
const out = formatResult(rows);
|
|
24
|
+
expect(out).toContain('showing 30 of 100 rows');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('handles scalar arrays via JSON fallback', () => {
|
|
28
|
+
const out = formatResult([1, 2, 3, 'foo']);
|
|
29
|
+
expect(out).toContain('<pre>');
|
|
30
|
+
expect(out).toContain('foo');
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it('handles single objects via JSON fallback', () => {
|
|
34
|
+
const out = formatResult({ total: 1377, n: 3 });
|
|
35
|
+
expect(out).toContain('1377');
|
|
36
|
+
expect(out).toContain('"n": 3');
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('escapes HTML in cell values so user data cannot inject markup', () => {
|
|
40
|
+
const out = formatResult([{ payload: '<script>alert(1)</script>' }]);
|
|
41
|
+
expect(out).not.toContain('<script>');
|
|
42
|
+
expect(out).toContain('<script>');
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
it('formatError wraps in <pre> with the ❌ prefix', () => {
|
|
46
|
+
const out = formatError(new Error('boom'));
|
|
47
|
+
expect(out).toMatch(/^❌/);
|
|
48
|
+
expect(out).toContain('<pre>');
|
|
49
|
+
expect(out).toContain('boom');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('escapeHtml handles ampersand, lt, gt', () => {
|
|
53
|
+
expect(escapeHtml('a & b < c > d')).toBe('a & b < c > d');
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it('truncates very wide cells with an ellipsis', () => {
|
|
57
|
+
const long = 'x'.repeat(100);
|
|
58
|
+
const out = formatResult([{ huge: long }]);
|
|
59
|
+
// Default max col width is 24; the rendered cell should not contain the full 100 chars.
|
|
60
|
+
expect(out).toContain('xxxxxxxxxxxxxxxxxxxxxxx…');
|
|
61
|
+
});
|
|
62
|
+
});
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Watch manager fire-condition tests.
|
|
3
|
+
*
|
|
4
|
+
* Doesn't touch Telegram — we replace the notifier with an in-memory recorder
|
|
5
|
+
* and drive the manager's tick() directly via a tiny private-call escape
|
|
6
|
+
* hatch. The point is to verify the *transition* logic (when_non_empty,
|
|
7
|
+
* when_empty, on_change) and the idempotency model.
|
|
8
|
+
*/
|
|
9
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
10
|
+
import { SourceManager } from '../src/server/sources/manager.js';
|
|
11
|
+
import { WatchManager } from '../src/server/watches/manager.js';
|
|
12
|
+
import type { Notifier, NotifyPayload } from '../src/telegram/notifier.js';
|
|
13
|
+
|
|
14
|
+
class RecordingNotifier implements Notifier {
|
|
15
|
+
calls: NotifyPayload[] = [];
|
|
16
|
+
async notify(payload: NotifyPayload): Promise<void> {
|
|
17
|
+
this.calls.push(payload);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Build a SourceManager seeded with a single static source whose data we can
|
|
23
|
+
* mutate between ticks to drive transitions.
|
|
24
|
+
*/
|
|
25
|
+
async function buildManagerWithSource(initial: unknown[]): Promise<{
|
|
26
|
+
sm: SourceManager;
|
|
27
|
+
setRows: (rows: unknown[]) => void;
|
|
28
|
+
}> {
|
|
29
|
+
const sm = new SourceManager(process.cwd());
|
|
30
|
+
let current = initial;
|
|
31
|
+
// Inject a fake adapter directly via the public addSource path. The static
|
|
32
|
+
// adapter would also work but it's easier to introspect a custom one.
|
|
33
|
+
await sm.addSource('events', { type: 'static', data: initial });
|
|
34
|
+
|
|
35
|
+
// Static adapter snapshots data at construction; for these tests we want
|
|
36
|
+
// mutability, so reach in. This is test-only and lets us simulate "the
|
|
37
|
+
// source data changed between ticks".
|
|
38
|
+
const setRows = (rows: unknown[]) => {
|
|
39
|
+
current = rows;
|
|
40
|
+
// SourceManager.getContext() reads from adapter.getData() each call —
|
|
41
|
+
// patch the adapter to return the current array each time.
|
|
42
|
+
const adapter = (sm as unknown as { sources: Map<string, { adapter: { getData: () => unknown[] } }> }).sources.get('events')!.adapter;
|
|
43
|
+
adapter.getData = () => current;
|
|
44
|
+
};
|
|
45
|
+
setRows(initial);
|
|
46
|
+
return { sm, setRows };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
describe('WatchManager', () => {
|
|
50
|
+
beforeEach(() => {
|
|
51
|
+
vi.useFakeTimers();
|
|
52
|
+
});
|
|
53
|
+
afterEach(() => {
|
|
54
|
+
vi.useRealTimers();
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('when_non_empty fires once on the empty → non-empty transition', async () => {
|
|
58
|
+
const { sm, setRows } = await buildManagerWithSource([]);
|
|
59
|
+
const wm = new WatchManager(sm);
|
|
60
|
+
const recorder = new RecordingNotifier();
|
|
61
|
+
// Seed via internal start() mirror — register a watch by reaching into
|
|
62
|
+
// private state so we can attach our recorder. (In production code this
|
|
63
|
+
// happens through buildNotifiers + WatchConfig.notify; bypassing here.)
|
|
64
|
+
(wm as unknown as {
|
|
65
|
+
watches: Map<string, unknown>;
|
|
66
|
+
startOne: (n: string, c: unknown) => void;
|
|
67
|
+
}).watches.set('w', {
|
|
68
|
+
config: { query: 'events | where(severity == "high")', fireWhen: 'when_non_empty' },
|
|
69
|
+
notifiers: [recorder],
|
|
70
|
+
intervalMs: 100,
|
|
71
|
+
timer: null,
|
|
72
|
+
lastWasEmpty: null,
|
|
73
|
+
lastHash: null,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
|
|
77
|
+
|
|
78
|
+
// First tick: result is empty (no high-severity events). Should NOT fire
|
|
79
|
+
// (transitions require a previous state).
|
|
80
|
+
await tick('w');
|
|
81
|
+
expect(recorder.calls).toHaveLength(0);
|
|
82
|
+
|
|
83
|
+
// Second tick still empty — no fire.
|
|
84
|
+
await tick('w');
|
|
85
|
+
expect(recorder.calls).toHaveLength(0);
|
|
86
|
+
|
|
87
|
+
// Mutate the source so the query now returns rows.
|
|
88
|
+
setRows([{ severity: 'high', msg: 'disk full' }]);
|
|
89
|
+
await tick('w');
|
|
90
|
+
expect(recorder.calls).toHaveLength(1);
|
|
91
|
+
expect(recorder.calls[0].reason).toMatch(/non-empty/);
|
|
92
|
+
|
|
93
|
+
// Subsequent ticks while still non-empty: no re-fire.
|
|
94
|
+
await tick('w');
|
|
95
|
+
await tick('w');
|
|
96
|
+
expect(recorder.calls).toHaveLength(1);
|
|
97
|
+
|
|
98
|
+
// Goes empty again — no fire on this transition (not the configured shape).
|
|
99
|
+
setRows([]);
|
|
100
|
+
await tick('w');
|
|
101
|
+
expect(recorder.calls).toHaveLength(1);
|
|
102
|
+
|
|
103
|
+
// And back to non-empty: re-fires.
|
|
104
|
+
setRows([{ severity: 'high', msg: 'oom' }]);
|
|
105
|
+
await tick('w');
|
|
106
|
+
expect(recorder.calls).toHaveLength(2);
|
|
107
|
+
|
|
108
|
+
wm.dispose();
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('when_empty fires on the inverse transition', async () => {
|
|
112
|
+
const { sm, setRows } = await buildManagerWithSource([{ severity: 'high', msg: 'oom' }]);
|
|
113
|
+
const wm = new WatchManager(sm);
|
|
114
|
+
const recorder = new RecordingNotifier();
|
|
115
|
+
(wm as unknown as { watches: Map<string, unknown> }).watches.set('w', {
|
|
116
|
+
config: { query: 'events | where(severity == "high")', fireWhen: 'when_empty' },
|
|
117
|
+
notifiers: [recorder],
|
|
118
|
+
intervalMs: 100,
|
|
119
|
+
timer: null,
|
|
120
|
+
lastWasEmpty: null,
|
|
121
|
+
lastHash: null,
|
|
122
|
+
});
|
|
123
|
+
const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
|
|
124
|
+
|
|
125
|
+
await tick('w'); // first tick, non-empty, no fire
|
|
126
|
+
expect(recorder.calls).toHaveLength(0);
|
|
127
|
+
|
|
128
|
+
setRows([]);
|
|
129
|
+
await tick('w'); // transition to empty — fires
|
|
130
|
+
expect(recorder.calls).toHaveLength(1);
|
|
131
|
+
expect(recorder.calls[0].reason).toMatch(/empty/);
|
|
132
|
+
|
|
133
|
+
await tick('w'); // still empty — no re-fire
|
|
134
|
+
expect(recorder.calls).toHaveLength(1);
|
|
135
|
+
|
|
136
|
+
wm.dispose();
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('on_change fires whenever the result hash differs', async () => {
|
|
140
|
+
const { sm, setRows } = await buildManagerWithSource([{ id: 1 }]);
|
|
141
|
+
const wm = new WatchManager(sm);
|
|
142
|
+
const recorder = new RecordingNotifier();
|
|
143
|
+
(wm as unknown as { watches: Map<string, unknown> }).watches.set('w', {
|
|
144
|
+
config: { query: 'events | first(10)', fireWhen: 'on_change' },
|
|
145
|
+
notifiers: [recorder],
|
|
146
|
+
intervalMs: 100,
|
|
147
|
+
timer: null,
|
|
148
|
+
lastWasEmpty: null,
|
|
149
|
+
lastHash: null,
|
|
150
|
+
});
|
|
151
|
+
const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
|
|
152
|
+
|
|
153
|
+
await tick('w'); // first tick — no fire (no previous hash)
|
|
154
|
+
expect(recorder.calls).toHaveLength(0);
|
|
155
|
+
|
|
156
|
+
await tick('w'); // same data — no fire
|
|
157
|
+
expect(recorder.calls).toHaveLength(0);
|
|
158
|
+
|
|
159
|
+
setRows([{ id: 2 }]);
|
|
160
|
+
await tick('w'); // different — fires
|
|
161
|
+
expect(recorder.calls).toHaveLength(1);
|
|
162
|
+
|
|
163
|
+
setRows([{ id: 2 }, { id: 3 }]);
|
|
164
|
+
await tick('w'); // different again — fires again
|
|
165
|
+
expect(recorder.calls).toHaveLength(2);
|
|
166
|
+
|
|
167
|
+
setRows([{ id: 2 }, { id: 3 }]);
|
|
168
|
+
await tick('w'); // identical to last — no fire
|
|
169
|
+
expect(recorder.calls).toHaveLength(2);
|
|
170
|
+
|
|
171
|
+
wm.dispose();
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it('a notifier that throws does not stop the watch advancing state', async () => {
|
|
175
|
+
const { sm, setRows } = await buildManagerWithSource([]);
|
|
176
|
+
const wm = new WatchManager(sm);
|
|
177
|
+
const throwing: Notifier = { notify: async () => { throw new Error('telegram down'); } };
|
|
178
|
+
(wm as unknown as { watches: Map<string, unknown> }).watches.set('w', {
|
|
179
|
+
config: { query: 'events | first(10)', fireWhen: 'when_non_empty' },
|
|
180
|
+
notifiers: [throwing],
|
|
181
|
+
intervalMs: 100,
|
|
182
|
+
timer: null,
|
|
183
|
+
lastWasEmpty: null,
|
|
184
|
+
lastHash: null,
|
|
185
|
+
});
|
|
186
|
+
const tick = (wm as unknown as { tick: (name: string) => Promise<void> }).tick.bind(wm);
|
|
187
|
+
|
|
188
|
+
await tick('w'); // empty
|
|
189
|
+
setRows([{ id: 1 }]);
|
|
190
|
+
// tick() awaits the notifier; since the notifier swallows the throw via
|
|
191
|
+
// the Promise.all(...catch) in WatchManager, the test should resolve.
|
|
192
|
+
await expect(tick('w')).resolves.toBeUndefined();
|
|
193
|
+
|
|
194
|
+
// State did advance (lastWasEmpty is now false), so a subsequent
|
|
195
|
+
// non-empty tick must NOT re-fire.
|
|
196
|
+
setRows([{ id: 1 }, { id: 2 }]);
|
|
197
|
+
await tick('w');
|
|
198
|
+
// Nothing to assert about the throwing notifier; the success criterion
|
|
199
|
+
// is simply that we got here.
|
|
200
|
+
wm.dispose();
|
|
201
|
+
});
|
|
202
|
+
});
|