@hedeqiang/dsh-nl2sql 0.12.1 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -1
- package/README.md +22 -0
- package/README.zh-CN.md +22 -0
- package/analysis.js +4 -1
- package/bin/nl2sql-mcp.mjs +103 -0
- package/channels/dingtalk.js +11 -9
- package/channels/lark.js +17 -9
- package/channels/message.js +48 -12
- package/channels/telegram.js +13 -6
- package/channels/wecom.js +10 -9
- package/cordis.patch.yml +4 -0
- package/docs/mcp.md +119 -0
- package/filters.js +87 -33
- package/index.js +21 -3
- package/lib/client.js +2122 -153
- package/package.json +6 -1
- package/service/alerts.js +60 -26
- package/service/bots.js +38 -12
- package/service/channels.js +26 -5
- package/service/dashboards.js +5 -2
- package/service/datasources.js +16 -3
- package/service/features.js +21 -0
- package/service/forecast.js +252 -0
- package/service/freshness.js +747 -0
- package/service/mcp.js +677 -0
- package/service/pptx.js +362 -0
- package/service/present.js +516 -0
- package/service/proactive.js +531 -0
- package/service/query.js +39 -10
- package/service/reconcile.js +565 -0
- package/service/shared.js +27 -0
- package/service/subscriptions.js +563 -0
- package/service/uploads.js +4 -1
- package/sql-guard.js +33 -2
- package/xlsx.js +12 -2
package/channels/wecom.js
CHANGED
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
*/
|
|
37
37
|
import { createHash, randomBytes } from 'node:crypto'
|
|
38
38
|
import { ChannelError, postJson } from './http.js'
|
|
39
|
-
import { renderChartPng as defaultRenderChartPng, chartable, resultMeta, isKpi, kpiText, resultAsMarkdownTable, toPlainMarkdown, mentionsAll, splitText } from './message.js'
|
|
39
|
+
import { renderChartPng as defaultRenderChartPng, chartable, cellText, defangMd, fmtNum, resultMeta, isKpi, kpiText, resultAsMarkdownTable, toPlainMarkdown, mentionsAll, splitText } from './message.js'
|
|
40
40
|
|
|
41
41
|
export const WEBHOOK_HOST = 'https://qyapi.weixin.qq.com'
|
|
42
42
|
export const DEFAULT_GATEWAY = 'wss://openws.work.weixin.qq.com'
|
|
@@ -95,8 +95,9 @@ export function toWecomMarkdown(md) {
|
|
|
95
95
|
}
|
|
96
96
|
|
|
97
97
|
function resultBlock(r, { includeSql, maxRows = 10, table = true } = {}) {
|
|
98
|
-
|
|
99
|
-
|
|
98
|
+
// Data-derived text (title, error, values) is defanged: no links / tags / mentions from data.
|
|
99
|
+
const head = `**${defangMd(r.title || '查询结果')}** ${resultMeta(r)}`
|
|
100
|
+
if (r.error) return `${head}\n${defangMd(r.error)}`
|
|
100
101
|
const lines = [head]
|
|
101
102
|
if (!r.rows?.length) lines.push('无数据')
|
|
102
103
|
else if (isKpi(r)) lines.push(kpiText(r, { separator: '\n' }))
|
|
@@ -109,7 +110,7 @@ function resultBlock(r, { includeSql, maxRows = 10, table = true } = {}) {
|
|
|
109
110
|
function resultAsText(r, maxRows) {
|
|
110
111
|
const cols = r.columns.slice(0, 6)
|
|
111
112
|
const rows = r.rows.slice(0, maxRows)
|
|
112
|
-
const out = [`> ${cols.map(c => c.name).join(' | ')}`, ...rows.map(row => `> ${cols.map((c, i) =>
|
|
113
|
+
const out = [`> ${cols.map(c => defangMd(c.name)).join(' | ')}`, ...rows.map(row => `> ${cols.map((c, i) => defangMd(c.kind === 'number' ? fmtNum(row[i]) : cellText(row[i], 60))).join(' | ')}`)]
|
|
113
114
|
if ((r.rowCount ?? r.rows.length) > rows.length) out.push(`<font color="comment">…共 ${r.rowCount ?? r.rows.length} 行</font>`)
|
|
114
115
|
return out.join('\n')
|
|
115
116
|
}
|
|
@@ -121,19 +122,19 @@ function resultAsText(r, maxRows) {
|
|
|
121
122
|
export function planWebhook(message) {
|
|
122
123
|
const sections = []
|
|
123
124
|
const images = []
|
|
124
|
-
if (message.title) sections.push(`## ${message.title}`)
|
|
125
|
+
if (message.title) sections.push(`## ${defangMd(message.title)}`)
|
|
125
126
|
if (message.text) sections.push(message.text)
|
|
126
127
|
let hasTable = /(^|\n)\|[^\n]*\|\s*\n\|[\s:|-]+\|/.test(message.text ?? '')
|
|
127
128
|
for (const r of (message.results ?? []).slice(0, 10)) {
|
|
128
129
|
if (message.chartMode !== 'table' && !r.error && chartable(r)) {
|
|
129
|
-
sections.push(`**${r.title || '查询结果'}** ${resultMeta(r)}(见下方图表)${r.sql && message.includeSql ? '\n```sql\n' + r.sql.slice(0, 1500) + '\n```' : ''}`)
|
|
130
|
+
sections.push(`**${defangMd(r.title || '查询结果')}** ${resultMeta(r)}(见下方图表)${r.sql && message.includeSql ? '\n```sql\n' + r.sql.slice(0, 1500) + '\n```' : ''}`)
|
|
130
131
|
images.push({ msgtype: 'image', result: r })
|
|
131
132
|
continue
|
|
132
133
|
}
|
|
133
134
|
if (r.rows?.length && !isKpi(r) && !r.error) hasTable = true
|
|
134
135
|
sections.push(resultBlock(r, { includeSql: message.includeSql }))
|
|
135
136
|
}
|
|
136
|
-
if (message.footer) sections.push(message.footer)
|
|
137
|
+
if (message.footer) sections.push(defangMd(toPlainMarkdown(message.footer, { mentionAll: '' })))
|
|
137
138
|
const md = sections.filter(Boolean).join('\n\n')
|
|
138
139
|
const msgtype = hasTable ? 'markdown_v2' : 'markdown'
|
|
139
140
|
const content = msgtype === 'markdown_v2' ? toPlainMarkdown(md, { mentionAll: '' }) : toWecomMarkdown(md)
|
|
@@ -238,10 +239,10 @@ function inboundText(body) {
|
|
|
238
239
|
|
|
239
240
|
function aibotMarkdown(message) {
|
|
240
241
|
const parts = []
|
|
241
|
-
if (message.title) parts.push(`**${message.title}**`)
|
|
242
|
+
if (message.title) parts.push(`**${defangMd(message.title)}**`)
|
|
242
243
|
if (message.text) parts.push(toPlainMarkdown(message.text, { mentionAll: '' }))
|
|
243
244
|
for (const r of (message.results ?? []).slice(0, 10)) parts.push(resultBlock(r, { includeSql: message.includeSql }))
|
|
244
|
-
if (message.footer) parts.push(toPlainMarkdown(message.footer, { mentionAll: '' }))
|
|
245
|
+
if (message.footer) parts.push(defangMd(toPlainMarkdown(message.footer, { mentionAll: '' })))
|
|
245
246
|
return parts.join('\n\n')
|
|
246
247
|
}
|
|
247
248
|
|
package/cordis.patch.yml
CHANGED
|
@@ -39,6 +39,10 @@
|
|
|
39
39
|
13. Local file data sources (type sqlite) hold user-uploaded CSV/Excel tables; write SQLite SQL for them (strftime/date, double-quoted identifiers, Chinese column names must be quoted).
|
|
40
40
|
14. Messages tagged [飞书…], [Telegram…], [企业微信…] or [钉钉…] come from IM chat members through a bot: answer in Markdown without asking questions (state assumptions instead); a member marked 只读成员 may only use nl2sql_metric or reuse verified example SQL from nl2sql_knowledge — if a query is refused for row-level or viewer restrictions, explain the restriction instead of retrying variations.
|
|
41
41
|
15. When the user confirms an answer is correct (e.g. "对", "没问题", "就是这个"), offer to store it; on agreement call nl2sql_knowledge action=save_example with the question and the final SQL.
|
|
42
|
+
16. In IM bot chats only: for "每天 9 点把这个发到群里" / "每周一推送" call nl2sql_subscribe (resultId of the result just shown, or a metric name + grain/range; time HH:MM, period daily/weekly/monthly); for "超过 X 提醒我" / "下跌 20% 告诉我" call nl2sql_alert_create (resultId or metric, condition threshold/change/anomaly, mentionMe when they say 提醒我); for "我的订阅" / "取消订阅" / "暂停告警" call nl2sql_subscriptions (list first, then pause/resume/cancel by id). Deliveries go to the current chat; confirm the schedule in one sentence. Page users are told to use 管理 → 告警与报告 instead.
|
|
43
|
+
17. For 对比 / 对账 / 核对 questions spanning two data sources (or two queries on one source) — e.g. "核对订单库和财务库昨天的金额", "两边的笔数对得上吗" — call nl2sql_reconcile once with one SQL per side (aggregate both to the same grain and return the key columns plus the values to compare, aliased consistently), keys and compare columns (add a small tolerance for amounts with rounding differences); do not run the two queries separately and compare by hand. Then explain the outcome in plain language: match rate, how many keys exist only on one side, which columns disagree and the size of the total differences, plus warnings (truncation, duplicate keys, masked keys); suggest likely causes (time-zone or date boundary, status filters, late-arriving data) and offer to save it as a scheduled reconciliation in 管理 → 对账.
|
|
44
|
+
18. For 预测 / forecast / "下周会是多少" questions, first get a daily (or weekly/monthly) time series with ONE measure (nl2sql_metric with grain, or nl2sql_query with time + value), then call nl2sql_forecast with its resultId and horizon; report the point forecast with its 95% range, name the method, and say it is a statistical extrapolation that ignores promotions, holidays and other known events.
|
|
45
|
+
19. For questions about recent periods (today / yesterday / this week / 最近 / 今天 / 昨天 / 本周), call nl2sql_freshness with the tables you query; when a table is stale or lagging (status stale/warn), say so before the numbers, e.g. "⚠️ withdrawals 最新数据停在昨天 17:55", and do not present missing recent data as a real drop.
|
|
42
46
|
|
|
43
47
|
You may only read data. Never attempt to modify data or schema; the tools reject anything but SELECT/SHOW/DESC/EXPLAIN.
|
|
44
48
|
- id: time-context
|
package/docs/mcp.md
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# MCP server
|
|
2
|
+
|
|
3
|
+
dsh-nl2sql can expose its governed data access to other AI tools — Claude Desktop, Cursor, Cline, VS Code (GitHub Copilot agent mode) and any other [Model Context Protocol](https://modelcontextprotocol.io/specification/) client. External models get the same guard rails as the built-in Data Q&A agent:
|
|
4
|
+
|
|
5
|
+
- every statement passes **read-only admission** (one SELECT / WITH / SHOW / EXPLAIN; writes, DDL, multiple statements and system catalogs are refused);
|
|
6
|
+
- **access policies** (hidden tables and columns, masked columns) and **sensitive-column masking** apply;
|
|
7
|
+
- each token runs as an **actor** — the local owner, an existing **member** (role, data source scope, row-level rules) or a **role + data sources** (+ optional row rule);
|
|
8
|
+
- the **privacy setting** "result data sent to the model" decides whether the model sees rows, per-column statistics, or only row counts;
|
|
9
|
+
- every call is **audited** in History with source `mcp` and the token name.
|
|
10
|
+
|
|
11
|
+
The server runs inside DeepSeek Harness, so Harness must be running while you use it.
|
|
12
|
+
|
|
13
|
+
## Enable it
|
|
14
|
+
|
|
15
|
+
1. Open **Data Q&A → 管理 / Admin → MCP 接入 / MCP access**.
|
|
16
|
+
2. Turn the server on. It listens on `http://127.0.0.1:47860/mcp` by default (change the port there; the server restarts automatically). It is **off by default** and never binds to other interfaces.
|
|
17
|
+
3. Click **New token**, give it a name and choose its scope:
|
|
18
|
+
- **Local owner** — the same access as asking on the page (policies still apply);
|
|
19
|
+
- **A member** — follows that member's role, data sources and row-level rules, live; disabling or deleting the member makes the token stop working;
|
|
20
|
+
- **Role + data sources** — analyst / viewer / admin on the selected data sources, with an optional row rule (`column IN (values)`).
|
|
21
|
+
4. Copy the token — it is shown **once**. Only its SHA-256 hash is stored; if you lose it, revoke it and create a new one.
|
|
22
|
+
|
|
23
|
+
The page shows ready-to-copy snippets for the clients below, filled in with the new token.
|
|
24
|
+
|
|
25
|
+
## Claude Desktop (stdio bridge)
|
|
26
|
+
|
|
27
|
+
Claude Desktop launches local MCP servers as processes speaking stdio. The package ships a small bridge, `nl2sql-mcp` (plain Node ≥ 18, no dependencies), that relays stdio to the HTTP endpoint. Add to `claude_desktop_config.json` (Settings → Developer → Edit Config):
|
|
28
|
+
|
|
29
|
+
```json
|
|
30
|
+
{
|
|
31
|
+
"mcpServers": {
|
|
32
|
+
"nl2sql": {
|
|
33
|
+
"command": "npx",
|
|
34
|
+
"args": ["-y", "-p", "@hedeqiang/dsh-nl2sql", "nl2sql-mcp"],
|
|
35
|
+
"env": {
|
|
36
|
+
"NL2SQL_MCP_URL": "http://127.0.0.1:47860/mcp",
|
|
37
|
+
"NL2SQL_MCP_TOKEN": "<YOUR_TOKEN>"
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
You can run the bridge by hand to check it: `NL2SQL_MCP_TOKEN=… npx -y -p @hedeqiang/dsh-nl2sql nl2sql-mcp`, then type `{"jsonrpc":"2.0","id":1,"method":"tools/list"}`.
|
|
45
|
+
|
|
46
|
+
| Variable | Default | Meaning |
|
|
47
|
+
|---|---|---|
|
|
48
|
+
| `NL2SQL_MCP_URL` | `http://127.0.0.1:47860/mcp` | The MCP endpoint shown in the page |
|
|
49
|
+
| `NL2SQL_MCP_TOKEN` | — | Bearer token (required) |
|
|
50
|
+
| `NL2SQL_MCP_TIMEOUT_MS` | 660000 | Per-request timeout |
|
|
51
|
+
|
|
52
|
+
## Cursor
|
|
53
|
+
|
|
54
|
+
Cursor speaks Streamable HTTP directly. Add to `~/.cursor/mcp.json` (or `.cursor/mcp.json` in a project):
|
|
55
|
+
|
|
56
|
+
```json
|
|
57
|
+
{
|
|
58
|
+
"mcpServers": {
|
|
59
|
+
"nl2sql": {
|
|
60
|
+
"url": "http://127.0.0.1:47860/mcp",
|
|
61
|
+
"headers": { "Authorization": "Bearer <YOUR_TOKEN>" }
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
## VS Code
|
|
68
|
+
|
|
69
|
+
Add to `.vscode/mcp.json` (or the user-level MCP configuration):
|
|
70
|
+
|
|
71
|
+
```json
|
|
72
|
+
{
|
|
73
|
+
"servers": {
|
|
74
|
+
"nl2sql": {
|
|
75
|
+
"type": "http",
|
|
76
|
+
"url": "http://127.0.0.1:47860/mcp",
|
|
77
|
+
"headers": { "Authorization": "Bearer <YOUR_TOKEN>" }
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Cline and other clients: use the HTTP URL with an `Authorization: Bearer …` header when the client supports remote/HTTP servers, otherwise the stdio bridge command above.
|
|
84
|
+
|
|
85
|
+
## What the server offers
|
|
86
|
+
|
|
87
|
+
**Tools** (all read-only):
|
|
88
|
+
|
|
89
|
+
| Tool | What it does |
|
|
90
|
+
|---|---|
|
|
91
|
+
| `list_datasources` | Data sources this token may use, with engine and SQL dialect |
|
|
92
|
+
| `get_schema` | Schema overview (visible tables, column roles, relations, dialect notes, glossary, metrics) or full detail of `tables` |
|
|
93
|
+
| `query` | One read-only SQL statement → columns, rows / statistics / count per privacy setting, `rowCount`, `truncated`, `resultId` |
|
|
94
|
+
| `query_metric` | A governed metric from the metric library (grain, dimensions, range, filters) |
|
|
95
|
+
| `search_knowledge` | Verified question→SQL examples, glossary and metric definitions related to a question |
|
|
96
|
+
| `drill` | Breakdown / detail of an earlier `resultId` (row rules re-applied) |
|
|
97
|
+
| `explain_change` | Attribution: which dimensions explain the change between two periods of a result |
|
|
98
|
+
| `reconcile` | Compare two queries (e.g. the same figures from two databases) — listed when the reconcile feature is installed |
|
|
99
|
+
| `forecast` | Forecast a time-series result — listed when the forecast feature is installed |
|
|
100
|
+
|
|
101
|
+
Tool errors (refused SQL, row-level restrictions, unknown tables) come back as results with `isError: true` so the model can correct itself.
|
|
102
|
+
|
|
103
|
+
**Resources:** `nl2sql://datasource/{id}/schema` for every visible data source (and `nl2sql://datasource/{id}/table/{table}` for one table), as plain text.
|
|
104
|
+
|
|
105
|
+
**Prompts:** `ask_data` — a template that walks the model through list → schema → knowledge → query.
|
|
106
|
+
|
|
107
|
+
## Protocol details
|
|
108
|
+
|
|
109
|
+
- Streamable HTTP, `POST /mcp`, plain JSON responses (no SSE); `GET` / `DELETE` → `405`.
|
|
110
|
+
- Dual-era: the 2026-07-28 revision (stateless, per-request `_meta`, `server/discover`, header checks `MCP-Protocol-Version` / `Mcp-Method` / `Mcp-Name`) and the `initialize`-based revisions 2025-11-25, 2025-06-18, 2025-03-26 and 2024-11-05 (`initialize` negotiates the version and mints an `Mcp-Session-Id`; unknown sessions → `404`). `ping`, `notifications/*` (→ `202`) and JSON-RPC batches are accepted.
|
|
111
|
+
- Unsupported versions get error `-32022` with the supported list.
|
|
112
|
+
|
|
113
|
+
## Security notes
|
|
114
|
+
|
|
115
|
+
- **Local only.** The server binds to `127.0.0.1`. Requests whose `Host` is not `localhost` / `127.0.0.1` / `[::1]` are refused with `403` (DNS-rebinding defence), as are requests with a non-local `Origin` header — so web pages in your browser cannot call it.
|
|
116
|
+
- **Tokens.** Every request needs `Authorization: Bearer <token>`; missing or unknown tokens get `401`. Tokens are 192-bit random values; only SHA-256 hashes are stored in `~/.dsh/nl2sql/state.json`, compared in constant time. Revoke a token in the page to cut a client off immediately. "Last used" is shown per token.
|
|
117
|
+
- **Least privilege.** Prefer member- or role-scoped tokens for tools you do not fully trust. Owner tokens can read everything the page can (still read-only and policy-filtered).
|
|
118
|
+
- **Data leaving the machine.** Whatever the MCP client receives is sent to *its* model provider. Use the privacy setting (Admin → Settings → Privacy, or per data source) to send only statistics (`summary`) or nothing (`none`) instead of rows.
|
|
119
|
+
- **Audit.** History lists every MCP query with source `mcp`, the token name and the actor; refusals are logged as `blocked`.
|
package/filters.js
CHANGED
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
* place safely (top-level UNION, no FROM) run unfiltered and report so.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
+
import { lexSql } from './sql-guard.js'
|
|
15
|
+
|
|
14
16
|
export const TIME_RANGES = ['today', 'yesterday', 'last7d', 'last30d', 'last90d', 'thisMonth', 'lastMonth', 'thisYear', 'custom']
|
|
15
17
|
|
|
16
18
|
export function normalizeFilters(input) {
|
|
@@ -68,50 +70,50 @@ export function describeFilter(f) {
|
|
|
68
70
|
|
|
69
71
|
const CLAUSE_END = /^(WHERE|GROUP|HAVING|ORDER|LIMIT|WINDOW|QUALIFY|FETCH|OFFSET|UNION|INTERSECT|EXCEPT|FOR|PIVOT)$/i
|
|
70
72
|
|
|
71
|
-
/**
|
|
72
|
-
|
|
73
|
+
/**
|
|
74
|
+
* Top-level (depth 0) word tokens of the outermost SELECT, with offsets.
|
|
75
|
+
* Tokenised with the dialect-exact lexer from sql-guard.js, so comments,
|
|
76
|
+
* strings and quoted identifiers are recognised exactly as the server does
|
|
77
|
+
* (e.g. MySQL `1--1` is code, not a comment).
|
|
78
|
+
*/
|
|
79
|
+
function topLevelWords(sql, dialect = 'mysql') {
|
|
73
80
|
const words = []
|
|
74
81
|
let depth = 0
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
if (
|
|
79
|
-
|
|
80
|
-
if (c === '\'' || c === '"' || c === '`') {
|
|
81
|
-
const q = c
|
|
82
|
-
const start = i
|
|
83
|
-
i++
|
|
84
|
-
while (i < sql.length) {
|
|
85
|
-
if (sql[i] === '\\' && q === '\'') { i += 2; continue }
|
|
86
|
-
if (sql[i] === q) { if (sql[i + 1] === q) { i += 2; continue } break }
|
|
87
|
-
i++
|
|
88
|
-
}
|
|
89
|
-
i++
|
|
90
|
-
if (depth === 0) words.push({ kind: 'quoted', text: sql.slice(start, i), start, end: i })
|
|
82
|
+
const { segments } = lexSql(sql, { dialect })
|
|
83
|
+
for (const seg of segments) {
|
|
84
|
+
if (seg.kind === 'comment') continue
|
|
85
|
+
if (seg.kind === 'string' || seg.kind === 'ident') {
|
|
86
|
+
if (depth === 0) words.push({ kind: 'quoted', text: seg.text, start: seg.start, end: seg.end })
|
|
91
87
|
continue
|
|
92
88
|
}
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
if (depth === 0) words.push({ kind: '
|
|
99
|
-
|
|
100
|
-
|
|
89
|
+
const text = seg.text
|
|
90
|
+
for (let k = 0; k < text.length;) {
|
|
91
|
+
const c = text[k]
|
|
92
|
+
const i = seg.start + k
|
|
93
|
+
if (c === '(') { depth++; if (depth === 1) words.push({ kind: 'open', start: i, end: i + 1 }); k++; continue }
|
|
94
|
+
if (c === ')') { depth--; if (depth === 0) words.push({ kind: 'close', start: i, end: i + 1 }); k++; continue }
|
|
95
|
+
if (/[A-Za-z_]/.test(c)) {
|
|
96
|
+
let j = k
|
|
97
|
+
while (j < text.length && /[\w$.\-*]/.test(text[j]) && !(text[j] === '-' && text[j + 1] === '-')) j++
|
|
98
|
+
if (depth === 0) words.push({ kind: 'word', text: text.slice(k, j), upper: text.slice(k, j).toUpperCase(), start: i, end: seg.start + j })
|
|
99
|
+
k = j
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
if (c === ',' && depth === 0) words.push({ kind: 'comma', start: i, end: i + 1 })
|
|
103
|
+
k++
|
|
101
104
|
}
|
|
102
|
-
if (c === ',' && depth === 0) words.push({ kind: 'comma', start: i, end: i + 1 })
|
|
103
|
-
i++
|
|
104
105
|
}
|
|
105
106
|
return words
|
|
106
107
|
}
|
|
107
108
|
|
|
108
109
|
function unquote(t) {
|
|
110
|
+
if (t.startsWith('[') && t.endsWith(']')) return t.slice(1, -1)
|
|
109
111
|
return t.replace(/^[`"]|[`"]$/g, '').replace(/``/g, '`').replace(/""/g, '"')
|
|
110
112
|
}
|
|
111
113
|
|
|
112
114
|
/** Parse the outermost query: FROM items with aliases, WHERE span, insertion point. */
|
|
113
|
-
export function parseOuterQuery(sql) {
|
|
114
|
-
const words = topLevelWords(sql)
|
|
115
|
+
export function parseOuterQuery(sql, { dialect = 'mysql' } = {}) {
|
|
116
|
+
const words = topLevelWords(sql, dialect)
|
|
115
117
|
// Skip a leading WITH ... list: the main SELECT is the first top-level SELECT after the CTE bodies.
|
|
116
118
|
let selectIdx = words.findIndex(w => w.kind === 'word' && w.upper === 'SELECT')
|
|
117
119
|
if (words[0]?.upper === 'WITH') {
|
|
@@ -190,7 +192,7 @@ function literalFor(value, dialect) {
|
|
|
190
192
|
*/
|
|
191
193
|
export function applyFilters(sql, filters, { dialect, schema, now }) {
|
|
192
194
|
if (!filters.length) return { sql, applied: [], skipped: [] }
|
|
193
|
-
const outer = parseOuterQuery(sql)
|
|
195
|
+
const outer = parseOuterQuery(sql, { dialect })
|
|
194
196
|
if (!outer || !outer.tables.length) return { sql, applied: [], skipped: filters.map(f => ({ ...f, reason: 'unsupported query shape' })) }
|
|
195
197
|
const tables = schema?.tables ?? []
|
|
196
198
|
const resolve = (ref) => {
|
|
@@ -224,14 +226,66 @@ export function applyFilters(sql, filters, { dialect, schema, now }) {
|
|
|
224
226
|
}
|
|
225
227
|
if (!predicates.length) return { sql, applied, skipped }
|
|
226
228
|
const pred = predicates.join(' AND ')
|
|
229
|
+
// A trailing line comment would swallow whatever follows on the same line: break the line first.
|
|
230
|
+
const nl = (text) => (endsInLineComment(text, dialect) ? '\n' : '')
|
|
227
231
|
let out
|
|
228
232
|
if (outer.where) {
|
|
233
|
+
const head = sql.slice(0, outer.where.keywordStart)
|
|
229
234
|
const existing = sql.slice(outer.where.keywordEnd, outer.where.end).trim()
|
|
230
|
-
out = `${
|
|
235
|
+
out = `${head}${nl(head)}WHERE (${pred}) AND (${existing}${nl(existing)})${outer.where.end < sql.length ? ' ' : ''}${sql.slice(outer.where.end)}`
|
|
231
236
|
} else {
|
|
232
237
|
const before = sql.slice(0, outer.insertAt).replace(/\s+$/, '')
|
|
233
238
|
const after = sql.slice(outer.insertAt)
|
|
234
|
-
out = `${before} WHERE ${pred}${after ? ` ${after}` : ''}`
|
|
239
|
+
out = `${before}${nl(before) || ' '}WHERE ${pred}${after ? ` ${after}` : ''}`
|
|
235
240
|
}
|
|
241
|
+
verifyInjection(out, pred, Boolean(outer.where), outer.tables, dialect)
|
|
236
242
|
return { sql: out, applied, skipped }
|
|
237
243
|
}
|
|
244
|
+
|
|
245
|
+
/** Raised when an injected predicate would not be a top-level conjunct of the rewritten query. */
|
|
246
|
+
export class FilterInjectionError extends Error {
|
|
247
|
+
constructor(message) {
|
|
248
|
+
super(message)
|
|
249
|
+
this.code = 'nl2sql/filter-injection'
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function endsInLineComment(text, dialect) {
|
|
254
|
+
const segs = lexSql(text, { dialect }).segments
|
|
255
|
+
const last = segs[segs.length - 1]
|
|
256
|
+
return Boolean(last && last.kind === 'comment' && last.end === text.length && !last.text.startsWith('/*'))
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
const tokenSig = (w) => (w.kind === 'word' ? `w:${w.upper}` : w.kind === 'quoted' ? `q:${w.text}` : w.kind)
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Defence in depth: re-lex the rewritten SQL with the dialect lexer and make
|
|
263
|
+
* sure the injected predicate is a top-level conjunct of the outer WHERE —
|
|
264
|
+
* balanced parentheses, the same FROM items, and a WHERE body made of exactly
|
|
265
|
+
* `(pred) AND (existing)` or exactly `pred`. Anything else (a stray `)`, a
|
|
266
|
+
* top-level OR next to the predicate, a comment swallowing it) is refused.
|
|
267
|
+
*/
|
|
268
|
+
function verifyInjection(out, pred, hadWhere, tables, dialect) {
|
|
269
|
+
const fail = () => { throw new FilterInjectionError('filter predicate could not be injected as a top-level condition; query refused') }
|
|
270
|
+
const lexed = lexSql(out, { dialect })
|
|
271
|
+
if (lexed.unterminated) fail()
|
|
272
|
+
let depth = 0
|
|
273
|
+
for (const seg of lexed.segments) {
|
|
274
|
+
if (seg.kind !== 'code') continue
|
|
275
|
+
for (const c of seg.text) {
|
|
276
|
+
if (c === '(') depth++
|
|
277
|
+
else if (c === ')' && --depth < 0) fail()
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
if (depth !== 0) fail()
|
|
281
|
+
const outer = parseOuterQuery(out, { dialect })
|
|
282
|
+
if (!outer?.where) fail()
|
|
283
|
+
const sameTables = outer.tables.length === tables.length && outer.tables.every((t, k) => t.name === tables[k].name && t.alias === tables[k].alias)
|
|
284
|
+
if (!sameTables) fail()
|
|
285
|
+
const body = topLevelWords(out, dialect).filter(w => w.start >= outer.where.keywordEnd && w.start < outer.where.end).map(tokenSig)
|
|
286
|
+
const expected = hadWhere ? ['open', 'close', 'w:AND', 'open', 'close'] : topLevelWords(pred, dialect).map(tokenSig)
|
|
287
|
+
if (body.length !== expected.length || body.some((t, k) => t !== expected[k])) fail()
|
|
288
|
+
// The predicate text must sit right after WHERE (inside the first parenthesis when ANDed with an existing condition).
|
|
289
|
+
const at = out.slice(outer.where.keywordEnd).replace(/^\s+/, '')
|
|
290
|
+
if (!at.startsWith(hadWhere ? `(${pred})` : pred)) fail()
|
|
291
|
+
}
|
package/index.js
CHANGED
|
@@ -25,6 +25,7 @@ import { methods as m8 } from './service/alerts.js'
|
|
|
25
25
|
import { methods as m9 } from './service/uploads.js'
|
|
26
26
|
import { methods as m10 } from './service/admin.js'
|
|
27
27
|
import { methods as mSuggest } from './service/suggestions.js'
|
|
28
|
+
import { FEATURES } from './service/features.js'
|
|
28
29
|
const { CHART_TYPES, DEFAULT_SETTINGS, RECORD_KINDS, RPC_PATH, Records, UserError, newId, pick, toUserError } = shared
|
|
29
30
|
|
|
30
31
|
export const name = 'nl2sql'
|
|
@@ -124,6 +125,16 @@ class Nl2SqlService {
|
|
|
124
125
|
return this.writing
|
|
125
126
|
}
|
|
126
127
|
|
|
128
|
+
/**
|
|
129
|
+
* Scheduler jobs contributed by feature modules: every prototype method named
|
|
130
|
+
* `jobs_<feature>(now)` returns an array of { id, due(now), run() }.
|
|
131
|
+
*/
|
|
132
|
+
schedulerJobs(now) {
|
|
133
|
+
const out = []
|
|
134
|
+
for (const k of Object.getOwnPropertyNames(Nl2SqlService.prototype)) if (k.startsWith('jobs_')) { try { out.push(...(this[k](now) ?? [])) } catch (error) { console.warn('nl2sql: %s failed: %s', k, error.message) } }
|
|
135
|
+
return out
|
|
136
|
+
}
|
|
137
|
+
|
|
127
138
|
async dispose() {
|
|
128
139
|
this.disposed = true
|
|
129
140
|
if (this.alertTimer) clearInterval(this.alertTimer)
|
|
@@ -145,7 +156,7 @@ class Nl2SqlService {
|
|
|
145
156
|
}
|
|
146
157
|
|
|
147
158
|
}
|
|
148
|
-
for (const m of [m0, m1, m2, m3, m4, m5, mChannels, m6, m7, m8, m9, m10, mSuggest]) Object.defineProperties(Nl2SqlService.prototype, Object.getOwnPropertyDescriptors(m))
|
|
159
|
+
for (const m of [m0, m1, m2, m3, m4, m5, mChannels, m6, m7, m8, m9, m10, mSuggest, ...FEATURES.map(f => f.methods)]) Object.defineProperties(Nl2SqlService.prototype, Object.getOwnPropertyDescriptors(m))
|
|
149
160
|
const DATASOURCE_PARAM = { type: 'string', description: 'Data source id or name. Omit to use the one bound to this session.' }
|
|
150
161
|
|
|
151
162
|
function textBlock(text) {
|
|
@@ -188,7 +199,8 @@ function registerTools(ctx, service) {
|
|
|
188
199
|
if (!args.datasource) throw new Error('`datasource` is required for action=use')
|
|
189
200
|
const ds = service.requireDatasource(args.datasource)
|
|
190
201
|
service.assertAccess(actor, ds)
|
|
191
|
-
|
|
202
|
+
// The agent tool binds only this session; the page's current data source is left alone.
|
|
203
|
+
const info = await service.bindSession(sessionId, ds.id, { setCurrent: false })
|
|
192
204
|
return { bound: info.datasourceId, datasources: visible.map(d => pick(d, ['id', 'name', 'type', 'database', 'serverVersion'])) }
|
|
193
205
|
}
|
|
194
206
|
const info = sessionId ? service.sessionInfo(sessionId) : null
|
|
@@ -353,7 +365,8 @@ function registerOverviewTool(ctx, service) {
|
|
|
353
365
|
const ds = service.datasourceForSession(exec.agent?.id, args.datasource)
|
|
354
366
|
service.assertCanWriteSql(service.actorFor(exec.agent?.id), '生成自动概览')
|
|
355
367
|
if (args.pin && service.actorFor(exec.agent?.id)) throw new Error('只有本机用户可以把图表固定到看板')
|
|
356
|
-
const
|
|
368
|
+
const sessionId = exec.agent?.id
|
|
369
|
+
const { table, results } = await service.tableOverview({ datasourceId: ds.id, table: args.table, pin: Boolean(args.pin), sessionId, actor: service.actorFor(sessionId), source: 'agent' })
|
|
357
370
|
return {
|
|
358
371
|
model: { table, pinned: Boolean(args.pin), charts: results.map(r => (r.error ? { title: r.title, error: r.error } : { title: r.title, ...service.modelView(r) })) },
|
|
359
372
|
display: { overview: true, table, results: results.map(r => (r.error ? { title: r.title, sql: r.sql, error: r.error } : displayMeta(r))) },
|
|
@@ -426,6 +439,8 @@ function displayMeta(result) {
|
|
|
426
439
|
|
|
427
440
|
function registerRpc(ctx, service) {
|
|
428
441
|
const methods = {
|
|
442
|
+
// Feature modules (service/features.js) contribute RPCs as rpc(service) → { name: fn }.
|
|
443
|
+
...Object.assign({}, ...FEATURES.map(f => f.rpc?.(service) ?? {})),
|
|
429
444
|
listDatasources: () => service.listDatasources(),
|
|
430
445
|
saveDatasource: (p) => service.saveDatasource(p),
|
|
431
446
|
deleteDatasource: (p) => service.deleteDatasource(p.id),
|
|
@@ -589,6 +604,9 @@ export async function apply(ctx, config) {
|
|
|
589
604
|
service.startBots().catch(error => console.warn('nl2sql bot: start failed: %s', error.message))
|
|
590
605
|
service.scheduleAlerts()
|
|
591
606
|
registerAnalysisTools(ctx, service)
|
|
607
|
+
// Feature modules register their own agent tools: tools(ctx, service, helpers).
|
|
608
|
+
for (const f of FEATURES) f.tools?.(ctx, service, { displayMeta, textBlock, DATASOURCE_PARAM })
|
|
609
|
+
for (const f of FEATURES) f.start?.(ctx, service)
|
|
592
610
|
}
|
|
593
611
|
|
|
594
612
|
function registerAnalysisTools(ctx, service) {
|