@hedeqiang/dsh-nl2sql 0.0.0-stage → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/LICENSE +21 -0
- package/README.en.md +62 -0
- package/README.md +88 -2
- package/analysis.js +888 -0
- package/cordis.patch.yml +68 -0
- package/drivers.js +694 -0
- package/filters.js +237 -0
- package/icon.svg +13 -0
- package/im.js +511 -0
- package/index.js +650 -0
- package/lark-bot.js +606 -0
- package/lib/client.echarts.js +57 -0
- package/lib/client.js +4845 -0
- package/locale/en.json +1 -0
- package/locale/zh.json +1 -0
- package/package.json +90 -4
- package/policy.js +362 -0
- package/service/access.js +145 -0
- package/service/admin.js +539 -0
- package/service/alerts.js +233 -0
- package/service/dashboards.js +359 -0
- package/service/datasources.js +392 -0
- package/service/evaluation.js +121 -0
- package/service/lark-actions.js +223 -0
- package/service/lark.js +250 -0
- package/service/push.js +314 -0
- package/service/query.js +411 -0
- package/service/schema.js +507 -0
- package/service/shared.js +424 -0
- package/service/suggestions.js +772 -0
- package/service/uploads.js +258 -0
- package/sql-guard.js +222 -0
- package/sqlite-driver.js +677 -0
- package/store.js +225 -0
- package/xlsx.js +1001 -0
package/locale/en.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"meta": {"title": "Data Q&A (NL2SQL)", "description": "Open Data Q&A in the sidebar to connect MySQL 5.7+, PostgreSQL or Elasticsearch (optionally over SSH), or upload CSV / Excel. After schema analysis, ask in plain language: the agent writes read-only SQL, runs it and charts the result in the conversation. Includes dashboards with global filters, drill-down and change attribution, a metric library and business knowledge, accuracy evaluation, alerts and scheduled reports, Lark / Feishu push and an @-mention bot, member scopes (data sources, viewer role, row-level rules) and audit. A setting controls what result data reaches the model (column statistics by default). Passwords live in the Harness credential store; data and logs stay in ~/.dsh/nl2sql. Turning the plugin off stops its page, tools and scheduled checks; saved data is kept."}}
|
package/locale/zh.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"meta": {"title": "数据问答(NL2SQL)", "description": "在左侧栏「数据问答」中连接 MySQL 5.7+ / PostgreSQL / Elasticsearch(可经 SSH 隧道)或上传 CSV / Excel,分析表结构后用自然语言提问:Agent 写只读 SQL、执行并在对话中画图。支持看板与全局筛选、下钻与变化归因、指标库与业务知识、准确率评测、告警与定时报告、飞书推送与群内 @机器人问答、成员权限(数据源范围、只读角色、行级权限)和审计。查询结果按设置决定发给模型的内容(默认只发统计摘要);数据源密码保存在 Harness 凭据库;数据和日志保存在本机 ~/.dsh/nl2sql。关闭插件后页面、工具和定时检查停止,已保存的数据保留。"}}
|
package/package.json
CHANGED
|
@@ -1,6 +1,92 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hedeqiang/dsh-nl2sql",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.10.0",
|
|
4
|
+
"description": "Data Q&A (NL2SQL) for DeepSeek Harness: ask MySQL / PostgreSQL / Elasticsearch / CSV in plain language, charts, dashboards, alerts, Lark push and bot, access control.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": "./index.js",
|
|
8
|
+
"./client": "./lib/client.js",
|
|
9
|
+
"./locale/*.json": "./locale/*.json",
|
|
10
|
+
"./icon": "./icon.svg"
|
|
11
|
+
},
|
|
12
|
+
"files": [
|
|
13
|
+
"index.js",
|
|
14
|
+
"service/*.js",
|
|
15
|
+
"drivers.js",
|
|
16
|
+
"policy.js",
|
|
17
|
+
"filters.js",
|
|
18
|
+
"im.js",
|
|
19
|
+
"store.js",
|
|
20
|
+
"xlsx.js",
|
|
21
|
+
"analysis.js",
|
|
22
|
+
"lark-bot.js",
|
|
23
|
+
"sqlite-driver.js",
|
|
24
|
+
"sql-guard.js",
|
|
25
|
+
"lib/client.js",
|
|
26
|
+
"lib/client.echarts.js",
|
|
27
|
+
"cordis.patch.yml",
|
|
28
|
+
"locale/*.json",
|
|
29
|
+
"icon.svg",
|
|
30
|
+
"README.md",
|
|
31
|
+
"README.en.md",
|
|
32
|
+
"LICENSE",
|
|
33
|
+
"CHANGELOG.md"
|
|
34
|
+
],
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"@larksuiteoapi/node-sdk": "1.76.0",
|
|
37
|
+
"@resvg/resvg-js": "2.6.2",
|
|
38
|
+
"echarts": "6.1.0",
|
|
39
|
+
"mysql2": "3.24.6",
|
|
40
|
+
"pg": "8.23.1",
|
|
41
|
+
"ssh2": "1.17.0"
|
|
42
|
+
},
|
|
43
|
+
"scripts": {
|
|
44
|
+
"build": "node scripts/build-echarts.mjs",
|
|
45
|
+
"test": "node test/run.mjs",
|
|
46
|
+
"build:client": "node scripts/build-client.mjs",
|
|
47
|
+
"test:fast": "node test/run.mjs --fast",
|
|
48
|
+
"check": "node scripts/build-client.mjs --check && node test/run.mjs --fast"
|
|
49
|
+
},
|
|
50
|
+
"dsh": {
|
|
51
|
+
"bundle": {
|
|
52
|
+
"patch": "./cordis.patch.yml"
|
|
53
|
+
},
|
|
54
|
+
"client": {
|
|
55
|
+
"platform": "web",
|
|
56
|
+
"immediately": true,
|
|
57
|
+
"inject": [
|
|
58
|
+
"@deepseek-ai/dsh-client-connection",
|
|
59
|
+
"@deepseek-ai/dsh-client-locale",
|
|
60
|
+
"@deepseek-ai/dsh-client-ui-layout",
|
|
61
|
+
"@deepseek-ai/dsh-client-ui-sidebar",
|
|
62
|
+
"@deepseek-ai/dsh-client-ui-conversation",
|
|
63
|
+
"@deepseek-ai/dsh-client-ui-workspace"
|
|
64
|
+
]
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"license": "MIT",
|
|
68
|
+
"author": "hedeqiang <laravel_code@163.com>",
|
|
69
|
+
"keywords": [
|
|
70
|
+
"deepseek-harness",
|
|
71
|
+
"dsh",
|
|
72
|
+
"dsh-bundle",
|
|
73
|
+
"nl2sql",
|
|
74
|
+
"text-to-sql",
|
|
75
|
+
"bi",
|
|
76
|
+
"mysql",
|
|
77
|
+
"postgresql",
|
|
78
|
+
"elasticsearch",
|
|
79
|
+
"lark",
|
|
80
|
+
"feishu"
|
|
81
|
+
],
|
|
82
|
+
"engines": {
|
|
83
|
+
"node": ">=24"
|
|
84
|
+
},
|
|
85
|
+
"repository": {
|
|
86
|
+
"type": "git",
|
|
87
|
+
"url": "git+https://github.com/hedeqiang/dsh-nl2sql.git"
|
|
88
|
+
},
|
|
89
|
+
"publishConfig": {
|
|
90
|
+
"access": "public"
|
|
91
|
+
}
|
|
92
|
+
}
|
package/policy.js
ADDED
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
import { lexSql } from './sql-guard.js'
|
|
2
|
+
/**
|
|
3
|
+
* Table/column access policy and SQL identifier analysis.
|
|
4
|
+
*
|
|
5
|
+
* Policy shape (per data source):
|
|
6
|
+
* { mode: 'all' | 'allow' | 'deny', tables: string[], hiddenColumns: string[], maskedColumns: string[] }
|
|
7
|
+
* Patterns are case-insensitive globs: `orders`, `ods_*`, `sales.*`; column
|
|
8
|
+
* patterns are `table.column` (`*.salary`, `hr_*.id_card`).
|
|
9
|
+
*
|
|
10
|
+
* Enforcement is syntactic and conservative: a statement is refused when it
|
|
11
|
+
* names a table the policy excludes, a system catalog, or a hidden column of a
|
|
12
|
+
* referenced table; hidden columns that arrive anyway (SELECT *) are dropped
|
|
13
|
+
* from the result. Database grants remain the strict boundary.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export const EMPTY_POLICY = Object.freeze({ mode: 'all', tables: [], hiddenColumns: [], maskedColumns: [] })
|
|
17
|
+
|
|
18
|
+
export function normalizePolicy(input) {
|
|
19
|
+
if (!input || typeof input !== 'object') return { ...EMPTY_POLICY }
|
|
20
|
+
const list = (v) => (Array.isArray(v) ? v : String(v ?? '').split(/[\n,,]/))
|
|
21
|
+
.map(x => String(x).trim()).filter(Boolean).slice(0, 2000).map(x => x.slice(0, 200))
|
|
22
|
+
const mode = ['all', 'allow', 'deny'].includes(input.mode) ? input.mode : 'all'
|
|
23
|
+
return { mode, tables: [...new Set(list(input.tables))], hiddenColumns: [...new Set(list(input.hiddenColumns))], maskedColumns: [...new Set(list(input.maskedColumns))] }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function policyActive(policy) {
|
|
27
|
+
return Boolean(policy) && (policy.mode !== 'all' || policy.hiddenColumns.length > 0 || policy.maskedColumns.length > 0)
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const globCache = new Map()
|
|
31
|
+
export function glob(pattern) {
|
|
32
|
+
const key = pattern.toLowerCase()
|
|
33
|
+
let re = globCache.get(key)
|
|
34
|
+
if (!re) {
|
|
35
|
+
re = new RegExp(`^${key.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.')}$`)
|
|
36
|
+
globCache.set(key, re)
|
|
37
|
+
}
|
|
38
|
+
return re
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function matchAny(patterns, name) {
|
|
42
|
+
const n = name.toLowerCase()
|
|
43
|
+
const last = n.split('.').pop()
|
|
44
|
+
return patterns.some(p => {
|
|
45
|
+
const re = glob(p)
|
|
46
|
+
return re.test(n) || (!p.includes('.') && re.test(last))
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Whether the policy lets the model see / query this table. */
|
|
51
|
+
export function tableAllowed(policy, table) {
|
|
52
|
+
if (!policy || policy.mode === 'all') return true
|
|
53
|
+
const hit = matchAny(policy.tables, table)
|
|
54
|
+
return policy.mode === 'allow' ? hit : !hit
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function columnMatches(patterns, table, column) {
|
|
58
|
+
const t = table.toLowerCase()
|
|
59
|
+
const tl = t.split('.').pop()
|
|
60
|
+
const c = column.toLowerCase()
|
|
61
|
+
return patterns.some(p => {
|
|
62
|
+
const i = p.lastIndexOf('.')
|
|
63
|
+
if (i < 0) return glob(p).test(c)
|
|
64
|
+
const tp = glob(p.slice(0, i))
|
|
65
|
+
return (tp.test(t) || tp.test(tl)) && glob(p.slice(i + 1)).test(c)
|
|
66
|
+
})
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function columnHidden(policy, table, column) {
|
|
70
|
+
return Boolean(policy) && columnMatches(policy.hiddenColumns, table, column)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function columnMasked(policy, table, column) {
|
|
74
|
+
return Boolean(policy) && columnMatches(policy.maskedColumns, table, column)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Schema as the model may see it: excluded tables and hidden columns removed. */
|
|
78
|
+
export function filterSchema(schema, policy) {
|
|
79
|
+
if (!policyActive(policy)) return schema
|
|
80
|
+
const tables = schema.tables
|
|
81
|
+
.filter(t => tableAllowed(policy, t.name))
|
|
82
|
+
.map(t => ({
|
|
83
|
+
...t,
|
|
84
|
+
columns: t.columns.filter(c => !columnHidden(policy, t.name, c.name)).map(c => (columnMasked(policy, t.name, c.name) ? { ...c, values: undefined, roles: [...new Set([...c.roles, 'sensitive'])] } : c)),
|
|
85
|
+
indexes: t.indexes.filter(i => i.columns.every(c => !columnHidden(policy, t.name, c))),
|
|
86
|
+
}))
|
|
87
|
+
const visible = new Set(tables.map(t => t.name))
|
|
88
|
+
const hiddenCol = (tbl, col) => columnHidden(policy, tbl, col)
|
|
89
|
+
const relations = schema.relations.filter(r => visible.has(r.from.table) && visible.has(r.to.table) && !hiddenCol(r.from.table, r.from.column) && !hiddenCol(r.to.table, r.to.column))
|
|
90
|
+
return { ...schema, tables, relations }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Annotate (not filter) a schema for the page: blocked tables and hidden/masked columns. */
|
|
94
|
+
export function annotateSchema(schema, policy) {
|
|
95
|
+
if (!policyActive(policy)) return schema
|
|
96
|
+
return {
|
|
97
|
+
...schema,
|
|
98
|
+
tables: schema.tables.map(t => ({
|
|
99
|
+
...t,
|
|
100
|
+
blocked: !tableAllowed(policy, t.name),
|
|
101
|
+
columns: t.columns.map(c => ({ ...c, hidden: columnHidden(policy, t.name, c.name), policyMasked: columnMasked(policy, t.name, c.name) })),
|
|
102
|
+
})),
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ------------------------------------------------------------ SQL analysis
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Identifiers of a statement, lower-cased, with dotted parts joined
|
|
110
|
+
* (`db`.`t` → db.t), string literals and comments skipped. Also returns the
|
|
111
|
+
* raw tokens that follow FROM/JOIN, which may contain `-` and `*` in
|
|
112
|
+
* Elasticsearch index names.
|
|
113
|
+
*/
|
|
114
|
+
export function analyzeIdentifiers(sql, { dialect = 'mysql' } = {}) {
|
|
115
|
+
const ids = []
|
|
116
|
+
const fromTargets = []
|
|
117
|
+
const src = normalizeForScan(String(sql), dialect)
|
|
118
|
+
let i = 0
|
|
119
|
+
let prevWord = ''
|
|
120
|
+
let inFromList = false // after FROM target: `t alias, next` keeps listing tables
|
|
121
|
+
let depth = 0
|
|
122
|
+
let fromDepth = -1
|
|
123
|
+
const pushId = (parts) => { if (parts.length) ids.push(parts.join('.').toLowerCase()) }
|
|
124
|
+
while (i < src.length) {
|
|
125
|
+
const c = src[i]
|
|
126
|
+
const n = src[i + 1]
|
|
127
|
+
void n
|
|
128
|
+
// Input is normalized by normalizeForScan: comments are gone and every string literal is exactly ''.
|
|
129
|
+
if (c === '\'') { i += 2; prevWord = ''; continue }
|
|
130
|
+
if (c === '`' || c === '"' || /[A-Za-z_]/.test(c)) {
|
|
131
|
+
// A possibly dotted identifier chain.
|
|
132
|
+
const parts = []
|
|
133
|
+
const start = i
|
|
134
|
+
for (;;) {
|
|
135
|
+
if (src[i] === '`' || src[i] === '"') {
|
|
136
|
+
const q = src[i]
|
|
137
|
+
let j = i + 1
|
|
138
|
+
let text = ''
|
|
139
|
+
while (j < src.length) {
|
|
140
|
+
if (src[j] === q) { if (src[j + 1] === q) { text += q; j += 2; continue } break }
|
|
141
|
+
text += src[j]
|
|
142
|
+
j++
|
|
143
|
+
}
|
|
144
|
+
parts.push(text)
|
|
145
|
+
i = j + 1
|
|
146
|
+
} else if (/[A-Za-z_]/.test(src[i] ?? '')) {
|
|
147
|
+
let j = i
|
|
148
|
+
while (j < src.length && /[\w$]/.test(src[j])) j++
|
|
149
|
+
parts.push(src.slice(i, j))
|
|
150
|
+
i = j
|
|
151
|
+
} else if (src[i] === '*' && parts.length) {
|
|
152
|
+
parts.push('*')
|
|
153
|
+
i++
|
|
154
|
+
break
|
|
155
|
+
} else {
|
|
156
|
+
break
|
|
157
|
+
}
|
|
158
|
+
if (src[i] === '.' && /[`"A-Za-z_*]/.test(src[i + 1] ?? '')) { i++; continue }
|
|
159
|
+
break
|
|
160
|
+
}
|
|
161
|
+
const word = parts.length === 1 ? parts[0].toUpperCase() : ''
|
|
162
|
+
if (/^(WHERE|GROUP|ORDER|HAVING|LIMIT|ON|USING|UNION|SELECT|WINDOW|FETCH|OFFSET|SET|RETURNING)$/.test(word) && depth === fromDepth) inFromList = false
|
|
163
|
+
if (/^(FROM|JOIN)$/.test(prevWord)) {
|
|
164
|
+
inFromList = true
|
|
165
|
+
fromDepth = depth
|
|
166
|
+
// Raw target, including hyphens/wildcards that bare identifiers stop at.
|
|
167
|
+
let j = i
|
|
168
|
+
while (j < src.length && /[\w\-*.:]/.test(src[j])) j++
|
|
169
|
+
const raw = (parts.length === 1 && !/[`"]/.test(src[start]) ? src.slice(start, j) : parts.join('.')).toLowerCase()
|
|
170
|
+
fromTargets.push(raw)
|
|
171
|
+
if (j > i && !/[`"]/.test(src[start])) i = j
|
|
172
|
+
}
|
|
173
|
+
pushId(parts)
|
|
174
|
+
prevWord = word
|
|
175
|
+
continue
|
|
176
|
+
}
|
|
177
|
+
if (c === '(') depth++
|
|
178
|
+
if (c === ')') { depth--; if (depth < fromDepth) inFromList = false }
|
|
179
|
+
if (c === ',' && inFromList && depth === fromDepth) { prevWord = 'FROM'; i++; continue }
|
|
180
|
+
if (!/\s/.test(c)) prevWord = ''
|
|
181
|
+
i++
|
|
182
|
+
}
|
|
183
|
+
return { ids, fromTargets }
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/**
|
|
187
|
+
* Rewrite SQL for identifier scanning using the dialect's exact lexical rules:
|
|
188
|
+
* comments become a space, string literals become '' (so nothing inside them is
|
|
189
|
+
* read as code), quoted identifiers are kept as "name", and whitespace around
|
|
190
|
+
* dots in name chains is removed (`information_schema . tables`).
|
|
191
|
+
*/
|
|
192
|
+
export function normalizeForScan(sql, dialect = 'mysql') {
|
|
193
|
+
const { segments } = lexSql(sql, { dialect })
|
|
194
|
+
const out = segments.map(sg => {
|
|
195
|
+
if (sg.kind === 'comment') return ' '
|
|
196
|
+
if (sg.kind === 'string') return "''"
|
|
197
|
+
if (sg.kind === 'ident') {
|
|
198
|
+
const t = sg.text
|
|
199
|
+
const inner = t[0] === '[' ? t.slice(1, -1) : t.slice(1, -1).split(t[0] + t[0]).join(t[0])
|
|
200
|
+
return `"${inner.replace(/"/g, '""')}"`
|
|
201
|
+
}
|
|
202
|
+
return sg.text
|
|
203
|
+
}).join('')
|
|
204
|
+
return out.replace(/\s*\.\s*(?=["A-Za-z_*])/g, '.').replace(/(["\w])\s+\./g, '$1.')
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const SYSTEM_PREFIX = /^(information_schema|mysql|performance_schema|sys|pg_catalog|pg_toast)\./
|
|
208
|
+
const SYSTEM_BARE = /^pg_(class|namespace|attribute|tables|views|user|shadow|roles|authid|stat\w*|settings|proc|database|index|indexes|constraint|description|type|matviews|policies|sequence\w*)$/
|
|
209
|
+
|
|
210
|
+
/** Tables of the schema that a statement references. */
|
|
211
|
+
export function referencedTables(sql, tables, { dialect = 'mysql' } = {}) {
|
|
212
|
+
const { ids, fromTargets } = analyzeIdentifiers(sql, { dialect })
|
|
213
|
+
const names = tables.map(t => t.name)
|
|
214
|
+
const lower = new Map(names.map(n => [n.toLowerCase(), n]))
|
|
215
|
+
const byLast = new Map()
|
|
216
|
+
for (const n of names) {
|
|
217
|
+
const last = n.toLowerCase().split('.').pop()
|
|
218
|
+
if (!byLast.has(last)) byLast.set(last, [])
|
|
219
|
+
byLast.get(last).push(n)
|
|
220
|
+
}
|
|
221
|
+
const out = new Set()
|
|
222
|
+
for (const id of [...ids, ...fromTargets]) {
|
|
223
|
+
if (lower.has(id)) { out.add(lower.get(id)); continue }
|
|
224
|
+
// db.table / schema.table / alias.column: try the last two and last parts.
|
|
225
|
+
const parts = id.split('.')
|
|
226
|
+
if (parts.length >= 2) {
|
|
227
|
+
const two = parts.slice(-2).join('.')
|
|
228
|
+
if (lower.has(two)) { out.add(lower.get(two)); continue }
|
|
229
|
+
}
|
|
230
|
+
// A bare name, or db.table where the last part names a table (MySQL database prefix).
|
|
231
|
+
const last = parts[parts.length - 1]
|
|
232
|
+
if ((parts.length === 1 || fromTargets.includes(id)) && byLast.has(last)) for (const n of byLast.get(last)) out.add(n)
|
|
233
|
+
// Elasticsearch wildcard targets: every matching index.
|
|
234
|
+
if (id.includes('*') && fromTargets.includes(id)) for (const n of names) if (glob(id).test(n.toLowerCase())) out.add(n)
|
|
235
|
+
}
|
|
236
|
+
return [...out]
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Refuse a statement the policy does not admit.
|
|
241
|
+
* @returns undefined when admitted, otherwise a refusal message.
|
|
242
|
+
*/
|
|
243
|
+
export function checkAccess(sql, policy, schema, head, { dialect = 'mysql' } = {}) {
|
|
244
|
+
if (!policyActive(policy)) return undefined
|
|
245
|
+
const { ids, fromTargets } = analyzeIdentifiers(sql, { dialect })
|
|
246
|
+
const scan = normalizeForScan(String(sql), dialect)
|
|
247
|
+
if (policy.mode !== 'all' && head === 'SHOW') return '当前数据源启用了表访问控制,不允许执行 SHOW 语句;请使用 nl2sql_schema 查看可访问的表。'
|
|
248
|
+
for (const id of [...ids, ...fromTargets]) {
|
|
249
|
+
if (SYSTEM_PREFIX.test(id) || SYSTEM_BARE.test(id) || (fromTargets.includes(id) && id.startsWith('.'))) return `当前数据源启用了访问控制,不允许查询系统目录(${id})。`
|
|
250
|
+
}
|
|
251
|
+
const tables = schema?.tables ?? []
|
|
252
|
+
const refs = referencedTables(sql, tables, { dialect })
|
|
253
|
+
const blocked = refs.filter(t => !tableAllowed(policy, t))
|
|
254
|
+
if (blocked.length) return `无权查询表:${blocked.join('、')}(数据源访问控制)。`
|
|
255
|
+
if (policy.mode === 'deny') {
|
|
256
|
+
// Deny patterns also cover FROM/JOIN targets not (yet) in the analyzed schema.
|
|
257
|
+
const raw = fromTargets.filter(t => t && !tableAllowed(policy, t.replace(/^"|"$/g, '')))
|
|
258
|
+
if (raw.length) return `无权查询表:${raw.join('、')}(数据源访问控制)。`
|
|
259
|
+
}
|
|
260
|
+
if (policy.mode === 'allow') {
|
|
261
|
+
// FROM/JOIN targets that resolve to no analyzed table (new tables, unanalyzed views) are refused too.
|
|
262
|
+
const resolves = (target) => referencedTables(`SELECT 1 FROM ${/[^\w.]/.test(target) ? `"${target}"` : target}`, tables, { dialect: 'postgres' }).length > 0
|
|
263
|
+
const ctes = new Set([...scan.matchAll(/(?:\bWITH(?:\s+RECURSIVE)?|,)\s*[`"]?([A-Za-z_][\w$]*)[`"]?\s*(?:\([^)]*\))?\s+AS\s*(?:NOT\s+)?(?:MATERIALIZED\s*)?\(/gi)].map(m => m[1].toLowerCase()))
|
|
264
|
+
const unknown = fromTargets.filter(t => t && !ctes.has(t) && !/^(select|lateral|unnest|generate_series|json_table|dual)$/.test(t) && !resolves(t))
|
|
265
|
+
if (unknown.length) return `无权查询未授权或未分析的表:${unknown.join('、')}。如果是新表,请先在页面上重新分析表结构并加入允许列表。`
|
|
266
|
+
}
|
|
267
|
+
const hidden = []
|
|
268
|
+
const masked = []
|
|
269
|
+
for (const t of refs) {
|
|
270
|
+
const table = tables.find(x => x.name === t)
|
|
271
|
+
for (const c of table?.columns ?? []) {
|
|
272
|
+
if (columnHidden(policy, t, c.name)) hidden.push({ table: t, column: c.name.toLowerCase() })
|
|
273
|
+
else if (columnMasked(policy, t, c.name)) masked.push({ table: t, column: c.name.toLowerCase() })
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
if (hidden.length || masked.length) {
|
|
277
|
+
const hiddenNames = new Set(hidden.map(h => h.column))
|
|
278
|
+
const hit = ids.find(id => hiddenNames.has(id.split('.').pop().replace(/^"|"$/g, '')))
|
|
279
|
+
if (hit) return `字段 ${hit} 已被数据源访问控制隐藏,不能在查询中使用。`
|
|
280
|
+
if (/\b(row_to_json|to_jsonb?|jsonb?_agg|json_object_agg|hstore|record_out|row)\s*\(/i.test(scan)) return '引用的表含有受保护的字段,不允许使用整行转换函数(row_to_json/to_json/json_agg/ROW(...) 等)。'
|
|
281
|
+
// Whole-row values: alias::type, CAST(alias AS …), (alias), alias in function args, alias.* inside a call.
|
|
282
|
+
const aliasNames = tableAliases(scan, refs)
|
|
283
|
+
for (const a of aliasNames) {
|
|
284
|
+
const q = a.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
285
|
+
const asValue = new RegExp(`(?:^|[^\\w."])${q}\\s*::|\\bCAST\\s*\\(\\s*${q}\\s+AS\\b|\\(\\s*${q}\\s*[,)]|,\\s*${q}\\s*\\)|\\(\\s*${q}\\.\\*`, 'i')
|
|
286
|
+
if (asValue.test(scan)) return '引用的表含有受保护的字段,不允许把整行(表或别名本身)作为值使用;请明确列出需要的字段。'
|
|
287
|
+
}
|
|
288
|
+
// Masked columns may only be selected as plain columns (so results can be masked), not inside expressions.
|
|
289
|
+
if (masked.length) {
|
|
290
|
+
const maskedNames = new Set(masked.map(m => m.column))
|
|
291
|
+
const exprUse = maskedInExpression(scan, maskedNames)
|
|
292
|
+
if (exprUse) return `字段 ${exprUse} 需要脱敏显示,只能直接选取,不能用于表达式、函数或改名(AS)。`
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return undefined
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** Table names and aliases introduced in FROM/JOIN for the referenced tables. */
|
|
299
|
+
function tableAliases(scan, refs) {
|
|
300
|
+
const out = new Set()
|
|
301
|
+
const lastParts = new Set(refs.map(r => r.toLowerCase().split('.').pop()))
|
|
302
|
+
for (const m of scan.matchAll(/\b(?:FROM|JOIN|,)\s+((?:"[^"]+"|[\w$]+)(?:\.(?:"[^"]+"|[\w$]+))*)(?:\s+(?:AS\s+)?("?[A-Za-z_][\w$]*"?))?/gi)) {
|
|
303
|
+
const name = m[1].split('.').pop().replace(/^"|"$/g, '').toLowerCase()
|
|
304
|
+
if (!lastParts.has(name)) continue
|
|
305
|
+
out.add(m[1].split('.').pop().replace(/^"|"$/g, ''))
|
|
306
|
+
const alias = m[2]?.replace(/^"|"$/g, '')
|
|
307
|
+
if (alias && !/^(WHERE|JOIN|ON|LEFT|RIGHT|INNER|OUTER|FULL|CROSS|GROUP|ORDER|LIMIT|UNION|USING|NATURAL|HAVING|WINDOW)$/i.test(alias)) out.add(alias)
|
|
308
|
+
}
|
|
309
|
+
return [...out]
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Whether a masked column appears anywhere other than as a plain select item
|
|
314
|
+
* (`col`, `t.col`) — e.g. inside a function, an operator, or renamed with AS.
|
|
315
|
+
*/
|
|
316
|
+
function maskedInExpression(scan, maskedNames) {
|
|
317
|
+
const m = /^\s*\(*\s*SELECT\s+(DISTINCT\s+)?([\s\S]*?)\s+FROM\s/i.exec(scan)
|
|
318
|
+
const items = m ? splitTop(m[2]) : []
|
|
319
|
+
for (const item of items) {
|
|
320
|
+
const text = item.trim()
|
|
321
|
+
const plain = /^(?:"?[\w$]+"?\.)*"?([\w$]+)"?$/.exec(text)
|
|
322
|
+
for (const name of maskedNames) {
|
|
323
|
+
const re = new RegExp(`(^|[^\\w$])"?${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}"?($|[^\\w$])`, 'i')
|
|
324
|
+
if (!re.test(text)) continue
|
|
325
|
+
if (plain && plain[1].toLowerCase() === name) continue
|
|
326
|
+
return name
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
// Outside the select list (WHERE/GROUP BY/ORDER BY) masked columns are fine: they do not reach results.
|
|
330
|
+
return undefined
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function splitTop(text) {
|
|
334
|
+
const out = []
|
|
335
|
+
let depth = 0
|
|
336
|
+
let cur = ''
|
|
337
|
+
for (const ch of text) {
|
|
338
|
+
if (ch === '(') depth++
|
|
339
|
+
if (ch === ')') depth--
|
|
340
|
+
if (ch === ',' && depth === 0) { out.push(cur); cur = ''; continue }
|
|
341
|
+
cur += ch
|
|
342
|
+
}
|
|
343
|
+
if (cur.trim()) out.push(cur)
|
|
344
|
+
return out
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** Result column indexes to drop (hidden) and to mask, given the referenced tables. */
|
|
348
|
+
export function resultColumnPolicy(policy, columns, refs, schema) {
|
|
349
|
+
const drop = new Set()
|
|
350
|
+
const mask = new Set()
|
|
351
|
+
if (!policyActive(policy)) return { drop, mask }
|
|
352
|
+
const tables = (schema?.tables ?? []).filter(t => refs.includes(t.name))
|
|
353
|
+
columns.forEach((col, i) => {
|
|
354
|
+
const name = String(col.name).toLowerCase()
|
|
355
|
+
const origin = col.table && col.column ? [{ table: col.table, column: col.column }] : tables.map(t => ({ table: t.name, column: name.split('.').pop() }))
|
|
356
|
+
for (const o of origin) {
|
|
357
|
+
if (columnHidden(policy, o.table, o.column)) drop.add(i)
|
|
358
|
+
else if (columnMasked(policy, o.table, o.column)) mask.add(i)
|
|
359
|
+
}
|
|
360
|
+
})
|
|
361
|
+
return { drop, mask }
|
|
362
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Nl2SqlService methods: access.
|
|
3
|
+
* Mixed into the service prototype by index.js; `this` is the service.
|
|
4
|
+
*/
|
|
5
|
+
import { ROLE_RANK, UserError, newId, oneOf, requireText, upsert } from './shared.js'
|
|
6
|
+
|
|
7
|
+
export const methods = {
|
|
8
|
+
// ------------------------------------------------------------ access
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Actors (multi-user): the local page/agent owner has full access; Lark
|
|
12
|
+
* users/chats are mapped to users with datasource scopes.
|
|
13
|
+
*/
|
|
14
|
+
assertAccess(actor, ds) {
|
|
15
|
+
if (!actor || actor.kind === 'owner') return
|
|
16
|
+
const allowed = actor.datasourceIds
|
|
17
|
+
if (Array.isArray(allowed) && !allowed.includes(ds.id)) throw new UserError('nl2sql/access-denied', `${actor.name ?? '当前用户'} 无权访问数据源「${ds.name}」`)
|
|
18
|
+
},
|
|
19
|
+
|
|
20
|
+
/** The actor a session runs as (Lark user, evaluation), or undefined for the local owner. */
|
|
21
|
+
actorFor(sessionId) {
|
|
22
|
+
return sessionId ? this.state.sessionActors?.[sessionId] : undefined
|
|
23
|
+
},
|
|
24
|
+
|
|
25
|
+
/** Viewer: only governed answers (metric library, verified examples, dashboards) — no free SQL. */
|
|
26
|
+
assertCanWriteSql(actor, what = '自由编写 SQL') {
|
|
27
|
+
if (actor?.role === 'viewer') throw new UserError('nl2sql/viewer', `当前成员是「只读」角色,不能${what}。只读成员可以:查询指标库中的指标(nl2sql_metric)、复用已验证的样例问答(nl2sql_knowledge search 后用样例 SQL 原样查询)、查看看板。请改用这些方式回答,或请管理员调整角色。`)
|
|
28
|
+
},
|
|
29
|
+
|
|
30
|
+
/** Whether a statement is (whitespace-insensitively) one of the verified example SQLs of the data source. */
|
|
31
|
+
isVerifiedExample(ds, sql) {
|
|
32
|
+
const norm = (q) => String(q ?? '').replace(/;\s*$/, '').replace(/\s+/g, ' ').trim().toLowerCase()
|
|
33
|
+
const target = norm(sql)
|
|
34
|
+
return this.knowledgeFor(ds.id).examples.some(e => norm(e.sql) === target) || this.state.dashboards.some(d => d.widgets.some(w => w.datasourceId === ds.id && norm(w.sql) === target))
|
|
35
|
+
},
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Row-level rules for an actor on a data source: [{ column, values }] combined
|
|
39
|
+
* from the member and the group. Every query must be rewritable to include them.
|
|
40
|
+
*/
|
|
41
|
+
rowRulesFor(actor, ds) {
|
|
42
|
+
const rules = []
|
|
43
|
+
for (const r of actor?.rowRules ?? []) if (!r.datasourceId || r.datasourceId === ds.id) rules.push({ column: r.column, values: r.values })
|
|
44
|
+
return rules
|
|
45
|
+
},
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Members: people (Lark open_id / email) and Lark chats, each with a role
|
|
49
|
+
* (admin | analyst | viewer) and a data source scope. The local owner is
|
|
50
|
+
* implicit. Lark messages from unknown people fall back to
|
|
51
|
+
* settings.larkDefaultRole (none = refuse).
|
|
52
|
+
*/
|
|
53
|
+
listUsers() {
|
|
54
|
+
return (this.state.users ?? []).map(u => ({ ...u, datasourceNames: (u.datasourceIds ?? []).map(id => this.findDatasource(id)?.name ?? id) }))
|
|
55
|
+
},
|
|
56
|
+
|
|
57
|
+
async saveUser(input) {
|
|
58
|
+
const existing = input.id ? (this.state.users ?? []).find(u => u.id === input.id) : undefined
|
|
59
|
+
const u = existing ? { ...existing } : { id: newId('u'), createdAt: Date.now() }
|
|
60
|
+
u.kind = oneOf(input.kind ?? u.kind ?? 'person', ['person', 'chat'], 'kind')
|
|
61
|
+
u.name = requireText(input.name ?? u.name, 'name', 60)
|
|
62
|
+
u.role = oneOf(input.role ?? u.role ?? 'analyst', ['admin', 'analyst', 'viewer'], 'role')
|
|
63
|
+
u.larkOpenId = String(input.larkOpenId ?? u.larkOpenId ?? '').trim().slice(0, 100) || undefined
|
|
64
|
+
u.email = String(input.email ?? u.email ?? '').trim().slice(0, 200) || undefined
|
|
65
|
+
u.larkChatId = String(input.larkChatId ?? u.larkChatId ?? '').trim().slice(0, 100) || undefined
|
|
66
|
+
if (u.kind === 'person' && !u.larkOpenId && !u.email) throw new UserError('nl2sql/invalid', '成员需要填写飞书 open_id 或邮箱,用于识别身份')
|
|
67
|
+
if (u.kind === 'chat' && !u.larkChatId) throw new UserError('nl2sql/invalid', '群成员需要填写飞书 chat_id')
|
|
68
|
+
const allDs = input.allDatasources ?? (input.datasourceIds === undefined ? u.datasourceIds === null : false)
|
|
69
|
+
u.datasourceIds = allDs ? null : (input.datasourceIds ?? u.datasourceIds ?? []).filter(id => this.findDatasource(id))
|
|
70
|
+
u.defaultDatasourceId = input.defaultDatasourceId && this.findDatasource(input.defaultDatasourceId) ? input.defaultDatasourceId : (u.datasourceIds?.[0] ?? null)
|
|
71
|
+
u.disabled = Boolean(input.disabled ?? u.disabled ?? false)
|
|
72
|
+
u.note = String(input.note ?? u.note ?? '').slice(0, 300)
|
|
73
|
+
if (input.rowRules !== undefined) {
|
|
74
|
+
u.rowRules = (Array.isArray(input.rowRules) ? input.rowRules : []).map(r => ({
|
|
75
|
+
datasourceId: r.datasourceId && this.findDatasource(r.datasourceId) ? r.datasourceId : null,
|
|
76
|
+
column: requireText(r.column, 'rowRules.column', 64),
|
|
77
|
+
values: (Array.isArray(r.values) ? r.values : String(r.values ?? '').split(/[,,]/)).map(v => String(v).trim()).filter(Boolean).slice(0, 200),
|
|
78
|
+
})).filter(r => r.values.length).slice(0, 20)
|
|
79
|
+
if (u.rowRules.some(r => !/^[A-Za-z_\u4e00-\u9fa5][\w$\u4e00-\u9fa5]*$/.test(r.column))) throw new UserError('nl2sql/invalid', '行级规则的字段名只能包含字母、数字、下划线或中文')
|
|
80
|
+
}
|
|
81
|
+
const dup = (this.state.users ?? []).find(x => x.id !== u.id && ((u.larkOpenId && x.larkOpenId === u.larkOpenId) || (u.email && x.email === u.email) || (u.larkChatId && x.larkChatId === u.larkChatId)))
|
|
82
|
+
if (dup) throw new UserError('nl2sql/duplicate', `该身份已属于成员「${dup.name}」`)
|
|
83
|
+
u.updatedAt = Date.now()
|
|
84
|
+
this.state.users = upsert(this.state.users ?? [], u)
|
|
85
|
+
this.auditAdmin('user.save', `${u.name}(${u.role})`)
|
|
86
|
+
await this.save()
|
|
87
|
+
return u
|
|
88
|
+
},
|
|
89
|
+
|
|
90
|
+
async deleteUser(id) {
|
|
91
|
+
const u = (this.state.users ?? []).find(x => x.id === id)
|
|
92
|
+
this.state.users = (this.state.users ?? []).filter(x => x.id !== id)
|
|
93
|
+
if (u) this.auditAdmin('user.delete', u.name)
|
|
94
|
+
await this.save()
|
|
95
|
+
return { deleted: id }
|
|
96
|
+
},
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Resolve the effective actor for a Lark message: the person's own scope,
|
|
100
|
+
* narrowed by the chat's scope in group chats. Returns { actor } or { refused }.
|
|
101
|
+
*/
|
|
102
|
+
resolveLarkActor({ openId, email, name, chatId, chatType }) {
|
|
103
|
+
const users = (this.state.users ?? []).filter(u => !u.disabled)
|
|
104
|
+
const person = users.find(u => u.kind === 'person' && ((openId && u.larkOpenId === openId) || (email && u.email && u.email.toLowerCase() === String(email).toLowerCase())))
|
|
105
|
+
const chat = chatType === 'group' ? users.find(u => u.kind === 'chat' && u.larkChatId === chatId) : undefined
|
|
106
|
+
const fallbackRole = this.state.settings.larkDefaultRole ?? 'none'
|
|
107
|
+
if (!person && !chat && fallbackRole === 'none') return { refused: '你还没有开通数据问答权限,请联系管理员在「数据问答 → 成员」中添加你的飞书账号。' }
|
|
108
|
+
const scopes = [person?.datasourceIds, chat?.datasourceIds].filter(x => x !== undefined)
|
|
109
|
+
let datasourceIds = null
|
|
110
|
+
for (const sc of scopes) if (Array.isArray(sc)) datasourceIds = datasourceIds === null ? [...sc] : datasourceIds.filter(id => sc.includes(id))
|
|
111
|
+
if (!person && !chat) datasourceIds = this.state.settings.larkDefaultDatasourceIds ?? []
|
|
112
|
+
const role = person?.role ?? chat?.role ?? fallbackRole
|
|
113
|
+
if (Array.isArray(datasourceIds) && datasourceIds.length === 0) return { refused: '你在当前群里没有可访问的数据源。' }
|
|
114
|
+
return {
|
|
115
|
+
actor: {
|
|
116
|
+
kind: 'lark', id: person?.id ?? `lark:${openId}`, name: person?.name ?? name ?? openId,
|
|
117
|
+
// Group chats: the narrower of the person's and the group's role.
|
|
118
|
+
role: [person?.role, chat?.role, (!person && !chat) ? fallbackRole : undefined].filter(Boolean).sort((a, b) => ROLE_RANK[a] - ROLE_RANK[b])[0] ?? role,
|
|
119
|
+
openId, chatId, chatName: chat?.name, datasourceIds,
|
|
120
|
+
rowRules: [...(person?.rowRules ?? []), ...(chat?.rowRules ?? [])],
|
|
121
|
+
defaultDatasourceId: chat?.defaultDatasourceId ?? person?.defaultDatasourceId ?? (datasourceIds?.[0] ?? null),
|
|
122
|
+
},
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
|
|
126
|
+
auditAdmin(action, detail, actor) {
|
|
127
|
+
this.records.put('admin', { id: newId('ad'), at: Date.now(), action, detail: String(detail).slice(0, 300), ...(actor ? { actor: { id: actor.id, name: actor.name, kind: actor.kind } } : {}) })
|
|
128
|
+
},
|
|
129
|
+
|
|
130
|
+
/** Per-actor usage for the members page. */
|
|
131
|
+
usageByActor(days = 30) {
|
|
132
|
+
const since = Date.now() - days * 86400000
|
|
133
|
+
const by = new Map()
|
|
134
|
+
for (const h of this.records.list('history', { since, limit: 100000 })) {
|
|
135
|
+
const key = h.actor?.id ?? 'owner'
|
|
136
|
+
const e = by.get(key) ?? { actorId: key, name: h.actor?.name ?? '本机', kind: h.actor?.kind ?? 'owner', queries: 0, errors: 0, blocked: 0, lastAt: 0 }
|
|
137
|
+
e.queries++
|
|
138
|
+
if (h.status === 'error') e.errors++
|
|
139
|
+
if (h.status === 'blocked') e.blocked++
|
|
140
|
+
e.lastAt = Math.max(e.lastAt, h.at)
|
|
141
|
+
by.set(key, e)
|
|
142
|
+
}
|
|
143
|
+
return [...by.values()].sort((a, b) => b.queries - a.queries)
|
|
144
|
+
},
|
|
145
|
+
}
|