@hedeqiang/dsh-nl2sql 0.0.0-stage → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/LICENSE +21 -0
- package/README.en.md +62 -0
- package/README.md +88 -2
- package/analysis.js +888 -0
- package/cordis.patch.yml +68 -0
- package/drivers.js +694 -0
- package/filters.js +237 -0
- package/icon.svg +13 -0
- package/im.js +511 -0
- package/index.js +650 -0
- package/lark-bot.js +606 -0
- package/lib/client.echarts.js +57 -0
- package/lib/client.js +4845 -0
- package/locale/en.json +1 -0
- package/locale/zh.json +1 -0
- package/package.json +90 -4
- package/policy.js +362 -0
- package/service/access.js +145 -0
- package/service/admin.js +539 -0
- package/service/alerts.js +233 -0
- package/service/dashboards.js +359 -0
- package/service/datasources.js +392 -0
- package/service/evaluation.js +121 -0
- package/service/lark-actions.js +223 -0
- package/service/lark.js +250 -0
- package/service/push.js +314 -0
- package/service/query.js +411 -0
- package/service/schema.js +507 -0
- package/service/shared.js +424 -0
- package/service/suggestions.js +772 -0
- package/service/uploads.js +258 -0
- package/sql-guard.js +222 -0
- package/sqlite-driver.js +677 -0
- package/store.js +225 -0
- package/xlsx.js +1001 -0
package/service/query.js
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Nl2SqlService methods: query.
|
|
3
|
+
* Mixed into the service prototype by index.js; `this` is the service.
|
|
4
|
+
*/
|
|
5
|
+
import { aliasExpr, buildAttributionSql, buildBreakdownSql, buildDetailSql, buildMetricSql, computeAttribution, isAdditive, parseSelect, quoteName, rankDimensions } from '../analysis.js'
|
|
6
|
+
import { applyFilters, normalizeFilters } from '../filters.js'
|
|
7
|
+
import { checkAccess, filterSchema, policyActive, referencedTables, resultColumnPolicy } from '../policy.js'
|
|
8
|
+
import { admitSql, prepareSql, stripSql } from '../sql-guard.js'
|
|
9
|
+
import { cacheKey } from '../store.js'
|
|
10
|
+
import { writeXlsx } from '../xlsx.js'
|
|
11
|
+
import { RESULT_CACHE_LIMIT, SENSITIVE_NAME, UserError, columnKind, fillChartHint, fmtNum, maskValue, newId, normalizeChart, suggestChart, summarize } from './shared.js'
|
|
12
|
+
|
|
13
|
+
export const methods = {
|
|
14
|
+
// ---------------------------------------------------------------- query
|
|
15
|
+
|
|
16
|
+
async runQuery({ sql, datasourceId, sessionId, title, chart, source, confirmScan, filters, useCache, actor, governed }) {
|
|
17
|
+
actor ??= this.actorFor(sessionId)
|
|
18
|
+
const ds = this.datasourceForSession(sessionId, datasourceId)
|
|
19
|
+
this.assertAccess(actor, ds)
|
|
20
|
+
// Viewers run only governed SQL: metric library, verified examples, dashboard widgets.
|
|
21
|
+
if (actor?.role === 'viewer' && !governed && !this.isVerifiedExample(ds, sql)) this.assertCanWriteSql(actor)
|
|
22
|
+
if (sessionId && !this.state.sessions[sessionId]) {
|
|
23
|
+
this.state.sessions[sessionId] = ds.id
|
|
24
|
+
await this.save()
|
|
25
|
+
}
|
|
26
|
+
const settings = this.settingsFor(ds)
|
|
27
|
+
const dialect = ds.type ?? 'mysql'
|
|
28
|
+
let admitted = admitSql(sql, { dialect })
|
|
29
|
+
const policy = this.policyFor(ds)
|
|
30
|
+
const fullSchema = policyActive(policy) || filters?.length ? await this.analyzeSchema(ds.id) : (this.schemas.get(ds.id) ?? await this.readSchemaFile(ds.id))
|
|
31
|
+
const denied = checkAccess(admitted.sql, policy, fullSchema, admitted.head, { dialect })
|
|
32
|
+
if (denied) {
|
|
33
|
+
this.audit({ ds, sessionId, source, sql: admitted.sql, title, status: 'blocked', error: denied, actor })
|
|
34
|
+
await this.save()
|
|
35
|
+
throw new UserError('nl2sql/access-denied', denied)
|
|
36
|
+
}
|
|
37
|
+
let appliedFilters = []
|
|
38
|
+
if (filters?.length) {
|
|
39
|
+
const f = applyFilters(admitted.sql, normalizeFilters(filters), { dialect, schema: filterSchema(fullSchema, policy) })
|
|
40
|
+
appliedFilters = f.applied
|
|
41
|
+
if (f.applied.length) admitted = admitSql(f.sql, { dialect })
|
|
42
|
+
}
|
|
43
|
+
// Row-level security: every rule must be injected into the outer query; otherwise refuse.
|
|
44
|
+
const rowRules = this.rowRulesFor(actor, ds)
|
|
45
|
+
if (rowRules.length) {
|
|
46
|
+
const schemaForRules = fullSchema ?? await this.analyzeSchema(ds.id)
|
|
47
|
+
const refs = referencedTables(admitted.sql, schemaForRules.tables, { dialect })
|
|
48
|
+
const needed = rowRules.filter(r => refs.some(t => schemaForRules.tables.find(x => x.name === t)?.columns.some(c => c.name.toLowerCase() === r.column.toLowerCase())))
|
|
49
|
+
if (needed.length) {
|
|
50
|
+
if (!/^(SELECT|WITH)$/.test(admitted.head)) throw new UserError('nl2sql/row-policy', '当前成员受行级权限限制,只能执行 SELECT 查询')
|
|
51
|
+
// Subqueries / CTEs / derived tables could read the table without the filter: allow only a single flat SELECT.
|
|
52
|
+
const flat = stripSql(admitted.sql, { dialect }).code
|
|
53
|
+
if (/\(\s*SELECT\b/i.test(flat) || /^\s*WITH\b/i.test(flat) || /\b(UNION|INTERSECT|EXCEPT)\b/i.test(flat)) {
|
|
54
|
+
this.audit({ ds, sessionId, source, sql: admitted.sql, title, status: 'blocked', error: 'row-level policy: nested query', actor })
|
|
55
|
+
await this.save()
|
|
56
|
+
throw new UserError('nl2sql/row-policy', `当前成员受行级权限限制(${needed.map(r => `${r.column} ∈ {${r.values.join(', ')}}`).join(';')}),查询不能包含子查询、CTE 或 UNION,请改写为单层查询。`)
|
|
57
|
+
}
|
|
58
|
+
const f = applyFilters(admitted.sql, needed.map(r => ({ kind: 'in', column: r.column, values: r.values })), { dialect, schema: schemaForRules })
|
|
59
|
+
const missing = needed.filter(r => !f.applied.some(a => a.column.toLowerCase() === r.column.toLowerCase()))
|
|
60
|
+
// Every referenced table that has the column must be covered.
|
|
61
|
+
const tablesWithCol = (col) => refs.filter(t => schemaForRules.tables.find(x => x.name === t)?.columns.some(c => c.name.toLowerCase() === col.toLowerCase()))
|
|
62
|
+
const uncovered = needed.filter(r => tablesWithCol(r.column).some(t => !f.applied.some(a => a.column.toLowerCase() === r.column.toLowerCase() && a.table === t)))
|
|
63
|
+
if (missing.length || uncovered.length) {
|
|
64
|
+
this.audit({ ds, sessionId, source, sql: admitted.sql, title, status: 'blocked', error: 'row-level policy not applicable', actor })
|
|
65
|
+
await this.save()
|
|
66
|
+
throw new UserError('nl2sql/row-policy', `当前成员受行级权限限制(${needed.map(r => `${r.column} ∈ {${r.values.join(', ')}}`).join(';')}),这条查询无法安全地加上该过滤,已拒绝。`)
|
|
67
|
+
}
|
|
68
|
+
admitted = admitSql(f.sql, { dialect })
|
|
69
|
+
appliedFilters = [...appliedFilters, ...f.applied.map(a => ({ ...a, description: `行级权限:${a.description ?? a.column}`, rowPolicy: true }))]
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
// Query cache: dashboards, reruns and reports reuse a recent identical result.
|
|
73
|
+
const ttlMs = (settings.cacheTtlSec ?? 0) * 1000
|
|
74
|
+
const key = ttlMs > 0 && useCache && this.store ? cacheKey([ds.id, admitted.sql, settings.maxRows, settings.maskSensitive, ds.policy ?? null, ds.updatedAt ?? 0, rowRules]) : null
|
|
75
|
+
if (key) {
|
|
76
|
+
const hit = this.store.cached(key, ttlMs)
|
|
77
|
+
if (hit) {
|
|
78
|
+
const result = { ...hit, resultId: newId('r'), title: title ? String(title).slice(0, 120) : hit.title, chart: normalizeChart(chart, hit.columns) ?? hit.chart, cached: true, cachedAt: hit.createdAt, createdAt: Date.now(), ...(appliedFilters.length ? { filters: appliedFilters, baseSql: sql } : {}) }
|
|
79
|
+
this.remember(result)
|
|
80
|
+
this.audit({ ds, sessionId, source, sql: admitted.sql, title, status: 'ok', rows: result.rowCount, elapsedMs: 0, resultId: result.resultId, cached: true, actor })
|
|
81
|
+
await this.save()
|
|
82
|
+
return result
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const driver = await this.withDriver(ds, d => d)
|
|
86
|
+
const prepared = prepareSql(admitted, { ...settings, useHint: dialect === 'mysql' && driver.useHint, dialect })
|
|
87
|
+
if (admitted.head === 'SELECT' && settings.scanGuardRows > 0 && !confirmScan && dialect !== 'elasticsearch') {
|
|
88
|
+
const risk = await this.scanRisk(ds, admitted.sql, settings)
|
|
89
|
+
if (risk) {
|
|
90
|
+
this.audit({ ds, sessionId, source, sql: admitted.sql, title, status: 'blocked', error: risk.message, actor })
|
|
91
|
+
await this.save()
|
|
92
|
+
throw new UserError('nl2sql/scan-guard', risk.message)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
const started = Date.now()
|
|
96
|
+
let res
|
|
97
|
+
try {
|
|
98
|
+
res = await this.withDriver(ds, d => d.query(prepared, { timeoutMs: settings.timeoutMs, maxRows: settings.maxRows }))
|
|
99
|
+
} catch (error) {
|
|
100
|
+
this.audit({ ds, sessionId, source, sql: admitted.sql, title, status: 'error', error: error.message, elapsedMs: Date.now() - started, actor })
|
|
101
|
+
await this.save()
|
|
102
|
+
throw error
|
|
103
|
+
}
|
|
104
|
+
const elapsedMs = Date.now() - started
|
|
105
|
+
const truncated = res.rows.length > settings.maxRows
|
|
106
|
+
let cells = truncated ? res.rows.slice(0, settings.maxRows) : res.rows
|
|
107
|
+
const schema = fullSchema
|
|
108
|
+
// Access policy: drop hidden columns that arrive anyway (SELECT *), force-mask policy-masked ones.
|
|
109
|
+
const refs = schema ? referencedTables(admitted.sql, schema.tables, { dialect }) : []
|
|
110
|
+
const colPolicy = resultColumnPolicy(policy, res.columns, refs, schema)
|
|
111
|
+
let masks = colPolicy.mask
|
|
112
|
+
if (colPolicy.drop.size) {
|
|
113
|
+
const keep = res.columns.map((_, i) => i).filter(i => !colPolicy.drop.has(i))
|
|
114
|
+
res = { ...res, columns: keep.map(i => res.columns[i]) }
|
|
115
|
+
cells = cells.map(r => keep.map(i => r[i]))
|
|
116
|
+
masks = new Set([...masks].map(i => keep.indexOf(i)).filter(i => i >= 0))
|
|
117
|
+
}
|
|
118
|
+
const sensitive = new Set(masks)
|
|
119
|
+
if (settings.maskSensitive) {
|
|
120
|
+
res.columns.forEach((f, i) => {
|
|
121
|
+
const col = f.table ? schema?.tables.find(t => t.name === f.table)?.columns.find(c => c.name === f.column) : undefined
|
|
122
|
+
if (col?.roles.includes('sensitive') || SENSITIVE_NAME.test(f.name)) sensitive.add(i)
|
|
123
|
+
})
|
|
124
|
+
}
|
|
125
|
+
for (const row of cells) for (const i of sensitive) row[i] = maskValue(res.columns[i].name, row[i])
|
|
126
|
+
const columns = res.columns.map((f, i) => ({ name: f.name, kind: columnKind(f, cells, i), ...(sensitive.has(i) ? { masked: true } : {}) }))
|
|
127
|
+
const suggested = suggestChart(columns, cells)
|
|
128
|
+
const resultId = newId('r')
|
|
129
|
+
const result = {
|
|
130
|
+
resultId,
|
|
131
|
+
datasourceId: ds.id,
|
|
132
|
+
datasourceName: ds.name,
|
|
133
|
+
sql: admitted.sql,
|
|
134
|
+
executedSql: prepared,
|
|
135
|
+
...(appliedFilters.length ? { filters: appliedFilters, baseSql: sql } : {}),
|
|
136
|
+
...(colPolicy.drop.size ? { hiddenColumns: colPolicy.drop.size } : {}),
|
|
137
|
+
title: title ? String(title).slice(0, 120) : '',
|
|
138
|
+
columns,
|
|
139
|
+
rows: cells,
|
|
140
|
+
rowCount: cells.length,
|
|
141
|
+
truncated,
|
|
142
|
+
elapsedMs,
|
|
143
|
+
chart: normalizeChart(chart, columns) ?? fillChartHint(chart, columns, suggested) ?? suggested,
|
|
144
|
+
suggestedChart: suggested,
|
|
145
|
+
createdAt: Date.now(),
|
|
146
|
+
}
|
|
147
|
+
this.remember(result)
|
|
148
|
+
if (key) try { this.store.remember(key, resultId) } catch {}
|
|
149
|
+
this.audit({ ds, sessionId, source, sql: admitted.sql, title, status: 'ok', rows: cells.length, elapsedMs, resultId, actor })
|
|
150
|
+
await this.save()
|
|
151
|
+
return result
|
|
152
|
+
},
|
|
153
|
+
|
|
154
|
+
audit({ ds, sessionId, source, sql, title, status, error, rows, elapsedMs, resultId, cached, actor }) {
|
|
155
|
+
const who = actor ?? (sessionId ? this.state.sessionActors?.[sessionId] : undefined)
|
|
156
|
+
this.records.put('history', {
|
|
157
|
+
id: newId('h'), at: Date.now(), datasourceId: ds.id, datasourceName: ds.name, sessionId: sessionId ?? null,
|
|
158
|
+
source: source ?? 'ui', title: title ? String(title).slice(0, 120) : '', sql, status,
|
|
159
|
+
...(cached ? { cached: true } : {}),
|
|
160
|
+
...(who ? { actor: { id: who.id, name: who.name, kind: who.kind } } : {}),
|
|
161
|
+
...(error ? { error: String(error).slice(0, 500) } : {}),
|
|
162
|
+
...(rows !== undefined ? { rows } : {}), ...(elapsedMs !== undefined ? { elapsedMs } : {}), ...(resultId ? { resultId } : {}),
|
|
163
|
+
})
|
|
164
|
+
},
|
|
165
|
+
|
|
166
|
+
/** EXPLAIN the statement and refuse full scans over large tables. */
|
|
167
|
+
async scanRisk(ds, sql, settings) {
|
|
168
|
+
let scans
|
|
169
|
+
try {
|
|
170
|
+
({ scans } = await (await this.driver(ds)).explain(sql, { timeoutMs: Math.min(settings.timeoutMs, 10000) }))
|
|
171
|
+
} catch {
|
|
172
|
+
return undefined // EXPLAIN failures surface when the query itself runs
|
|
173
|
+
}
|
|
174
|
+
const big = scans.filter(r => r.rows >= settings.scanGuardRows)
|
|
175
|
+
if (!big.length) return undefined
|
|
176
|
+
const detail = big.map(r => `${r.table}(约 ${Number(r.rows).toLocaleString()} 行)`).join('、')
|
|
177
|
+
return { message: `执行计划显示会全表扫描大表:${detail},超过阈值 ${settings.scanGuardRows.toLocaleString()} 行。请增加带索引的过滤条件(如时间范围、主键/索引列),或改用聚合;确认要执行时,使用 confirmScan=true 重新调用。` }
|
|
178
|
+
},
|
|
179
|
+
|
|
180
|
+
async explain({ sql, datasourceId, sessionId }) {
|
|
181
|
+
const ds = this.datasourceForSession(sessionId, datasourceId)
|
|
182
|
+
const admitted = admitSql(sql, { dialect: ds.type ?? 'mysql' })
|
|
183
|
+
if (admitted.head !== 'SELECT') throw new UserError('nl2sql/invalid', '只能对 SELECT 生成执行计划')
|
|
184
|
+
const denied = checkAccess(admitted.sql, this.policyFor(ds), await this.analyzeSchema(ds.id), admitted.head, { dialect: ds.type ?? 'mysql' })
|
|
185
|
+
if (denied) throw new UserError('nl2sql/access-denied', denied)
|
|
186
|
+
const res = await this.withDriver(ds, d => d.explain(admitted.sql, { timeoutMs: this.settingsFor(ds).timeoutMs }))
|
|
187
|
+
return { datasourceId: ds.id, dialect: ds.type ?? 'mysql', plan: res.plan, scans: res.scans }
|
|
188
|
+
},
|
|
189
|
+
|
|
190
|
+
remember(result) {
|
|
191
|
+
this.results.set(result.resultId, result)
|
|
192
|
+
while (this.results.size > RESULT_CACHE_LIMIT) this.results.delete(this.results.keys().next().value)
|
|
193
|
+
try { this.store?.put(result) } catch (error) { console.warn('nl2sql: cannot persist result: %s', error.message) }
|
|
194
|
+
},
|
|
195
|
+
|
|
196
|
+
/** A result from memory, falling back to the durable store (survives restarts). */
|
|
197
|
+
lookupResult(resultId) {
|
|
198
|
+
if (!resultId) return undefined
|
|
199
|
+
let r = this.results.get(resultId)
|
|
200
|
+
if (!r && this.store) {
|
|
201
|
+
try { r = this.store.get(resultId) } catch {}
|
|
202
|
+
if (r) {
|
|
203
|
+
this.results.set(resultId, r)
|
|
204
|
+
while (this.results.size > RESULT_CACHE_LIMIT) this.results.delete(this.results.keys().next().value)
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
return r
|
|
208
|
+
},
|
|
209
|
+
|
|
210
|
+
getResult(resultId) {
|
|
211
|
+
return this.lookupResult(resultId) ?? null
|
|
212
|
+
},
|
|
213
|
+
|
|
214
|
+
/** A scoped actor may only use results of data sources in scope (and with its own row rules applied). */
|
|
215
|
+
assertResultAccess(r, actor) {
|
|
216
|
+
if (!actor || actor.kind === 'owner') return
|
|
217
|
+
this.assertAccess(actor, this.requireDatasource(r.datasourceId))
|
|
218
|
+
if (this.rowRulesFor(actor, this.requireDatasource(r.datasourceId)).length && !r.filters?.some(f => f.rowPolicy)) throw new UserError('nl2sql/row-policy', '该结果未按你的行级权限过滤,不能使用,请重新查询')
|
|
219
|
+
},
|
|
220
|
+
|
|
221
|
+
cacheStats() {
|
|
222
|
+
return this.store ? this.store.stats() : { results: this.results.size, bytes: 0, cacheEntries: 0 }
|
|
223
|
+
},
|
|
224
|
+
|
|
225
|
+
async clearCache() {
|
|
226
|
+
this.store?.clearCache()
|
|
227
|
+
return this.cacheStats()
|
|
228
|
+
},
|
|
229
|
+
|
|
230
|
+
async rechart({ resultId, chart, title, sessionId }) {
|
|
231
|
+
const r = this.lookupResult(resultId)
|
|
232
|
+
if (r) this.assertResultAccess(r, this.actorFor(sessionId))
|
|
233
|
+
if (!r) throw new UserError('nl2sql/result-expired', `结果 ${resultId} 不存在或已过期,请重新执行 nl2sql_query。`)
|
|
234
|
+
const normalized = normalizeChart(chart, r.columns)
|
|
235
|
+
if (!normalized) throw new UserError('nl2sql/invalid-chart', `图表配置无效。可用列:${r.columns.map(c => `${c.name}(${c.kind})`).join(', ')}`)
|
|
236
|
+
const next = { ...r, resultId: newId('r'), chart: normalized, title: title ? String(title).slice(0, 120) : r.title, createdAt: Date.now() }
|
|
237
|
+
this.remember(next)
|
|
238
|
+
return next
|
|
239
|
+
},
|
|
240
|
+
|
|
241
|
+
/** The part of a result the model may see, per privacy settings. */
|
|
242
|
+
modelView(result) {
|
|
243
|
+
const ds = this.findDatasource(result.datasourceId)
|
|
244
|
+
const settings = this.settingsFor(ds)
|
|
245
|
+
const view = {
|
|
246
|
+
resultId: result.resultId,
|
|
247
|
+
datasource: result.datasourceName,
|
|
248
|
+
rowCount: result.rowCount,
|
|
249
|
+
truncated: result.truncated,
|
|
250
|
+
elapsedMs: result.elapsedMs,
|
|
251
|
+
columns: result.columns.map(c => `${c.name}:${c.kind}${c.masked ? '(masked)' : ''}`),
|
|
252
|
+
chart: result.chart,
|
|
253
|
+
}
|
|
254
|
+
if (settings.resultToModel === 'summary') view.summary = summarize(result.columns, result.rows)
|
|
255
|
+
if (settings.resultToModel === 'rows') view.rows = result.rows.slice(0, settings.modelRows)
|
|
256
|
+
view.privacy = settings.resultToModel
|
|
257
|
+
return view
|
|
258
|
+
},
|
|
259
|
+
|
|
260
|
+
// -------------------------------------------------------------- metrics
|
|
261
|
+
|
|
262
|
+
findMetric(ds, ref) {
|
|
263
|
+
const k = this.knowledgeFor(ds.id)
|
|
264
|
+
const q = String(ref ?? '').trim().toLowerCase()
|
|
265
|
+
return k.metrics.find(m => m.id === ref || m.name.toLowerCase() === q || (m.synonyms ?? []).some(x => x.toLowerCase() === q))
|
|
266
|
+
},
|
|
267
|
+
|
|
268
|
+
/** Query a governed metric from the metric library (definition enforced, not re-derived by the model). */
|
|
269
|
+
async queryMetric({ datasourceId, sessionId, metric: ref, grain, dimensions = [], range, from, to, filters = [], limit, title, source = 'agent', actor, chart }) {
|
|
270
|
+
const ds = this.datasourceForSession(sessionId, datasourceId)
|
|
271
|
+
const metric = this.findMetric(ds, ref)
|
|
272
|
+
if (!metric) {
|
|
273
|
+
const all = this.knowledgeFor(ds.id).metrics
|
|
274
|
+
throw new UserError('nl2sql/unknown-metric', `指标库中没有「${ref}」。${all.length ? `可用指标:${all.map(m => m.name).join('、')}` : '指标库为空,请先在「业务知识 → 指标库」中定义。'}`)
|
|
275
|
+
}
|
|
276
|
+
if (metric.dimensions?.length && dimensions.length) {
|
|
277
|
+
const bad = dimensions.filter(d => !metric.dimensions.includes(d))
|
|
278
|
+
if (bad.length) throw new UserError('nl2sql/invalid', `指标「${metric.name}」不支持按 ${bad.join('、')} 拆分。允许的维度:${metric.dimensions.join('、')}`)
|
|
279
|
+
}
|
|
280
|
+
const built = buildMetricSql({ metric, dialect: ds.type ?? 'mysql', grain, dimensions, range, from, to, filters, limit: limit ?? 1000, now: new Date() })
|
|
281
|
+
if (built.error) throw new UserError('nl2sql/invalid', built.error)
|
|
282
|
+
const ch = chart ?? (grain ? { type: dimensions.length ? 'line' : 'area', x: '时间', y: [metric.name], ...(dimensions.length ? { series: dimensions[0] } : {}) } : dimensions.length ? { type: 'bar', x: dimensions[0], y: [metric.name] } : { type: 'kpi', y: [metric.name] })
|
|
283
|
+
const result = await this.runQuery({ governed: true, sql: built.sql, datasourceId: ds.id, sessionId, title: title || `${metric.name}${grain ? ` · 按${{ day: '天', week: '周', month: '月', hour: '小时', year: '年' }[grain] ?? grain}` : ''}${dimensions.length ? ` · 按${dimensions.join('、')}` : ''}`, chart: ch, source, actor, useCache: true })
|
|
284
|
+
return { ...result, metric: { id: metric.id, name: metric.name, unit: metric.unit, expression: metric.expression } }
|
|
285
|
+
},
|
|
286
|
+
|
|
287
|
+
// --------------------------------------------------------- drill & why
|
|
288
|
+
|
|
289
|
+
/** Drill a result: detail rows behind one point, or breakdown by a dimension. */
|
|
290
|
+
async drill({ resultId, mode, xValue, xColumn, dimension, sessionId, actor, source = 'ui' }) {
|
|
291
|
+
const r = this.lookupResult(resultId)
|
|
292
|
+
if (!r) throw new UserError('nl2sql/result-expired', '结果已过期,请重新查询')
|
|
293
|
+
const ds = this.requireDatasource(r.datasourceId)
|
|
294
|
+
this.assertResultAccess(r, actor ?? this.actorFor(sessionId))
|
|
295
|
+
const dialect = ds.type ?? 'mysql'
|
|
296
|
+
// Drill from the user-visible SQL; row-level rules are re-applied by runQuery.
|
|
297
|
+
const base = r.baseSql ?? r.sql
|
|
298
|
+
const x = xColumn ?? r.chart?.x
|
|
299
|
+
let built
|
|
300
|
+
if (mode === 'detail') built = buildDetailSql({ sql: base, dialect, xColumn: x, xValue, limit: 200 })
|
|
301
|
+
else {
|
|
302
|
+
if (!dimension) throw new UserError('nl2sql/invalid', '请选择拆分维度')
|
|
303
|
+
built = buildBreakdownSql({ sql: base, dialect, xColumn: xValue === undefined ? undefined : x, xValue, dimension: quoteName(dimension, dialect), dimensionLabel: dimension, limit: 30 })
|
|
304
|
+
}
|
|
305
|
+
if (built.error) throw new UserError('nl2sql/invalid', built.error)
|
|
306
|
+
const label = xValue === undefined ? '' : `${x}=${xValue}`
|
|
307
|
+
const title = mode === 'detail' ? `${r.title || '明细'} · ${label} 明细` : `${r.title || '拆分'}${label ? ` · ${label}` : ''} · 按${dimension}`
|
|
308
|
+
if (mode === 'detail') this.assertCanWriteSql(actor ?? this.actorFor(sessionId), '查看明细数据')
|
|
309
|
+
return this.runQuery({ governed: true, sql: built.sql, datasourceId: ds.id, sessionId, title, source, actor, chart: mode === 'detail' ? { type: 'table' } : { type: 'bar', x: built.label ?? dimension, ...(built.measures?.length ? { y: built.measures.slice(0, 1) } : {}) } })
|
|
310
|
+
},
|
|
311
|
+
|
|
312
|
+
/** Columns of the tables a result reads that make sense as breakdown dimensions. */
|
|
313
|
+
async drillDimensions(resultId) {
|
|
314
|
+
const r = this.lookupResult(resultId)
|
|
315
|
+
if (!r) throw new UserError('nl2sql/result-expired', '结果已过期,请重新查询')
|
|
316
|
+
const ds = this.requireDatasource(r.datasourceId)
|
|
317
|
+
const schema = await this.visibleSchema(ds)
|
|
318
|
+
const refs = referencedTables(r.baseSql ?? r.sql, schema.tables, { dialect: ds.type ?? 'mysql' })
|
|
319
|
+
const out = []
|
|
320
|
+
for (const t of schema.tables.filter(x => refs.includes(x.name))) {
|
|
321
|
+
for (const c of t.columns) {
|
|
322
|
+
if (c.roles.includes('sensitive') || c.roles.includes('id') || c.roles.includes('measure') || c.roles.includes('time')) continue
|
|
323
|
+
const lowCard = c.distinct !== undefined && c.distinct !== '>12' && c.distinct > 1
|
|
324
|
+
const categorical = lowCard || c.roles.includes('enum') || c.roles.includes('geo') || c.dataType === 'keyword' || c.dataType === 'enum' || c.dataType === 'bool' || c.dataType === 'boolean'
|
|
325
|
+
// Free-text / high-cardinality columns (hash, address, name) make poor dimensions.
|
|
326
|
+
const shortText = /^(varchar|char|character varying|text)$/.test(c.dataType) && (((/\((\d+)\)/.test(c.columnType ?? '') && Number(/\((\d+)\)/.exec(c.columnType)[1]) <= 32)) || ds.type === 'sqlite') && !/(hash|addr|address|url|name|title|desc|memo|remark|content|email|phone|token|key|code)$/i.test(c.name)
|
|
327
|
+
if (!categorical && !shortText) continue
|
|
328
|
+
out.push({ column: refs.length > 1 ? `${t.name.split('.').pop()}.${c.name}` : c.name, label: c.comment || c.name, table: t.name, values: c.values?.length ?? null, rank: categorical ? 0 : 1 })
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
out.sort((a, b) => a.rank - b.rank)
|
|
332
|
+
return { x: r.chart?.x ?? null, dimensions: out.slice(0, 40).map(({ rank: _r, ...d }) => d) }
|
|
333
|
+
},
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* "Why did it change": compare two periods of a time-series result and
|
|
337
|
+
* rank dimensions by how well they explain the delta.
|
|
338
|
+
*/
|
|
339
|
+
async explainChange({ resultId, current, baseline, dimensions, measure, sessionId, actor }) {
|
|
340
|
+
const r = this.lookupResult(resultId)
|
|
341
|
+
if (!r) throw new UserError('nl2sql/result-expired', '结果已过期,请重新查询')
|
|
342
|
+
const ds = this.requireDatasource(r.datasourceId)
|
|
343
|
+
this.assertResultAccess(r, actor ?? this.actorFor(sessionId))
|
|
344
|
+
const dialect = ds.type ?? 'mysql'
|
|
345
|
+
const parsed = parseSelect(r.baseSql ?? r.sql, { dialect })
|
|
346
|
+
if (!parsed) throw new UserError('nl2sql/invalid', '这个查询结构太复杂(UNION / 无 FROM),无法自动归因')
|
|
347
|
+
const x = r.chart?.x
|
|
348
|
+
const timeExpr = aliasExpr(parsed, x)
|
|
349
|
+
if (!timeExpr) throw new UserError('nl2sql/invalid', '需要一个按时间(或类别)分组的结果,才能比较两个时段')
|
|
350
|
+
const measureItem = measure ? parsed.select.find(i => i.alias === measure) : parsed.select.find(i => i.isAggregate && (!r.chart?.y?.length || r.chart.y.includes(i.alias)))
|
|
351
|
+
if (!measureItem?.isAggregate) throw new UserError('nl2sql/invalid', '需要一个聚合指标(SUM / COUNT 等)才能归因')
|
|
352
|
+
// Default periods: last two points of the result.
|
|
353
|
+
const xi = r.columns.findIndex(c => c.name === x)
|
|
354
|
+
const points = r.rows.map(row => row[xi]).filter(v => v !== null && v !== undefined)
|
|
355
|
+
const cur = current ?? points.at(-1)
|
|
356
|
+
const base = baseline ?? points.at(-2)
|
|
357
|
+
if (cur === undefined || base === undefined) throw new UserError('nl2sql/invalid', '至少需要两个时间点')
|
|
358
|
+
let dims = dimensions
|
|
359
|
+
if (!dims?.length) dims = (await this.drillDimensions(resultId)).dimensions.filter(d => d.values === null || d.values <= 60).slice(0, 6).map(d => d.column)
|
|
360
|
+
const additive = isAdditive(measureItem.expr)
|
|
361
|
+
const results = []
|
|
362
|
+
for (const dim of dims.slice(0, 8)) {
|
|
363
|
+
const built = buildAttributionSql({ dialect, from: parsed.from, where: parsed.where, timeExpr, measureExpr: measureItem.expr, dimension: quoteName(dim, dialect), current: cur, baseline: base, limit: 200 })
|
|
364
|
+
if (built.error) { results.push({ dimension: dim, error: built.error }); continue }
|
|
365
|
+
try {
|
|
366
|
+
const res = await this.runQuery({ governed: true, sql: built.sql, datasourceId: ds.id, sessionId, title: `归因 · ${dim}`, source: sessionId ? 'agent' : 'ui', actor, useCache: true, confirmScan: true })
|
|
367
|
+
const rows = res.rows.map(row => ({ dim: row[0], period: row[1], value: row[2] === null ? 0 : Number(row[2]) }))
|
|
368
|
+
const attribution = computeAttribution(rows, { additive })
|
|
369
|
+
results.push({ dimension: dim, attribution })
|
|
370
|
+
} catch (error) {
|
|
371
|
+
results.push({ dimension: dim, error: error.message })
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
const ranked = rankDimensions(results.filter(x => x.attribution))
|
|
375
|
+
const top = ranked[0]
|
|
376
|
+
const total = top?.attribution.total
|
|
377
|
+
const mixShift = top && Math.abs(top.attribution.total.delta) < top.attribution.items.reduce((a, i) => a + Math.abs(i.delta), 0) / 4
|
|
378
|
+
const summary = top
|
|
379
|
+
? `${x}「${cur}」对比「${base}」:${measureItem.alias ?? '指标'} 从 ${fmtNum(total.baseline)} 变为 ${fmtNum(total.current)}(${total.delta >= 0 ? '+' : ''}${fmtNum(total.delta)}${total.pct === null ? '' : `,${total.pct >= 0 ? '+' : ''}${(total.pct * 100).toFixed(1)}%`})。最能解释变化的维度是「${top.dimension}」:${top.attribution.items.slice(0, 3).map(i => `${i.value} ${i.delta >= 0 ? '+' : ''}${fmtNum(i.delta)}${i.share === null || Math.abs(i.share) > 3 ? '' : `(贡献 ${(i.share * 100).toFixed(0)}%)`}`).join(',')}。${mixShift ? '总量变化不大,但各分组此消彼长(结构性变化)。' : ''}`
|
|
380
|
+
: '没有找到可解释变化的维度。'
|
|
381
|
+
return { current: cur, baseline: base, measure: measureItem.alias ?? measureItem.expr, additive, dimensions: [...ranked, ...results.filter(x => !x.attribution)], summary }
|
|
382
|
+
},
|
|
383
|
+
|
|
384
|
+
// ------------------------------------------------------------- export
|
|
385
|
+
|
|
386
|
+
/** Excel workbook (base64) for results, a dashboard, or one result; each result becomes a sheet. */
|
|
387
|
+
async exportExcel({ resultIds = [], dashboardId, title, includeSql = true }) {
|
|
388
|
+
let results = resultIds.map(id => this.lookupResult(id)).filter(Boolean)
|
|
389
|
+
let name = title
|
|
390
|
+
if (dashboardId) {
|
|
391
|
+
const d = this.requireDashboard(dashboardId)
|
|
392
|
+
name = name || d.name
|
|
393
|
+
results = []
|
|
394
|
+
for (const w of d.widgets.slice(0, 50)) {
|
|
395
|
+
try { results.push({ ...(await this.runWidget(w.id, d.filters)), title: w.title }) } catch (error) { results.push({ title: w.title, error: error.message, columns: [{ name: '错误', kind: 'text' }], rows: [[error.message]] }) }
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
if (!results.length) throw new UserError('nl2sql/result-expired', '没有可导出的结果(可能已过期),请重新查询')
|
|
399
|
+
const sheets = results.map((r, i) => ({
|
|
400
|
+
name: (r.title || `结果${i + 1}`).slice(0, 31),
|
|
401
|
+
title: r.title || undefined,
|
|
402
|
+
note: [r.datasourceName, r.rowCount !== undefined ? `${r.rowCount} 行${r.truncated ? '(已截断)' : ''}` : '', r.filters?.length ? `筛选:${r.filters.map(f => f.description).join(';')}` : '', r.createdAt ? new Date(r.createdAt).toLocaleString('zh-CN', { hour12: false }) : ''].filter(Boolean).join(' · ') || undefined,
|
|
403
|
+
columns: r.columns,
|
|
404
|
+
rows: r.rows,
|
|
405
|
+
}))
|
|
406
|
+
if (includeSql) sheets.push({ name: 'SQL', columns: [{ name: '结果', kind: 'text' }, { name: '数据源', kind: 'text' }, { name: 'SQL', kind: 'text' }], rows: results.map((r, i) => [r.title || `结果${i + 1}`, r.datasourceName ?? '', r.sql ?? '']) })
|
|
407
|
+
const buf = writeXlsx(sheets, { creator: 'DeepSeek Harness 数据问答' })
|
|
408
|
+
const file = `${(name || results[0].title || '数据问答').replace(/[\\/:*?"<>|]/g, '_').slice(0, 60)}-${new Date().toISOString().slice(0, 10)}.xlsx`
|
|
409
|
+
return { fileName: file, mediaType: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', base64: buf.toString('base64'), bytes: buf.length, sheets: sheets.length }
|
|
410
|
+
},
|
|
411
|
+
}
|