@hedeqiang/dsh-nl2sql 0.0.0-stage → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/LICENSE +21 -0
- package/README.en.md +62 -0
- package/README.md +88 -2
- package/analysis.js +888 -0
- package/cordis.patch.yml +68 -0
- package/drivers.js +694 -0
- package/filters.js +237 -0
- package/icon.svg +13 -0
- package/im.js +511 -0
- package/index.js +650 -0
- package/lark-bot.js +606 -0
- package/lib/client.echarts.js +57 -0
- package/lib/client.js +4845 -0
- package/locale/en.json +1 -0
- package/locale/zh.json +1 -0
- package/package.json +90 -4
- package/policy.js +362 -0
- package/service/access.js +145 -0
- package/service/admin.js +539 -0
- package/service/alerts.js +233 -0
- package/service/dashboards.js +359 -0
- package/service/datasources.js +392 -0
- package/service/evaluation.js +121 -0
- package/service/lark-actions.js +223 -0
- package/service/lark.js +250 -0
- package/service/push.js +314 -0
- package/service/query.js +411 -0
- package/service/schema.js +507 -0
- package/service/shared.js +424 -0
- package/service/suggestions.js +772 -0
- package/service/uploads.js +258 -0
- package/sql-guard.js +222 -0
- package/sqlite-driver.js +677 -0
- package/store.js +225 -0
- package/xlsx.js +1001 -0
package/analysis.js
ADDED
|
@@ -0,0 +1,888 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pure analysis helpers for the NL2SQL plugin: SELECT parsing, drill-down,
|
|
3
|
+
* metric SQL, attribution, alert evaluation and result comparison.
|
|
4
|
+
*
|
|
5
|
+
* Every export is synchronous and side-effect free. SQL text produced here is
|
|
6
|
+
* still subject to the regular read-only admission (sql-guard.js) before it
|
|
7
|
+
* runs; these builders only assemble statements from already-admitted SQL,
|
|
8
|
+
* quoted identifiers and escaped literals.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { timeBounds, TIME_RANGES } from './filters.js'
|
|
12
|
+
|
|
13
|
+
// ------------------------------------------------------------ quoting
|
|
14
|
+
|
|
15
|
+
/** Quote one identifier (no dot splitting). */
|
|
16
|
+
export function quoteIdent(name, dialect) {
|
|
17
|
+
const s = String(name)
|
|
18
|
+
if (dialect === 'mysql') return `\`${s.replace(/`/g, '``')}\``
|
|
19
|
+
return `"${s.replace(/"/g, '""')}"`
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Quote a possibly dotted name: mysql/postgres/sqlite quote each part,
|
|
24
|
+
* Elasticsearch quotes the whole name (dots are part of index/field names).
|
|
25
|
+
*/
|
|
26
|
+
export function quoteName(name, dialect) {
|
|
27
|
+
const s = String(name)
|
|
28
|
+
if (dialect === 'elasticsearch') return quoteIdent(s, dialect)
|
|
29
|
+
return s.split('.').map(p => quoteIdent(p, dialect)).join('.')
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Single-quoted string literal; MySQL additionally doubles backslashes. */
|
|
33
|
+
export function quoteString(value, dialect) {
|
|
34
|
+
let s = String(value)
|
|
35
|
+
if (dialect === 'mysql') s = s.replace(/\\/g, '\\\\')
|
|
36
|
+
return `'${s.replace(/'/g, "''")}'`
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const pad = (n, w = 2) => String(n).padStart(w, '0')
|
|
40
|
+
|
|
41
|
+
/** Local-time 'YYYY-MM-DD' for midnight dates, else 'YYYY-MM-DD HH:MM:SS'. */
|
|
42
|
+
function formatDate(d, { full = false } = {}) {
|
|
43
|
+
const date = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}`
|
|
44
|
+
const time = `${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`
|
|
45
|
+
if (!full && time === '00:00:00' && d.getMilliseconds() === 0) return date
|
|
46
|
+
return `${date} ${time}`
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const STRING_EXPR = /^\s*(DATE_FORMAT|TO_CHAR|STRFTIME|CONCAT|CONCAT_WS|LPAD|RPAD|SUBSTR|SUBSTRING|LEFT|RIGHT|UPPER|LOWER|TRIM|LTRIM|RTRIM|FORMAT|REPLACE|GROUP_CONCAT|STRING_AGG|MONTHNAME|DAYNAME)\s*\(|::\s*(text|varchar|char)\b|\bAS\s+(CHAR|VARCHAR|TEXT|NCHAR)\b/i
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* SQL literal for a value compared against `expr`. Finite numbers stay bare
|
|
53
|
+
* unless the expression produces a string; Dates are formatted in local time;
|
|
54
|
+
* everything else is a quoted string. null/undefined → 'NULL'.
|
|
55
|
+
*/
|
|
56
|
+
export function literalFor(value, dialect, expr = '') {
|
|
57
|
+
if (value === null || value === undefined) return 'NULL'
|
|
58
|
+
if (typeof value === 'boolean') return value ? 'TRUE' : 'FALSE'
|
|
59
|
+
if (typeof value === 'bigint') return STRING_EXPR.test(expr) ? quoteString(String(value), dialect) : String(value)
|
|
60
|
+
if (typeof value === 'number') {
|
|
61
|
+
if (Number.isFinite(value) && !STRING_EXPR.test(expr)) return String(value)
|
|
62
|
+
return quoteString(String(value), dialect)
|
|
63
|
+
}
|
|
64
|
+
if (value instanceof Date && !Number.isNaN(value.getTime())) return quoteString(formatDate(value), dialect)
|
|
65
|
+
return quoteString(String(value), dialect)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// ------------------------------------------------------------ tokenizer
|
|
69
|
+
|
|
70
|
+
const WORD_START = /[\p{L}_]/u
|
|
71
|
+
const WORD_CHAR = /[\p{L}\p{N}_$]/u
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Tokenize SQL (comments dropped). Each token: { kind, text, upper?, start, end, depth }.
|
|
75
|
+
* kinds: word, ident (" or ` quoted), string ('…'), number, open, close, comma, op.
|
|
76
|
+
* `depth` is the parenthesis depth the token sits at (open/close carry the outer depth).
|
|
77
|
+
*/
|
|
78
|
+
function tokenize(sql, { dialect } = {}) {
|
|
79
|
+
const tokens = []
|
|
80
|
+
let depth = 0
|
|
81
|
+
const backslash = dialect !== 'postgres' && dialect !== 'sqlite' && dialect !== 'elasticsearch'
|
|
82
|
+
for (let i = 0; i < sql.length;) {
|
|
83
|
+
const c = sql[i]
|
|
84
|
+
const n = sql[i + 1]
|
|
85
|
+
if (/\s/.test(c)) { i++; continue }
|
|
86
|
+
if ((c === '-' && n === '-') || (c === '#' && dialect !== 'postgres')) { while (i < sql.length && sql[i] !== '\n') i++; continue }
|
|
87
|
+
if (c === '/' && n === '*') { const e = sql.indexOf('*/', i + 2); i = e < 0 ? sql.length : e + 2; continue }
|
|
88
|
+
if (c === '\'' || c === '"' || c === '`') {
|
|
89
|
+
const start = i
|
|
90
|
+
i++
|
|
91
|
+
while (i < sql.length) {
|
|
92
|
+
if (sql[i] === '\\' && c === '\'' && backslash) { i += 2; continue }
|
|
93
|
+
if (sql[i] === c) { if (sql[i + 1] === c) { i += 2; continue } break }
|
|
94
|
+
i++
|
|
95
|
+
}
|
|
96
|
+
if (i >= sql.length) return undefined // unterminated
|
|
97
|
+
i++
|
|
98
|
+
tokens.push({ kind: c === '\'' ? 'string' : 'ident', text: sql.slice(start, i), start, end: i, depth })
|
|
99
|
+
continue
|
|
100
|
+
}
|
|
101
|
+
if (c === '(') { tokens.push({ kind: 'open', text: c, start: i, end: i + 1, depth }); depth++; i++; continue }
|
|
102
|
+
if (c === ')') {
|
|
103
|
+
depth--
|
|
104
|
+
if (depth < 0) return undefined
|
|
105
|
+
tokens.push({ kind: 'close', text: c, start: i, end: i + 1, depth }); i++; continue
|
|
106
|
+
}
|
|
107
|
+
if (c === ',') { tokens.push({ kind: 'comma', text: c, start: i, end: i + 1, depth }); i++; continue }
|
|
108
|
+
if (/[0-9]/.test(c) || (c === '.' && /[0-9]/.test(n ?? ''))) {
|
|
109
|
+
const m = /^(\d*\.?\d+(?:[eE][+-]?\d+)?|\d+\.)/.exec(sql.slice(i))
|
|
110
|
+
const len = m ? m[0].length : 1
|
|
111
|
+
tokens.push({ kind: 'number', text: sql.slice(i, i + len), start: i, end: i + len, depth }); i += len; continue
|
|
112
|
+
}
|
|
113
|
+
if (WORD_START.test(c)) {
|
|
114
|
+
let j = i
|
|
115
|
+
while (j < sql.length) {
|
|
116
|
+
if (WORD_CHAR.test(sql[j])) { j++; continue }
|
|
117
|
+
if (sql[j] === '.' && sql[j + 1] && (WORD_START.test(sql[j + 1]) || sql[j + 1] === '*' || /[0-9]/.test(sql[j + 1]))) { j++; if (sql[j] === '*') { j++; break } continue }
|
|
118
|
+
break
|
|
119
|
+
}
|
|
120
|
+
const text = sql.slice(i, j)
|
|
121
|
+
tokens.push({ kind: 'word', text, upper: text.toUpperCase(), start: i, end: j, depth }); i = j; continue
|
|
122
|
+
}
|
|
123
|
+
tokens.push({ kind: 'op', text: c, start: i, end: i + 1, depth }); i++
|
|
124
|
+
}
|
|
125
|
+
if (depth !== 0) return undefined
|
|
126
|
+
return tokens
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function unquoteIdent(text) {
|
|
130
|
+
if (/^`.*`$/s.test(text)) return text.slice(1, -1).replace(/``/g, '`')
|
|
131
|
+
if (/^".*"$/s.test(text)) return text.slice(1, -1).replace(/""/g, '"')
|
|
132
|
+
if (/^'.*'$/s.test(text)) return text.slice(1, -1).replace(/''/g, '\'').replace(/\\(.)/g, '$1')
|
|
133
|
+
return text
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// ------------------------------------------------------------ parseSelect
|
|
137
|
+
|
|
138
|
+
const AGG_FUNCS = new Set(['COUNT', 'SUM', 'AVG', 'MIN', 'MAX', 'STDDEV', 'STDDEV_POP', 'STDDEV_SAMP', 'STD', 'VARIANCE', 'VAR_POP', 'VAR_SAMP',
|
|
139
|
+
'MEDIAN', 'GROUP_CONCAT', 'STRING_AGG', 'ARRAY_AGG', 'JSON_AGG', 'JSONB_AGG', 'JSON_ARRAYAGG', 'JSON_OBJECTAGG', 'BOOL_AND', 'BOOL_OR', 'BIT_AND',
|
|
140
|
+
'BIT_OR', 'BIT_XOR', 'EVERY', 'COUNT_BIG', 'APPROX_COUNT_DISTINCT', 'KURTOSIS', 'SKEWNESS', 'MAD', 'SUM_OF_SQUARES', 'FIRST', 'LAST', 'TOTAL',
|
|
141
|
+
'CORR', 'COVAR_POP', 'COVAR_SAMP', 'REGR_SLOPE', 'REGR_INTERCEPT', 'REGR_COUNT', 'MODE'])
|
|
142
|
+
|
|
143
|
+
const isAggName = (word) => {
|
|
144
|
+
const name = word.split('.').pop().toUpperCase()
|
|
145
|
+
return AGG_FUNCS.has(name) || name.startsWith('PERCENTILE')
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Aggregate call in a token run (outside nested SELECTs, not a bare window function). */
|
|
149
|
+
function containsAggregate(tokens) {
|
|
150
|
+
for (let k = 0; k < tokens.length; k++) {
|
|
151
|
+
const t = tokens[k]
|
|
152
|
+
if (t.kind === 'open' && tokens[k + 1]?.kind === 'word' && /^(SELECT|WITH)$/.test(tokens[k + 1].upper)) {
|
|
153
|
+
// Skip the nested subquery entirely.
|
|
154
|
+
const close = tokens.findIndex((x, j) => j > k && x.kind === 'close' && x.depth === t.depth)
|
|
155
|
+
if (close < 0) return false
|
|
156
|
+
k = close
|
|
157
|
+
continue
|
|
158
|
+
}
|
|
159
|
+
if (t.kind === 'word' && tokens[k + 1]?.kind === 'open' && isAggName(t.text)) {
|
|
160
|
+
const close = tokens.findIndex((x, j) => j > k + 1 && x.kind === 'close' && x.depth === tokens[k + 1].depth)
|
|
161
|
+
let after = close + 1
|
|
162
|
+
// FILTER (WHERE …) / WITHIN GROUP (…) belong to the aggregate.
|
|
163
|
+
while (tokens[after]?.kind === 'word' && /^(FILTER|WITHIN)$/.test(tokens[after].upper)) {
|
|
164
|
+
let o = after + 1
|
|
165
|
+
while (tokens[o] && tokens[o].kind !== 'open') o++
|
|
166
|
+
const c2 = tokens.findIndex((x, j) => j > o && x.kind === 'close' && x.depth === tokens[o]?.depth)
|
|
167
|
+
after = c2 < 0 ? tokens.length : c2 + 1
|
|
168
|
+
}
|
|
169
|
+
if (tokens[after]?.kind === 'word' && tokens[after].upper === 'OVER') continue // window function, inner tokens still scanned
|
|
170
|
+
return true
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
return false
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const PREV_BLOCK = new Set(['AS', 'AND', 'OR', 'NOT', 'IS', 'IN', 'LIKE', 'ILIKE', 'RLIKE', 'REGEXP', 'BETWEEN', 'THEN', 'ELSE', 'WHEN', 'CASE',
|
|
177
|
+
'DISTINCT', 'ALL', 'INTERVAL', 'COLLATE', 'BINARY', 'ESCAPE', 'SIMILAR', 'TO', 'DIV', 'MOD', 'XOR', 'OVER', 'ANY', 'SOME', 'EXISTS', 'SELECT',
|
|
178
|
+
'AT', 'ZONE', 'DATE', 'TIMESTAMP', 'TIME', 'FROM', 'BY'])
|
|
179
|
+
const LAST_BLOCK = new Set(['END', 'NULL', 'TRUE', 'FALSE', 'UNKNOWN', 'CURRENT_DATE', 'CURRENT_TIMESTAMP', 'CURRENT_TIME', 'LOCALTIME',
|
|
180
|
+
'LOCALTIMESTAMP', 'CURRENT_USER', 'SESSION_USER', 'ASC', 'DESC', 'AND', 'OR', 'NOT'])
|
|
181
|
+
|
|
182
|
+
function splitSelectItem(sql, toks) {
|
|
183
|
+
const text = sql.slice(toks[0].start, toks[toks.length - 1].end).trim()
|
|
184
|
+
const top = toks.filter(t => t.depth === toks[0].depth)
|
|
185
|
+
const last = toks[toks.length - 1]
|
|
186
|
+
const prev = toks[toks.length - 2]
|
|
187
|
+
let alias
|
|
188
|
+
let exprEnd = last.end
|
|
189
|
+
if (toks.length >= 3 && prev.kind === 'word' && prev.upper === 'AS' && prev.depth === last.depth && /^(word|ident|string)$/.test(last.kind)) {
|
|
190
|
+
alias = last.kind === 'word' ? last.text : unquoteIdent(last.text)
|
|
191
|
+
exprEnd = prev.start
|
|
192
|
+
} else if (toks.length >= 2 && top.length >= 2 && /^(word|ident|string)$/.test(last.kind) && last.depth === toks[0].depth &&
|
|
193
|
+
!(last.kind === 'word' && (LAST_BLOCK.has(last.upper) || last.text.includes('.'))) &&
|
|
194
|
+
(prev.kind === 'close' || prev.kind === 'ident' || prev.kind === 'string' || prev.kind === 'number' ||
|
|
195
|
+
(prev.kind === 'word' && !PREV_BLOCK.has(prev.upper))) &&
|
|
196
|
+
!(toks[toks.length - 3]?.kind === 'word' && toks[toks.length - 3].upper === 'INTERVAL') &&
|
|
197
|
+
// A trailing string is an alias only after a plain column/call, never a typed literal (DATE '…', _utf8'…').
|
|
198
|
+
!(last.kind === 'string' && (prev.kind === 'string' || prev.kind === 'number' || (prev.kind === 'word' && prev.text.startsWith('_'))))) {
|
|
199
|
+
alias = last.kind === 'word' ? last.text : unquoteIdent(last.text)
|
|
200
|
+
exprEnd = prev.end
|
|
201
|
+
}
|
|
202
|
+
const expr = sql.slice(toks[0].start, exprEnd).trim()
|
|
203
|
+
const exprToks = toks.filter(t => t.end <= exprEnd)
|
|
204
|
+
return { expr, alias, isAggregate: containsAggregate(exprToks), text }
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Split the outermost SELECT into clause texts.
|
|
209
|
+
* @param {string} sql
|
|
210
|
+
* @param {{ dialect?: string }} [opts]
|
|
211
|
+
* @returns {undefined | { prefix: string, select: { expr: string, alias?: string, isAggregate: boolean }[], distinct: boolean,
|
|
212
|
+
* from: string, where?: string, groupBy?: string, having?: string, orderBy?: string, limit?: string, tail?: string }}
|
|
213
|
+
*/
|
|
214
|
+
export function parseSelect(sql, opts = {}) {
|
|
215
|
+
if (typeof sql !== 'string') return undefined
|
|
216
|
+
let src = sql.trim()
|
|
217
|
+
while (src.endsWith(';')) src = src.slice(0, -1).trimEnd()
|
|
218
|
+
const tokens = tokenize(src, opts)
|
|
219
|
+
if (!tokens || !tokens.length) return undefined
|
|
220
|
+
const first = tokens[0]
|
|
221
|
+
if (first.kind !== 'word' || !/^(SELECT|WITH)$/.test(first.upper)) return undefined
|
|
222
|
+
const selIdx = tokens.findIndex(t => t.depth === 0 && t.kind === 'word' && t.upper === 'SELECT')
|
|
223
|
+
if (selIdx < 0) return undefined
|
|
224
|
+
const prefix = src.slice(0, tokens[selIdx].start).trim()
|
|
225
|
+
const top = []
|
|
226
|
+
for (let k = selIdx; k < tokens.length; k++) if (tokens[k].depth === 0) top.push(k)
|
|
227
|
+
if (top.some(k => tokens[k].kind === 'word' && /^(UNION|INTERSECT|EXCEPT|MINUS)$/.test(tokens[k].upper))) return undefined
|
|
228
|
+
// Locate clause keywords at depth 0.
|
|
229
|
+
const clauses = []
|
|
230
|
+
for (const k of top) {
|
|
231
|
+
const t = tokens[k]
|
|
232
|
+
if (t.kind !== 'word') continue
|
|
233
|
+
const next = tokens[k + 1]
|
|
234
|
+
let key
|
|
235
|
+
if (t.upper === 'FROM') {
|
|
236
|
+
// a IS [NOT] DISTINCT FROM b
|
|
237
|
+
if (tokens[k - 1]?.upper === 'DISTINCT' && /^(IS|NOT)$/.test(tokens[k - 2]?.upper ?? '')) continue
|
|
238
|
+
if (clauses.some(c => c.key === 'from')) continue
|
|
239
|
+
key = 'from'
|
|
240
|
+
} else if (!clauses.length) continue
|
|
241
|
+
else if (t.upper === 'WHERE') key = 'where'
|
|
242
|
+
else if (t.upper === 'GROUP' && next?.upper === 'BY') key = 'groupBy'
|
|
243
|
+
else if (t.upper === 'HAVING') key = 'having'
|
|
244
|
+
else if (t.upper === 'ORDER' && next?.upper === 'BY') key = 'orderBy'
|
|
245
|
+
else if (t.upper === 'LIMIT') key = 'limit'
|
|
246
|
+
else if (/^(WINDOW|QUALIFY|OFFSET|FETCH|FOR|LOCK)$/.test(t.upper) && !clauses.some(c => c.key === 'tail')) {
|
|
247
|
+
if (t.upper === 'OFFSET' && clauses.at(-1)?.key === 'limit') continue
|
|
248
|
+
key = 'tail'
|
|
249
|
+
} else continue
|
|
250
|
+
if (key !== 'tail' && clauses.some(c => c.key === key)) return undefined
|
|
251
|
+
const kwEnd = (key === 'groupBy' || key === 'orderBy') ? next.end : t.end
|
|
252
|
+
clauses.push({ key, start: t.start, bodyStart: key === 'tail' ? t.start : kwEnd })
|
|
253
|
+
}
|
|
254
|
+
if (!clauses.length || clauses[0].key !== 'from') return undefined
|
|
255
|
+
const out = { prefix, select: [], distinct: false, from: '', where: undefined, groupBy: undefined, having: undefined, orderBy: undefined, limit: undefined }
|
|
256
|
+
clauses.forEach((c, i) => {
|
|
257
|
+
const end = i + 1 < clauses.length ? clauses[i + 1].start : src.length
|
|
258
|
+
out[c.key] = src.slice(c.bodyStart, end).trim()
|
|
259
|
+
})
|
|
260
|
+
if (!out.from) return undefined
|
|
261
|
+
// Select list.
|
|
262
|
+
let k = selIdx + 1
|
|
263
|
+
while (tokens[k]?.kind === 'word' && /^(DISTINCT|ALL|DISTINCTROW|SQL_CALC_FOUND_ROWS|SQL_NO_CACHE|SQL_CACHE|HIGH_PRIORITY|STRAIGHT_JOIN|SQL_SMALL_RESULT|SQL_BIG_RESULT|SQL_BUFFER_RESULT)$/.test(tokens[k].upper)) {
|
|
264
|
+
if (/^DISTINCT/.test(tokens[k].upper)) {
|
|
265
|
+
// PostgreSQL DISTINCT ON (…) stays part of the list prefix.
|
|
266
|
+
if (tokens[k + 1]?.upper === 'ON') return undefined
|
|
267
|
+
out.distinct = true
|
|
268
|
+
}
|
|
269
|
+
k++
|
|
270
|
+
}
|
|
271
|
+
const fromTok = tokens.findIndex(t => t.start === clauses[0].start)
|
|
272
|
+
const items = []
|
|
273
|
+
let cur = []
|
|
274
|
+
for (; k < fromTok; k++) {
|
|
275
|
+
const t = tokens[k]
|
|
276
|
+
if (t.kind === 'comma' && t.depth === 0) { items.push(cur); cur = []; continue }
|
|
277
|
+
cur.push(t)
|
|
278
|
+
}
|
|
279
|
+
items.push(cur)
|
|
280
|
+
if (items.some(it => !it.length)) return undefined
|
|
281
|
+
out.select = items.map(it => {
|
|
282
|
+
const { expr, alias, isAggregate } = splitSelectItem(src, it)
|
|
283
|
+
return alias === undefined ? { expr, isAggregate } : { expr, alias, isAggregate }
|
|
284
|
+
})
|
|
285
|
+
return out
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
const IDENT_PATH = /^\s*(?:(?:`(?:[^`]|``)*`|"(?:[^"]|"")*"|[\p{L}_][\p{L}\p{N}_$]*)\s*\.\s*)*(`(?:[^`]|``)*`|"(?:[^"]|"")*"|[\p{L}_][\p{L}\p{N}_$]*)\s*$/u
|
|
289
|
+
|
|
290
|
+
/** Unquoted last part of a bare column reference, or undefined if expr is not one. */
|
|
291
|
+
function bareColumnName(expr) {
|
|
292
|
+
const m = IDENT_PATH.exec(expr)
|
|
293
|
+
return m ? unquoteIdent(m[1]) : undefined
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* The select expression producing result column `name`.
|
|
298
|
+
* @returns {string | undefined}
|
|
299
|
+
*/
|
|
300
|
+
export function aliasExpr(parsed, name) {
|
|
301
|
+
const item = findItem(parsed, name)
|
|
302
|
+
return item?.expr
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function findItem(parsed, name) {
|
|
306
|
+
if (!parsed || name === undefined || name === null) return undefined
|
|
307
|
+
const n = String(name)
|
|
308
|
+
const lower = n.toLowerCase()
|
|
309
|
+
const sel = parsed.select ?? []
|
|
310
|
+
return sel.find(s => s.alias === n) ??
|
|
311
|
+
sel.find(s => s.alias !== undefined && s.alias.toLowerCase() === lower) ??
|
|
312
|
+
sel.find(s => s.alias === undefined && bareColumnName(s.expr)?.toLowerCase() === lower) ??
|
|
313
|
+
sel.find(s => s.alias === undefined && s.expr.replace(/\s+/g, '').toLowerCase() === n.replace(/\s+/g, '').toLowerCase())
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// ------------------------------------------------------------ drill-down
|
|
317
|
+
|
|
318
|
+
function needsParens(expr) {
|
|
319
|
+
if (IDENT_PATH.test(expr)) return false
|
|
320
|
+
const toks = tokenize(expr) ?? []
|
|
321
|
+
// A single function call: word ( … ) spanning the whole expression.
|
|
322
|
+
if (toks.length >= 3 && toks[0].kind === 'word' && toks[1].kind === 'open' && toks.at(-1).kind === 'close' &&
|
|
323
|
+
toks.slice(2, -1).every(t => t.depth >= 1)) return false
|
|
324
|
+
return true
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function equalsPredicate(expr, value, dialect) {
|
|
328
|
+
const lhs = needsParens(expr) ? `(${expr})` : expr
|
|
329
|
+
if (value === null || value === undefined) return `${lhs} IS NULL`
|
|
330
|
+
return `${lhs} = ${literalFor(value, dialect, expr)}`
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function joinWhere(where, preds) {
|
|
334
|
+
const parts = []
|
|
335
|
+
if (where && where.trim()) parts.push(`(${where.trim()})`)
|
|
336
|
+
parts.push(...preds.filter(Boolean))
|
|
337
|
+
return parts.length ? ` WHERE ${parts.join(' AND ')}` : ''
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
const withPrefix = (prefix, body) => (prefix ? `${prefix} ${body}` : body)
|
|
341
|
+
const clampLimit = (n, def, max = 100000) => {
|
|
342
|
+
const v = Math.floor(Number(n))
|
|
343
|
+
return Number.isFinite(v) && v > 0 ? Math.min(v, max) : def
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function drillContext({ sql, dialect, xColumn, xValue }) {
|
|
347
|
+
const parsed = parseSelect(sql, { dialect })
|
|
348
|
+
if (!parsed) return { error: '无法解析该查询:仅支持单个 SELECT(可带 WITH),不支持 UNION 或缺少 FROM 的语句' }
|
|
349
|
+
if (xColumn === undefined || xColumn === null || xColumn === '') return { parsed, pred: undefined }
|
|
350
|
+
const item = findItem(parsed, xColumn)
|
|
351
|
+
if (!item) return { error: `找不到结果列「${xColumn}」对应的查询表达式` }
|
|
352
|
+
if (item.isAggregate) return { error: `结果列「${xColumn}」是聚合指标,无法按它下钻` }
|
|
353
|
+
if (/^\*$|\.\*$/.test(item.expr.trim())) return { error: `结果列「${xColumn}」无法定位到具体表达式` }
|
|
354
|
+
return { parsed, pred: equalsPredicate(item.expr, xValue, dialect) }
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Detail rows behind one chart point.
|
|
359
|
+
* @returns {{ sql: string } | { error: string }}
|
|
360
|
+
*/
|
|
361
|
+
export function buildDetailSql({ sql, dialect, xColumn, xValue, limit = 100 } = {}) {
|
|
362
|
+
const ctx = drillContext({ sql, dialect, xColumn, xValue })
|
|
363
|
+
if (ctx.error) return { error: ctx.error }
|
|
364
|
+
const { parsed, pred } = ctx
|
|
365
|
+
return { sql: withPrefix(parsed.prefix, `SELECT * FROM ${parsed.from}${joinWhere(parsed.where, [pred])} LIMIT ${clampLimit(limit, 100)}`) }
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function dimensionLabelOf(dimension) {
|
|
369
|
+
const name = bareColumnName(dimension)
|
|
370
|
+
return name ?? dimension.trim()
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/**
|
|
374
|
+
* Break the aggregates of a query (optionally at one x value) down by another dimension.
|
|
375
|
+
* @returns {{ sql: string, label: string, measures: string[] } | { error: string }}
|
|
376
|
+
*/
|
|
377
|
+
export function buildBreakdownSql({ sql, dialect, xColumn, xValue, dimension, dimensionLabel, limit = 20 } = {}) {
|
|
378
|
+
if (typeof dimension !== 'string' || !dimension.trim()) return { error: '缺少拆解维度' }
|
|
379
|
+
const ctx = drillContext({ sql, dialect, xColumn, xValue })
|
|
380
|
+
if (ctx.error) return { error: ctx.error }
|
|
381
|
+
const { parsed, pred } = ctx
|
|
382
|
+
const aggs = parsed.select.filter(s => s.isAggregate)
|
|
383
|
+
if (!aggs.length) return { error: '该查询没有聚合指标(COUNT/SUM/AVG 等),无法按维度拆解' }
|
|
384
|
+
const dim = dimension.trim()
|
|
385
|
+
let label = (dimensionLabel ?? dimensionLabelOf(dim)) || '维度'
|
|
386
|
+
const measures = aggs.map(a => a.alias ?? a.expr)
|
|
387
|
+
if (measures.some(m => m.toLowerCase() === label.toLowerCase())) label = `${label}(维度)`
|
|
388
|
+
const cols = [`${dim} AS ${quoteIdent(label, dialect)}`, ...aggs.map((a, i) => `${a.expr} AS ${quoteIdent(measures[i], dialect)}`)]
|
|
389
|
+
const body = `SELECT ${cols.join(', ')} FROM ${parsed.from}${joinWhere(parsed.where, [pred])} GROUP BY ${dim} ORDER BY ${quoteIdent(measures[0], dialect)} DESC LIMIT ${clampLimit(limit, 20)}`
|
|
390
|
+
return { sql: withPrefix(parsed.prefix, body), label, measures }
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
// ------------------------------------------------------------ metrics
|
|
394
|
+
|
|
395
|
+
export const GRAINS = ['hour', 'day', 'week', 'month', 'year']
|
|
396
|
+
|
|
397
|
+
/** Time bucket expression for a quoted column. */
|
|
398
|
+
export function timeBucket(col, grain, dialect) {
|
|
399
|
+
if (!GRAINS.includes(grain)) return undefined
|
|
400
|
+
switch (dialect) {
|
|
401
|
+
case 'postgres':
|
|
402
|
+
return {
|
|
403
|
+
day: `(${col})::date`,
|
|
404
|
+
week: `to_char(${col}, 'IYYY-"W"IW')`,
|
|
405
|
+
month: `to_char(${col}, 'YYYY-MM')`,
|
|
406
|
+
hour: `to_char(${col}, 'YYYY-MM-DD HH24:00')`,
|
|
407
|
+
year: `EXTRACT(YEAR FROM ${col})::int`,
|
|
408
|
+
}[grain]
|
|
409
|
+
case 'sqlite':
|
|
410
|
+
return {
|
|
411
|
+
day: `date(${col})`,
|
|
412
|
+
week: `strftime('%Y-W%W', ${col})`,
|
|
413
|
+
month: `strftime('%Y-%m', ${col})`,
|
|
414
|
+
hour: `strftime('%Y-%m-%d %H:00', ${col})`,
|
|
415
|
+
year: `strftime('%Y', ${col})`,
|
|
416
|
+
}[grain]
|
|
417
|
+
case 'elasticsearch':
|
|
418
|
+
return `DATE_TRUNC('${grain}', ${col})`
|
|
419
|
+
default:
|
|
420
|
+
return {
|
|
421
|
+
day: `DATE(${col})`,
|
|
422
|
+
week: `DATE_FORMAT(${col}, '%x-W%v')`,
|
|
423
|
+
month: `DATE_FORMAT(${col}, '%Y-%m')`,
|
|
424
|
+
hour: `DATE_FORMAT(${col}, '%Y-%m-%d %H:00')`,
|
|
425
|
+
year: `YEAR(${col})`,
|
|
426
|
+
}[grain]
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
const FILTER_OPS = new Set(['=', '!=', '<>', '>', '>=', '<', '<=', 'in', 'not in', 'like'])
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* Build the SQL for a semantic metric.
|
|
434
|
+
* @returns {{ sql: string, columns: { time?: string, dimensions: string[], value: string } } | { error: string }}
|
|
435
|
+
*/
|
|
436
|
+
export function buildMetricSql({ metric, dialect = 'mysql', grain, dimensions = [], range, from, to, filters = [], limit = 1000, now } = {}) {
|
|
437
|
+
if (!metric || typeof metric !== 'object') return { error: '缺少指标定义' }
|
|
438
|
+
const expression = typeof metric.expression === 'string' ? metric.expression.trim() : ''
|
|
439
|
+
const table = typeof metric.table === 'string' ? metric.table.trim() : ''
|
|
440
|
+
if (!expression) return { error: `指标「${metric.name ?? ''}」缺少计算表达式` }
|
|
441
|
+
if (!table) return { error: `指标「${metric.name ?? ''}」缺少数据表` }
|
|
442
|
+
const name = String(metric.name || 'value')
|
|
443
|
+
if (grain !== undefined && grain !== null && grain !== '' && !GRAINS.includes(grain)) return { error: `不支持的时间粒度「${grain}」` }
|
|
444
|
+
const hasGrain = !!grain
|
|
445
|
+
const timeCol = metric.timeColumn ? quoteName(metric.timeColumn, dialect) : undefined
|
|
446
|
+
if (hasGrain && !timeCol) return { error: `指标「${name}」未配置时间列,无法按${grain}聚合` }
|
|
447
|
+
// Time range.
|
|
448
|
+
let bounds
|
|
449
|
+
if (from || to) bounds = { from, to }
|
|
450
|
+
else if (range) {
|
|
451
|
+
if (!TIME_RANGES.includes(range)) return { error: `不支持的时间范围「${range}」` }
|
|
452
|
+
bounds = timeBounds({ range }, now ?? new Date())
|
|
453
|
+
}
|
|
454
|
+
const where = []
|
|
455
|
+
if (metric.filter && String(metric.filter).trim()) where.push(`(${String(metric.filter).trim()})`)
|
|
456
|
+
if (bounds && (bounds.from || bounds.to)) {
|
|
457
|
+
if (!timeCol) return { error: `指标「${name}」未配置时间列,无法按时间范围过滤` }
|
|
458
|
+
if (bounds.from) where.push(`${timeCol} >= ${quoteString(bounds.from, dialect)}`)
|
|
459
|
+
if (bounds.to) where.push(`${timeCol} < ${quoteString(bounds.to, dialect)}`)
|
|
460
|
+
}
|
|
461
|
+
for (const f of Array.isArray(filters) ? filters : []) {
|
|
462
|
+
if (!f || typeof f.column !== 'string' || !f.column.trim()) return { error: '过滤条件缺少列名' }
|
|
463
|
+
const op = String(f.op ?? '=').toLowerCase()
|
|
464
|
+
if (!FILTER_OPS.has(op)) return { error: `不支持的过滤运算符「${f.op}」` }
|
|
465
|
+
const col = quoteName(f.column.trim(), dialect)
|
|
466
|
+
if (op === 'in' || op === 'not in') {
|
|
467
|
+
const values = Array.isArray(f.values) ? f.values : (f.value !== undefined ? [f.value] : [])
|
|
468
|
+
if (!values.length) return { error: `过滤条件「${f.column}」缺少取值` }
|
|
469
|
+
where.push(`${col} ${op.toUpperCase()} (${values.map(v => literalFor(v, dialect)).join(', ')})`)
|
|
470
|
+
} else if (f.value === null || f.value === undefined) {
|
|
471
|
+
if (op === '=') where.push(`${col} IS NULL`)
|
|
472
|
+
else if (op === '!=' || op === '<>') where.push(`${col} IS NOT NULL`)
|
|
473
|
+
else return { error: `过滤条件「${f.column}」缺少取值` }
|
|
474
|
+
} else {
|
|
475
|
+
where.push(`${col} ${op === 'like' ? 'LIKE' : op} ${literalFor(f.value, dialect)}`)
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
const dims = (Array.isArray(dimensions) ? dimensions : []).filter(d => typeof d === 'string' && d.trim()).map(d => d.trim())
|
|
479
|
+
const select = []
|
|
480
|
+
const group = []
|
|
481
|
+
const timeLabel = '时间'
|
|
482
|
+
if (hasGrain) {
|
|
483
|
+
const bucket = timeBucket(timeCol, grain, dialect)
|
|
484
|
+
select.push(`${bucket} AS ${quoteIdent(timeLabel, dialect)}`)
|
|
485
|
+
group.push(bucket)
|
|
486
|
+
}
|
|
487
|
+
for (const d of dims) {
|
|
488
|
+
const col = quoteName(d, dialect)
|
|
489
|
+
select.push(`${col} AS ${quoteIdent(d, dialect)}`)
|
|
490
|
+
group.push(col)
|
|
491
|
+
}
|
|
492
|
+
select.push(`${expression} AS ${quoteIdent(name, dialect)}`)
|
|
493
|
+
let sql = `SELECT ${select.join(', ')} FROM ${quoteName(table, dialect)}`
|
|
494
|
+
if (where.length) sql += ` WHERE ${where.join(' AND ')}`
|
|
495
|
+
if (group.length) sql += ` GROUP BY ${group.join(', ')}`
|
|
496
|
+
sql += hasGrain
|
|
497
|
+
? ` ORDER BY ${quoteIdent(timeLabel, dialect)} ASC${dims.length ? `, ${quoteIdent(name, dialect)} DESC` : ''}`
|
|
498
|
+
: (group.length ? ` ORDER BY ${quoteIdent(name, dialect)} DESC` : '')
|
|
499
|
+
sql += ` LIMIT ${clampLimit(limit, 1000)}`
|
|
500
|
+
return { sql, columns: { ...(hasGrain ? { time: timeLabel } : {}), dimensions: dims, value: name } }
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
// ------------------------------------------------------------ attribution
|
|
504
|
+
|
|
505
|
+
function periodPredicate(timeExpr, p, dialect) {
|
|
506
|
+
const lhs = needsParens(timeExpr) ? `(${timeExpr})` : timeExpr
|
|
507
|
+
if (p && typeof p === 'object' && !(p instanceof Date)) {
|
|
508
|
+
const parts = []
|
|
509
|
+
if (p.from !== undefined && p.from !== null) parts.push(`${lhs} >= ${literalFor(p.from, dialect, timeExpr)}`)
|
|
510
|
+
if (p.to !== undefined && p.to !== null) parts.push(`${lhs} < ${literalFor(p.to, dialect, timeExpr)}`)
|
|
511
|
+
return parts.length ? parts.join(' AND ') : undefined
|
|
512
|
+
}
|
|
513
|
+
return `${lhs} = ${literalFor(p, dialect, timeExpr)}`
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
const isRange = (p) => p && typeof p === 'object' && !(p instanceof Date)
|
|
517
|
+
|
|
518
|
+
/**
|
|
519
|
+
* SQL producing rows (dim, period, value) with period 'current' | 'baseline'.
|
|
520
|
+
* @returns {{ sql: string } | { error: string }}
|
|
521
|
+
*/
|
|
522
|
+
export function buildAttributionSql({ dialect = 'mysql', from, where, timeExpr, measureExpr, dimension, current, baseline, limit = 50 } = {}) {
|
|
523
|
+
if (!from || !String(from).trim()) return { error: '缺少数据来源(FROM)' }
|
|
524
|
+
if (!timeExpr || !measureExpr || !dimension) return { error: '缺少时间表达式、指标表达式或维度' }
|
|
525
|
+
if (current === undefined || baseline === undefined) return { error: '缺少对比的本期或基期' }
|
|
526
|
+
const curPred = periodPredicate(timeExpr, current, dialect)
|
|
527
|
+
const basePred = periodPredicate(timeExpr, baseline, dialect)
|
|
528
|
+
if (!curPred || !basePred) return { error: '对比时间段为空' }
|
|
529
|
+
const lhs = needsParens(timeExpr) ? `(${timeExpr})` : timeExpr
|
|
530
|
+
let periodFilter
|
|
531
|
+
if (!isRange(current) && !isRange(baseline)) periodFilter = `${lhs} IN (${literalFor(current, dialect, timeExpr)}, ${literalFor(baseline, dialect, timeExpr)})`
|
|
532
|
+
else periodFilter = `((${curPred}) OR (${basePred}))`
|
|
533
|
+
const period = `CASE WHEN ${curPred} THEN 'current' ELSE 'baseline' END`
|
|
534
|
+
const q = (s) => quoteIdent(s, dialect)
|
|
535
|
+
const sql = `SELECT ${dimension} AS ${q('dim')}, ${period} AS ${q('period')}, ${measureExpr} AS ${q('value')} FROM ${String(from).trim()}` +
|
|
536
|
+
`${joinWhere(where, [periodFilter])} GROUP BY ${dimension}, ${period} ORDER BY ${q('value')} DESC LIMIT ${clampLimit(limit, 50) * 2}`
|
|
537
|
+
return { sql }
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
const toNum = (v) => {
|
|
541
|
+
if (v === null || v === undefined || v === '') return 0
|
|
542
|
+
const n = typeof v === 'number' ? v : Number(v)
|
|
543
|
+
return Number.isFinite(n) ? n : 0
|
|
544
|
+
}
|
|
545
|
+
const ratio = (a, b) => (b === 0 ? null : a / b)
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Attribute the change of a measure across one dimension.
|
|
549
|
+
* @param {Array<{ dim, period, value } | [dim, period, value]>} rows
|
|
550
|
+
* @param {{ additive?: boolean, total?: { baseline: number, current: number }, top?: number }} [opts]
|
|
551
|
+
*/
|
|
552
|
+
export function computeAttribution(rows, { additive = true, total, top = 10 } = {}) {
|
|
553
|
+
const groups = new Map()
|
|
554
|
+
for (const r of Array.isArray(rows) ? rows : []) {
|
|
555
|
+
const [dim, period, value] = Array.isArray(r) ? r : [r.dim ?? r.value_dim ?? r.dimension, r.period, r.value]
|
|
556
|
+
const key = dim instanceof Date ? formatDate(dim) : dim === null || dim === undefined ? null : String(dim)
|
|
557
|
+
const k = key === null ? '\u0000null' : key
|
|
558
|
+
if (!groups.has(k)) groups.set(k, { value: key, baseline: 0, current: 0 })
|
|
559
|
+
const g = groups.get(k)
|
|
560
|
+
if (period === 'current') g.current += toNum(value)
|
|
561
|
+
else if (period === 'baseline') g.baseline += toNum(value)
|
|
562
|
+
}
|
|
563
|
+
const all = [...groups.values()]
|
|
564
|
+
const sumB = all.reduce((s, g) => s + g.baseline, 0)
|
|
565
|
+
const sumC = all.reduce((s, g) => s + g.current, 0)
|
|
566
|
+
const tb = total && Number.isFinite(Number(total.baseline)) ? Number(total.baseline) : sumB
|
|
567
|
+
const tc = total && Number.isFinite(Number(total.current)) ? Number(total.current) : sumC
|
|
568
|
+
const tDelta = tc - tb
|
|
569
|
+
const finish = (g) => {
|
|
570
|
+
const delta = g.current - g.baseline
|
|
571
|
+
return { value: g.value, baseline: g.baseline, current: g.current, delta, share: ratio(delta, tDelta), pct: ratio(delta, g.baseline) }
|
|
572
|
+
}
|
|
573
|
+
const sorted = all.map(finish).sort((a, b) => Math.abs(b.delta) - Math.abs(a.delta) || String(a.value).localeCompare(String(b.value)))
|
|
574
|
+
const concentration = Math.min(1, sorted.slice(0, 3).reduce((s, it) => s + Math.abs(it.share ?? 0), 0))
|
|
575
|
+
let items = sorted
|
|
576
|
+
if (sorted.length > top) {
|
|
577
|
+
const rest = sorted.slice(top)
|
|
578
|
+
const agg = rest.reduce((s, it) => ({ baseline: s.baseline + it.baseline, current: s.current + it.current }), { baseline: 0, current: 0 })
|
|
579
|
+
items = [...sorted.slice(0, top), { ...finish({ value: '其他', ...agg }), count: rest.length }]
|
|
580
|
+
}
|
|
581
|
+
return {
|
|
582
|
+
total: { baseline: tb, current: tc, delta: tDelta, pct: ratio(tDelta, tb) },
|
|
583
|
+
items,
|
|
584
|
+
concentration,
|
|
585
|
+
distinct: all.length,
|
|
586
|
+
additive: !!additive,
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/** Sort dimension attributions by explanatory power. */
|
|
591
|
+
export function rankDimensions(results) {
|
|
592
|
+
return [...(Array.isArray(results) ? results : [])].sort((a, b) => {
|
|
593
|
+
const ca = a?.attribution?.concentration ?? 0
|
|
594
|
+
const cb = b?.attribution?.concentration ?? 0
|
|
595
|
+
if (cb !== ca) return cb - ca
|
|
596
|
+
const da = a?.attribution?.distinct ?? a?.attribution?.items?.length ?? 0
|
|
597
|
+
const db = b?.attribution?.distinct ?? b?.attribution?.items?.length ?? 0
|
|
598
|
+
return da - db
|
|
599
|
+
})
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/** Whether per-dimension values of the measure sum to the total. */
|
|
603
|
+
export function isAdditive(measureExpr) {
|
|
604
|
+
if (typeof measureExpr !== 'string' || !measureExpr.trim()) return false
|
|
605
|
+
const toks = tokenize(measureExpr)
|
|
606
|
+
if (!toks) return false
|
|
607
|
+
let sawAgg = false
|
|
608
|
+
for (let k = 0; k < toks.length; k++) {
|
|
609
|
+
const t = toks[k]
|
|
610
|
+
if (t.kind === 'op' && t.text === '/') return false
|
|
611
|
+
if (t.kind === 'word' && /^(DIV|MOD)$/.test(t.upper)) return false
|
|
612
|
+
if (t.kind === 'word' && toks[k + 1]?.kind === 'open') {
|
|
613
|
+
const name = t.text.split('.').pop().toUpperCase()
|
|
614
|
+
if (name === 'SUM') { sawAgg = true; continue }
|
|
615
|
+
if (name === 'COUNT' || name === 'COUNT_BIG') {
|
|
616
|
+
if (toks[k + 2]?.kind === 'word' && toks[k + 2].upper === 'DISTINCT') return false
|
|
617
|
+
sawAgg = true
|
|
618
|
+
continue
|
|
619
|
+
}
|
|
620
|
+
if (isAggName(name)) return false
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
return sawAgg
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
// ------------------------------------------------------------ alerts
|
|
627
|
+
|
|
628
|
+
const fmt = (v) => {
|
|
629
|
+
if (v === null || v === undefined || Number.isNaN(v)) return '空'
|
|
630
|
+
if (!Number.isFinite(v)) return v > 0 ? '∞' : '-∞'
|
|
631
|
+
const r = Math.round(v * 100) / 100
|
|
632
|
+
return Math.abs(r) >= 1e4 ? r.toLocaleString('en-US') : String(r)
|
|
633
|
+
}
|
|
634
|
+
const fmtPct = (p) => (Number.isFinite(p) ? `${p > 0 ? '+' : ''}${fmt(p)}%` : (p > 0 ? '+∞' : '-∞'))
|
|
635
|
+
const isNum = (v) => typeof v === 'number' && Number.isFinite(v)
|
|
636
|
+
const num = (v) => (v === null || v === undefined || v === '' ? null : (isNum(Number(v)) ? Number(v) : null))
|
|
637
|
+
|
|
638
|
+
const OP_TEXT = { '>': '大于', '>=': '大于等于', '<': '小于', '<=': '小于等于', '==': '等于', '!=': '不等于' }
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* Evaluate an alert condition against a time series.
|
|
642
|
+
* @param {object} condition
|
|
643
|
+
* @param {{ t: any, v: number | null }[]} series - ascending by t.
|
|
644
|
+
*/
|
|
645
|
+
export function evaluateAlert(condition, series, { now } = {}) {
|
|
646
|
+
const pts = (Array.isArray(series) ? series : []).map(p => ({ t: p?.t, v: num(p?.v) }))
|
|
647
|
+
const last = pts.at(-1)
|
|
648
|
+
const value = last ? last.v : null
|
|
649
|
+
const type = condition?.type
|
|
650
|
+
if (type === 'missing') {
|
|
651
|
+
if (!last) return { triggered: true, value: null, message: '没有返回任何数据' }
|
|
652
|
+
if (value === null) return { triggered: true, value: null, message: `最新数据点(${String(last.t)})的值为空` }
|
|
653
|
+
const staleMs = Number(condition.staleMinutes) * 60000
|
|
654
|
+
if (staleMs > 0 && now !== undefined && last.t !== undefined) {
|
|
655
|
+
const lt = new Date(last.t).getTime()
|
|
656
|
+
const nt = new Date(now).getTime()
|
|
657
|
+
if (Number.isFinite(lt) && Number.isFinite(nt) && nt - lt > staleMs) return { triggered: true, value, message: `数据已超过 ${condition.staleMinutes} 分钟未更新(最新为 ${String(last.t)})` }
|
|
658
|
+
}
|
|
659
|
+
return { triggered: false, value, message: `数据正常,最新值为 ${fmt(value)}` }
|
|
660
|
+
}
|
|
661
|
+
if (value === null) return { triggered: false, value: null, message: last ? '最新数据点为空,无法判断' : '没有数据,无法判断' }
|
|
662
|
+
if (type === 'threshold') {
|
|
663
|
+
const op = condition.op
|
|
664
|
+
const target = num(condition.value)
|
|
665
|
+
if (!OP_TEXT[op] || target === null) return { triggered: false, value, message: '阈值条件无效' }
|
|
666
|
+
const triggered = { '>': value > target, '>=': value >= target, '<': value < target, '<=': value <= target, '==': value === target, '!=': value !== target }[op]
|
|
667
|
+
return { triggered, value, baseline: target, message: triggered ? `最新值 ${fmt(value)} ${OP_TEXT[op]}阈值 ${fmt(target)}` : `最新值 ${fmt(value)},未触发(条件:${OP_TEXT[op]} ${fmt(target)})` }
|
|
668
|
+
}
|
|
669
|
+
if (type === 'change') {
|
|
670
|
+
const prev = pts.length >= 2 ? pts.at(-2).v : null
|
|
671
|
+
if (prev === null) return { triggered: false, value, message: '缺少上一期数据,无法计算变化' }
|
|
672
|
+
const changePct = prev === 0 ? (value === 0 ? 0 : (value > 0 ? Infinity : -Infinity)) : (value - prev) / Math.abs(prev) * 100
|
|
673
|
+
const pct = Math.abs(Number(condition.pct) || 0)
|
|
674
|
+
const dir = condition.direction ?? 'any'
|
|
675
|
+
const triggered = dir === 'up' ? changePct >= pct && changePct > 0
|
|
676
|
+
: dir === 'down' ? changePct <= -pct && changePct < 0
|
|
677
|
+
: Math.abs(changePct) >= pct && changePct !== 0
|
|
678
|
+
const desc = prev === 0 ? `从 0 变为 ${fmt(value)}` : `较上期 ${fmt(prev)} 变化 ${fmtPct(changePct)}`
|
|
679
|
+
const dirText = { up: '上涨', down: '下跌', any: '变化' }[dir] ?? '变化'
|
|
680
|
+
return { triggered, value, baseline: prev, changePct, message: triggered ? `最新值 ${fmt(value)},${desc},超过${dirText}阈值 ${pct}%` : `最新值 ${fmt(value)},${desc},未触发` }
|
|
681
|
+
}
|
|
682
|
+
if (type === 'anomaly') {
|
|
683
|
+
const z0 = Number(condition.z ?? 3) > 0 ? Number(condition.z ?? 3) : 3
|
|
684
|
+
const window = Math.max(2, Math.floor(Number(condition.window ?? 14)) || 14)
|
|
685
|
+
const minPoints = Math.max(2, Math.floor(Number(condition.minPoints ?? 7)) || 7)
|
|
686
|
+
const hist = pts.slice(0, -1).map(p => p.v).filter(v => v !== null).slice(-window)
|
|
687
|
+
if (hist.length < minPoints) return { triggered: false, value, message: `历史数据不足(${hist.length}/${minPoints} 个点),暂不判断异常` }
|
|
688
|
+
const mean = hist.reduce((s, v) => s + v, 0) / hist.length
|
|
689
|
+
const std = Math.sqrt(hist.reduce((s, v) => s + (v - mean) ** 2, 0) / (hist.length - 1))
|
|
690
|
+
const expected = { low: mean - z0 * std, high: mean + z0 * std }
|
|
691
|
+
const eps = 1e-12 * Math.max(1, Math.abs(mean))
|
|
692
|
+
let z
|
|
693
|
+
if (std <= eps) z = Math.abs(value - mean) <= eps ? 0 : (value > mean ? Infinity : -Infinity)
|
|
694
|
+
else z = (value - mean) / std
|
|
695
|
+
const dir = condition.direction ?? 'any'
|
|
696
|
+
const triggered = dir === 'up' ? z >= z0 : dir === 'down' ? z <= -z0 : Math.abs(z) >= z0
|
|
697
|
+
const band = `[${fmt(expected.low)}, ${fmt(expected.high)}]`
|
|
698
|
+
return {
|
|
699
|
+
triggered, value, baseline: mean, z, expected,
|
|
700
|
+
message: triggered ? `最新值 ${fmt(value)} 偏离近 ${hist.length} 期均值 ${fmt(mean)}(z=${fmt(z)}),超出预期区间 ${band}` : `最新值 ${fmt(value)} 在预期区间 ${band} 内`,
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
return { triggered: false, value, message: '未知的告警条件类型' }
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// ------------------------------------------------------------ result comparison
|
|
707
|
+
|
|
708
|
+
function normCell(v) {
|
|
709
|
+
if (v === null || v === undefined) return null
|
|
710
|
+
if (typeof v === 'boolean') return v ? 1 : 0
|
|
711
|
+
if (typeof v === 'bigint') return Number(v)
|
|
712
|
+
if (typeof v === 'number') return Number.isNaN(v) ? 'NaN' : v
|
|
713
|
+
if (v instanceof Date) return Number.isNaN(v.getTime()) ? 'Invalid Date' : formatDate(v, { full: true })
|
|
714
|
+
if (typeof Buffer !== 'undefined' && Buffer.isBuffer?.(v)) return v.toString('hex')
|
|
715
|
+
if (typeof v === 'object') return JSON.stringify(v)
|
|
716
|
+
const s = String(v)
|
|
717
|
+
const t = s.trim()
|
|
718
|
+
if (/^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/.test(t)) return Number(t)
|
|
719
|
+
const m = /^(\d{4}-\d{2}-\d{2})(?:[ T](\d{2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?$/.exec(t)
|
|
720
|
+
if (m) return `${m[1]} ${m[2] ?? '00'}:${m[3] ?? '00'}:${m[4] ?? '00'}`
|
|
721
|
+
return s
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
const typeRank = (v) => (v === null ? 0 : typeof v === 'number' ? 1 : 2)
|
|
725
|
+
|
|
726
|
+
function cellEq(a, b, tol) {
|
|
727
|
+
if (a === null || b === null) return a === b
|
|
728
|
+
if (typeof a === 'number' && typeof b === 'number') {
|
|
729
|
+
if (a === b) return true
|
|
730
|
+
return Math.abs(a - b) <= tol * Math.max(1, Math.abs(a), Math.abs(b))
|
|
731
|
+
}
|
|
732
|
+
return a === b
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
function cellCmp(a, b, tol) {
|
|
736
|
+
const ra = typeRank(a); const rb = typeRank(b)
|
|
737
|
+
if (ra !== rb) return ra - rb
|
|
738
|
+
if (ra === 0) return 0
|
|
739
|
+
if (ra === 1) return cellEq(a, b, tol) ? 0 : a - b
|
|
740
|
+
return a < b ? -1 : a > b ? 1 : 0
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
function rowsOf(result) {
|
|
744
|
+
const cols = (result?.columns ?? []).map(c => (typeof c === 'string' ? { name: c } : c))
|
|
745
|
+
const rows = (result?.rows ?? []).map(r => (Array.isArray(r) ? r : cols.map(c => r?.[c.name])).map(normCell))
|
|
746
|
+
return { cols, rows }
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
function multisetEq(a, b, tol) {
|
|
750
|
+
if (a.length !== b.length) return false
|
|
751
|
+
const x = [...a].sort((p, q) => cellCmp(p, q, tol))
|
|
752
|
+
const y = [...b].sort((p, q) => cellCmp(p, q, tol))
|
|
753
|
+
return x.every((v, i) => cellEq(v, y[i], tol))
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
const rowCmp = (tol) => (p, q) => {
|
|
757
|
+
for (let i = 0; i < p.length; i++) { const c = cellCmp(p[i], q[i], tol); if (c) return c }
|
|
758
|
+
return 0
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
function rowsMatch(expRows, actRows, mapping, ordered, tol) {
|
|
762
|
+
const proj = actRows.map(r => mapping.map(j => r[j]))
|
|
763
|
+
if (ordered) {
|
|
764
|
+
for (let i = 0; i < expRows.length; i++) if (!expRows[i].every((v, c) => cellEq(v, proj[i][c], tol))) return { ok: false, index: i }
|
|
765
|
+
return { ok: true }
|
|
766
|
+
}
|
|
767
|
+
const x = [...expRows].sort(rowCmp(tol))
|
|
768
|
+
const y = proj.sort(rowCmp(tol))
|
|
769
|
+
for (let i = 0; i < x.length; i++) if (!x[i].every((v, c) => cellEq(v, y[i][c], tol))) return { ok: false, row: x[i] }
|
|
770
|
+
return { ok: true }
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* Compare two query results ignoring column names/order and (unless ordered) row order.
|
|
775
|
+
* @param {{ columns: ({ name: string, kind?: string } | string)[], rows: any[][] | object[] }} expected
|
|
776
|
+
* @param {typeof expected} actual
|
|
777
|
+
* @returns {{ match: boolean, reason?: string, detail?: object }}
|
|
778
|
+
*/
|
|
779
|
+
/**
|
|
780
|
+
* Long vs wide: one side has a single row whose numeric cells equal the
|
|
781
|
+
* multiset of one numeric column of the other side (whose other columns are
|
|
782
|
+
* category labels). Only for small results (≤ 12 categories).
|
|
783
|
+
*/
|
|
784
|
+
function pivotEquivalent(e, a, tolerance) {
|
|
785
|
+
const tryPair = (long, wide) => {
|
|
786
|
+
if (wide.rows.length !== 1 || long.rows.length < 2 || long.rows.length > 12) return false
|
|
787
|
+
const wideNums = wide.rows[0].filter(v => typeof v === 'number')
|
|
788
|
+
if (wideNums.length !== long.rows.length) return false
|
|
789
|
+
for (let j = 0; j < long.cols.length; j++) {
|
|
790
|
+
const col = long.rows.map(r => r[j])
|
|
791
|
+
if (!col.every(v => typeof v === 'number')) continue
|
|
792
|
+
const others = long.cols.map((_, k) => k).filter(k => k !== j)
|
|
793
|
+
if (!others.every(k => long.rows.every(r => typeof r[k] !== 'number'))) continue
|
|
794
|
+
if (multisetEq(col, wideNums, tolerance)) return true
|
|
795
|
+
}
|
|
796
|
+
return false
|
|
797
|
+
}
|
|
798
|
+
return tryPair(e, a) || tryPair(a, e)
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
export function compareResults(expected, actual, { tolerance = 1e-6, ordered = false } = {}) {
|
|
802
|
+
const e = rowsOf(expected)
|
|
803
|
+
const a = rowsOf(actual)
|
|
804
|
+
if (e.rows.length !== a.rows.length) {
|
|
805
|
+
// Same numbers in a pivoted shape (e.g. one row per category vs one row with a column per category).
|
|
806
|
+
if (pivotEquivalent(e, a, tolerance)) return { match: true, detail: { pivot: true } }
|
|
807
|
+
return { match: false, reason: `行数不同:期望 ${e.rows.length} 行,实际 ${a.rows.length} 行`, detail: { expectedRows: e.rows.length, actualRows: a.rows.length } }
|
|
808
|
+
}
|
|
809
|
+
if (a.cols.length < e.cols.length) return { match: false, reason: `列数不足:期望 ${e.cols.length} 列,实际 ${a.cols.length} 列` }
|
|
810
|
+
const colVals = (rows, j) => rows.map(r => r[j])
|
|
811
|
+
const candidates = e.cols.map((_, i) => {
|
|
812
|
+
const ev = colVals(e.rows, i)
|
|
813
|
+
return a.cols.map((_, j) => j).filter(j => multisetEq(ev, colVals(a.rows, j), tolerance))
|
|
814
|
+
})
|
|
815
|
+
const missing = candidates.findIndex(c => !c.length)
|
|
816
|
+
if (missing >= 0) {
|
|
817
|
+
const name = e.cols[missing].name
|
|
818
|
+
const same = a.cols.findIndex(c => String(c.name).toLowerCase() === String(name).toLowerCase())
|
|
819
|
+
if (same >= 0) {
|
|
820
|
+
const ev = [...colVals(e.rows, missing)].sort((p, q) => cellCmp(p, q, tolerance))
|
|
821
|
+
const av = [...colVals(a.rows, same)].sort((p, q) => cellCmp(p, q, tolerance))
|
|
822
|
+
const k = ev.findIndex((v, i) => !cellEq(v, av[i], tolerance))
|
|
823
|
+
const numeric = typeof ev[k] === 'number' || typeof av[k] === 'number'
|
|
824
|
+
return { match: false, reason: `${numeric ? '数值不同' : '取值不同'}:列「${name}」期望 ${JSON.stringify(ev[k])},实际 ${JSON.stringify(av[k])}`, detail: { column: name, expected: ev[k], actual: av[k] } }
|
|
825
|
+
}
|
|
826
|
+
return { match: false, reason: `缺少与期望列「${name}」一致的列`, detail: { column: name } }
|
|
827
|
+
}
|
|
828
|
+
if (!e.cols.length) return { match: true, detail: { mapping: {} } }
|
|
829
|
+
// Backtracking over candidate mappings (prefer injective), bounded.
|
|
830
|
+
// Expected columns that duplicate an earlier one row-for-row share its actual column; the rest map injectively.
|
|
831
|
+
const dupOf = e.cols.map((_, i) => {
|
|
832
|
+
for (let j = 0; j < i; j++) if (e.rows.every(r => cellEq(r[i], r[j], tolerance))) return j
|
|
833
|
+
return -1
|
|
834
|
+
})
|
|
835
|
+
const order = candidates.map((c, i) => i).filter(i => dupOf[i] < 0).sort((p, q) => candidates[p].length - candidates[q].length)
|
|
836
|
+
const mapping = new Array(e.cols.length)
|
|
837
|
+
const fillDups = () => { dupOf.forEach((j, i) => { if (j >= 0) mapping[i] = mapping[j] }) }
|
|
838
|
+
const used = new Set()
|
|
839
|
+
let attempts = 0
|
|
840
|
+
let lastFail
|
|
841
|
+
const search = (d) => {
|
|
842
|
+
if (attempts > 2000) return false
|
|
843
|
+
if (d === order.length) {
|
|
844
|
+
attempts++
|
|
845
|
+
fillDups()
|
|
846
|
+
const r = rowsMatch(e.rows, a.rows, mapping, ordered, tolerance)
|
|
847
|
+
if (!r.ok) lastFail = r
|
|
848
|
+
return r.ok
|
|
849
|
+
}
|
|
850
|
+
const i = order[d]
|
|
851
|
+
for (const j of candidates[i]) {
|
|
852
|
+
if (used.has(j)) continue
|
|
853
|
+
mapping[i] = j
|
|
854
|
+
used.add(j)
|
|
855
|
+
if (search(d + 1)) return true
|
|
856
|
+
used.delete(j)
|
|
857
|
+
}
|
|
858
|
+
return false
|
|
859
|
+
}
|
|
860
|
+
if (search(0)) {
|
|
861
|
+
return { match: true, detail: { mapping: Object.fromEntries(e.cols.map((c, i) => [c.name, a.cols[mapping[i]].name])) } }
|
|
862
|
+
}
|
|
863
|
+
if (ordered && lastFail?.index !== undefined) {
|
|
864
|
+
// Would the rows match ignoring order?
|
|
865
|
+
const unordered = compareResults(expected, actual, { tolerance, ordered: false })
|
|
866
|
+
if (unordered.match) return { match: false, reason: `行顺序不同:第 ${lastFail.index + 1} 行起不一致`, detail: { row: lastFail.index } }
|
|
867
|
+
}
|
|
868
|
+
return { match: false, reason: '各列取值一致,但行内组合不同', detail: lastFail?.row ? { expectedRow: lastFail.row } : undefined }
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
/** Token-level Jaccard similarity of two SQL texts (display only). */
|
|
872
|
+
export function scoreSql(expectedSql, actualSql) {
|
|
873
|
+
const norm = (sql) => {
|
|
874
|
+
const toks = tokenize(String(sql ?? '').trim().replace(/;+\s*$/, '')) ?? []
|
|
875
|
+
return new Set(toks.filter(t => !/^(comma|open|close)$/.test(t.kind)).map(t => {
|
|
876
|
+
if (t.kind === 'word') return t.text.toLowerCase()
|
|
877
|
+
if (t.kind === 'ident') return unquoteIdent(t.text).toLowerCase()
|
|
878
|
+
if (t.kind === 'number') return String(Number(t.text))
|
|
879
|
+
return t.text
|
|
880
|
+
}))
|
|
881
|
+
}
|
|
882
|
+
const x = norm(expectedSql)
|
|
883
|
+
const y = norm(actualSql)
|
|
884
|
+
if (!x.size && !y.size) return 1
|
|
885
|
+
let inter = 0
|
|
886
|
+
for (const t of x) if (y.has(t)) inter++
|
|
887
|
+
return inter / (x.size + y.size - inter)
|
|
888
|
+
}
|