@hedeqiang/dsh-nl2sql 0.11.1 → 0.12.0

package/service/bots.js

@@ -1,5 +1,5 @@
 /**
- * Nl2SqlService methods: inbound IM bots (see docs/CHANNELS.md).
+ * Nl2SqlService methods: inbound IM bots (see docs/channels.md).
  * Mixed into the service prototype by index.js; `this` is the service.
  *
  * Every channel whose kind has `capabilities.bot` may carry
@@ -279,10 +279,15 @@ export const methods = {
     const name = label(platform)
     const userId = m.sender?.id != null ? String(m.sender.id) : undefined
     const resolved = this.resolveActor({ platform, userId, email: m.sender?.email, name: m.sender?.name, chatId: m.chatId != null ? String(m.chatId) : m.chatId, chatType: m.chatType })
-    if (resolved.refused) { await reply(resolved.refused); return }
+    // Remember who/which group talked to the bot: the member form offers them as one-click picks.
+    let chatName
+    if (m.chatType === 'group') { try { chatName = typeof m.chatName === 'function' ? await m.chatName() : m.chatName } catch {} }
+    if (userId) this.noteSeenIdentity({ platform, kind: 'person', id: userId, name: m.sender?.name, email: m.sender?.email, channelId: m.channelId, refused: Boolean(resolved.refused) })
+    if (m.chatType === 'group' && m.chatId) this.noteSeenIdentity({ platform, kind: 'chat', id: String(m.chatId), name: chatName, chatName, channelId: m.channelId, refused: Boolean(resolved.refused) })
+    if (resolved.refused) { await this.save(); await reply(resolved.refused); return }
     const actor = resolved.actor
     // Commands: 帮助 / 数据源 / 切换 <名称> / 新对话
-    const cmd = text.match(/^\/?(帮助|help|数据源|切换数据源|切换|新对话|reset|start)\s*(.*)$/i)
+    const cmd = text.match(/^\/?(帮助|help|数据源|切换数据源|切换|自动|auto|新对话|reset|start)\s*(.*)$/i)
     // Members with different scopes must not share a conversation: per-user sessions
     // whenever the sender's effective scope or rules differ from the chat's.
     const scopeSig = JSON.stringify([actor.datasourceIds, actor.role, actor.rowRules ?? []])
@@ -292,23 +297,35 @@ export const methods = {
     this.state.botConversations ??= {}
     const convs = this.state.botConversations
     const conv = convs[key] ?? { key, channelId: m.channelId ?? null, platform, chatId: m.chatId, chatType: m.chatType, sessionId: null, datasourceId: actor.defaultDatasourceId, createdAt: Date.now() }
+    if (chatName && !conv.chatName) conv.chatName = chatName
     const allowedDs = this.state.datasources.filter(d => !Array.isArray(actor.datasourceIds) || actor.datasourceIds.includes(d.id))
     if (conv.datasourceId && !allowedDs.some(d => d.id === conv.datasourceId)) conv.datasourceId = allowedDs[0]?.id ?? null
     if (cmd) {
       const verb = cmd[1].toLowerCase()
-      if (verb === '帮助' || verb === 'help' || verb === 'start') { await reply(`直接${m.chatType === 'group' ? ' @我 ' : ''}提问即可，例如「上周每天的订单量」。\n命令：数据源（查看可用数据源）、切换 <名称>、新对话。\n当前数据源：${this.findDatasource(conv.datasourceId)?.name ?? '未选择'}${actor.role === 'viewer' ? '\n你是只读成员：可以查询指标库中的指标、已验证的常见问题和看板。' : ''}`); return }
-      if (verb === '数据源') { await reply(`可用数据源：${allowedDs.map(d => d.name).join('、') || '无'}\n当前：${this.findDatasource(conv.datasourceId)?.name ?? '未选择'}`); return }
+      if (verb === '帮助' || verb === 'help' || verb === 'start') { await reply(`直接${m.chatType === 'group' ? ' @我 ' : ''}提问即可，例如「上周每天的订单量」。\n命令：数据源（查看可用数据源）、切换 <名称>、自动（按问题自动选择数据源）、新对话。\n当前数据源：${this.findDatasource(conv.datasourceId)?.name ?? '未选择'}${conv.datasourcePinned ? '（已指定）' : allowedDs.length > 1 ? '（按问题自动选择）' : ''}${actor.role === 'viewer' ? '\n你是只读成员：可以查询指标库中的指标、已验证的常见问题和看板。' : ''}`); return }
+      if (verb === '数据源') { await reply(`可用数据源：\n${allowedDs.map(d => `· ${d.name}${d.description ? ` — ${d.description}` : ''}`).join('\n') || '无'}\n当前：${this.findDatasource(conv.datasourceId)?.name ?? '未选择'}${conv.datasourcePinned ? '（已指定，发送「自动」恢复按问题自动选择）' : allowedDs.length > 1 ? '（按问题自动选择；发送「切换 <名称>」固定）' : ''}`); return }
+      if (verb === '自动' || verb === 'auto') { delete conv.datasourcePinned; convs[key] = conv; await this.save(); await reply(`已改为按问题自动选择数据源（可用：${allowedDs.map(d => d.name).join('、')}）。`); return }
       if (verb === '新对话' || verb === 'reset') { conv.sessionId = null; convs[key] = conv; await this.save(); await reply('已开始新对话。'); return }
-      const target = allowedDs.find(d => d.name === cmd[2].trim() || d.id === cmd[2].trim())
+      const want = cmd[2].trim().toLowerCase()
+      const target = allowedDs.find(d => d.name.toLowerCase() === want || d.id === cmd[2].trim()) ?? (want ? allowedDs.filter(d => d.name.toLowerCase().includes(want) || (d.database ?? '').toLowerCase() === want) : []).find((_, __, arr) => arr.length === 1)
       if (!target) { await reply(`找不到可用的数据源「${cmd[2]}」。可用：${allowedDs.map(d => d.name).join('、')}`); return }
       conv.datasourceId = target.id
+      conv.datasourcePinned = true
       if (conv.sessionId) { this.state.sessions[conv.sessionId] = target.id }
       convs[key] = conv
       await this.save()
-      await reply(`已切换到数据源「${target.name}」。`)
+      await reply(`已切换到数据源「${target.name}」。之后的提问都会查询这个数据源；发送「自动」恢复按问题自动选择。`)
       return
     }
-    const ds = this.findDatasource(conv.datasourceId) ?? allowedDs[0]
+    // Which data source answers: an explicit 切换 wins; otherwise the question picks the most
+    // relevant one among those the member may use (schema + business knowledge), so people
+    // don't need to know data source names. The agent can still switch with nl2sql_datasource.
+    let ds = this.findDatasource(conv.datasourceId) ?? allowedDs[0]
+    let routed = null
+    if (!conv.datasourcePinned && allowedDs.length > 1) {
+      routed = await this.routeDatasource(text, allowedDs).catch(() => null)
+      if (routed?.ds) ds = routed.ds
+    }
     if (!ds) { await reply('没有可访问的数据源，请联系管理员。'); return }
     this.assertAccess(actor, ds)
     if (!this.ctx.get('sessionController')) { await reply('当前 Harness 不支持创建会话（缺少 sessionController）。'); return }
@@ -338,9 +355,17 @@ export const methods = {
         await this.save()
       }
       const where = m.chatType === 'group' ? `群「${conv.chatName ?? m.chatId}」` : '私聊'
-      const prompt = `[${name}${where} · ${actor.name}${actor.role === 'viewer' ? ' · 只读成员' : ''}] ${text}\n\n（回答会以${name}消息发回该聊天：请用简洁的中文 Markdown 写结论，必要时附上查询结果；不要调用 nl2sql_send，也不要向用户提问——有歧义时说明你的假设后直接回答。${actor.role === 'viewer' ? '该成员是只读角色：只能用 nl2sql_metric 查询指标库，或复用 nl2sql_knowledge 中已验证的样例 SQL；不能自由编写 SQL。' : ''}）`
+      const dsLine = allowedDs.length > 1
+        ? (conv.datasourcePinned
+          ? `数据源：${ds.name}（群内已用「切换」指定，请只查询这个数据源）。`
+          : `当前数据源：${ds.name}${routed?.ds ? '（按问题自动选择）' : ''}。该成员可用的数据源：\n${await this.datasourceCatalog(allowedDs)}\n请先判断问题属于哪个数据源：若不是当前数据源，先用 nl2sql_datasource action=use 切换再查询；在回答开头注明使用的数据源。`)
+        : `数据源：${ds.name}。`
+      const prompt = `[${name}${where} · ${actor.name}${actor.role === 'viewer' ? ' · 只读成员' : ''}] ${text}\n\n（${dsLine}\n回答会以${name}消息发回该聊天：请用简洁的中文 Markdown 写结论，必要时附上查询结果；不要调用 nl2sql_send，也不要向用户提问——有歧义时说明你的假设后直接回答。${actor.role === 'viewer' ? '该成员是只读角色：只能用 nl2sql_metric 查询指标库，或复用 nl2sql_knowledge 中已验证的样例 SQL；不能自由编写 SQL。' : ''}）`
       const requestId = platform === 'lark' ? `lark-${m.messageId}` : `${platform}-${m.chatId}-${m.messageId}`
+      if (conv.sessionId && this.state.sessions[conv.sessionId] !== ds.id) { this.state.sessions[conv.sessionId] = ds.id; await this.save() }
       const turn = await this.runAgentTurn({ sessionId: conv.sessionId, title: `${name} · ${conv.chatName ?? actor.name}`, requestId, text: prompt, prepare, timeoutMs: 600000 })
+      const usedDs = this.findDatasource(this.state.sessions[conv.sessionId]) ?? ds
+      if (usedDs.id !== conv.datasourceId) { conv.datasourceId = usedDs.id; convs[key] = conv }
       let body = turn.text
       if (turn.reason === 'timeout') body = `${body ? `${body}\n\n` : ''}<font color='grey'>（查询超过 10 分钟仍未完成，已停止。请缩小范围后重试。）</font>`
       if (turn.reason === 'input-required') body = `${body ? `${body}\n\n` : ''}<font color='grey'>（需要在 Harness 中确认后才能继续，已取消本次回答。）</font>`
@@ -350,7 +375,7 @@ export const methods = {
       let status = turn.reason === 'idle' ? 'ok' : 'error'
       let error = turn.reason === 'idle' ? undefined : turn.reason
       try {
-        await m.reply({ title: text.length > 40 ? `${text.slice(0, 40)}…` : text, text: body || '（没有生成文字回答）', results: shown, footer: `数据问答 · ${ds.name} · ${actor.name}` })
+        await m.reply({ title: text.length > 40 ? `${text.slice(0, 40)}…` : text, text: body || '（没有生成文字回答）', results: shown, footer: `数据问答 · ${usedDs.name} · ${actor.name}` })
       } catch (e) {
         status = 'error'
         error = e.message
@@ -362,6 +387,48 @@ export const methods = {
     })
   },
 
+  /**
+   * Pick the data source a question is about, among `candidates`, only on
+   * strong evidence: the question names the source (name / database /
+   * description), or it matches that source's business knowledge (verified
+   * examples, metrics, glossary). Schema keyword overlap alone is too weak
+   * (English table names vs Chinese questions), so otherwise returns null and
+   * the agent chooses from the catalog in the prompt.
+   */
+  async routeDatasource(question, candidates) {
+    const q = String(question ?? '').toLowerCase()
+    const scored = candidates.map(d => {
+      const named = [d.name, d.database, d.description].filter(x => x && String(x).length >= 2).some(x => q.includes(String(x).toLowerCase()))
+      let know = 0
+      try {
+        const f = this.searchKnowledge(d.id, question, 5)
+        know = Math.max(0, ...f.examples.map(x => x.score), ...f.metrics.map(x => x.score), ...f.glossary.map(x => x.score))
+      } catch {}
+      return { ds: d, score: (named ? 2 : 0) + know }
+    }).sort((a, b) => b.score - a.score)
+    const [top, second] = scored
+    if (!top || top.score < 0.5) return null
+    return top.score - (second?.score ?? 0) >= 0.3 ? { ds: top.ds, score: top.score } : null
+  },
+
+  /** One line per data source for the bot prompt: name, type, description and main tables. */
+  async datasourceCatalog(list) {
+    const lines = []
+    for (const d of list) {
+      let tables = ''
+      try {
+        const schema = this.schemas.get(d.id) ?? await this.readSchemaFile(d.id)
+        if (schema?.tables?.length) {
+          const rank = { fact: 0, dimension: 1, view: 2, other: 3, log: 4, config: 5 }
+          const top = [...schema.tables].sort((a, b) => (rank[a.role] ?? 3) - (rank[b.role] ?? 3) || (b.rows ?? 0) - (a.rows ?? 0)).slice(0, 12)
+          tables = `；主要表：${top.map(t => `${t.name}${t.comment ? `(${String(t.comment).split(/[\s,，。;；(（]/)[0].slice(0, 12)})` : ''}`).join(', ')}${schema.tables.length > 12 ? ` 等 ${schema.tables.length} 张` : ''}`
+        }
+      } catch {}
+      lines.push(`- ${d.name}（${d.type ?? 'mysql'}${d.database ? ` · ${d.database}` : ''}）${d.description ? `：${d.description.slice(0, 60)}` : ''}${tables}`)
+    }
+    return lines.join('\n')
+  },
+
   /** Run tasks with the same key one after another. */
   serialize(key, task) {
     this.queues ??= new Map()

package/service/channels.js

@@ -1,5 +1,5 @@
 /**
- * Nl2SqlService methods: IM channels (see docs/CHANNELS.md).
+ * Nl2SqlService methods: IM channels (see docs/channels.md).
  * Mixed into the service prototype by index.js; `this` is the service.
  *
  * A channel record (state.channels[]):
@@ -11,7 +11,8 @@
  * Secrets live in one credential record per channel: `nl2sql/ch-<id>`.
  */
 import { ChannelError } from '../channels/http.js'
-import { renderChartPng } from '../channels/message.js'
+import { needsAttachment, renderChartPng } from '../channels/message.js'
+import { writeXlsx } from '../xlsx.js'
 import { getAdapter, kindOf, listCatalog, loadAdapters, requireKind, secretKeys } from '../channels/index.js'
 import { UserError, newId, pick, requireText, upsert } from './shared.js'
 
@@ -292,7 +293,7 @@ export const methods = {
    * process' cache) or given inline; each is re-checked against privacy:
    * masked values stay masked because results are stored masked.
    */
-  async sendToChannel({ channelId, title, text, resultIds = [], results = [], includeSql = false, source = 'ui', sessionId, footer, chartMode, template, mentionAll }) {
+  async sendToChannel({ channelId, title, text, resultIds = [], results = [], includeSql = false, source = 'ui', sessionId, footer, chartMode, template, mentionAll, attachFull }) {
     await loadAdapters()
     const c = this.requireChannel(channelId)
     const resolved = []
@@ -319,15 +320,30 @@ export const methods = {
       ...(template ? { template } : {}),
       footer: footer ?? `数据问答 · ${new Date().toLocaleString('zh-CN', { hour12: false })}${resolved[0]?.datasourceName ? ` · ${resolved[0].datasourceName}` : ''}`,
     }
+    // Results with more rows than the message can show: attach the full data as Excel
+    // where the platform can send files (default on; per send `attachFull: false` to skip).
+    const caps = found.kind.capabilities ?? {}
+    const shown = caps.tableRows ?? 10
+    const wantAttach = attachFull ?? c.attachFull ?? true
+    if (caps.files && wantAttach) {
+      const idx = message.results.map((r, i) => (needsAttachment(r, shown) ? i : -1)).filter(i => i >= 0)
+      if (idx.length) {
+        const sheets = idx.map(i => ({ name: (message.results[i].title || `结果${i + 1}`).slice(0, 31), columns: message.results[i].columns, rows: message.results[i].rows }))
+        if (includeSql) sheets.push({ name: 'SQL', columns: [{ name: '结果', kind: 'text' }, { name: 'SQL', kind: 'text' }], rows: idx.map(i => [message.results[i].title ?? '', message.results[i].sql ?? '']) })
+        const buffer = writeXlsx(sheets, { creator: 'DeepSeek Harness 数据问答' })
+        const safe = String(message.title).replace(/[\\/:*?"<>|\s]+/g, '_').slice(0, 60) || '数据'
+        message.attachment = { buffer, fileName: `${safe}_${new Date().toISOString().slice(0, 10)}.xlsx`, resultIndexes: idx }
+      }
+    }
     const started = Date.now()
     try {
       if (c.needsSecret) throw new ChannelError('channel/config', `渠道「${c.name}」需要重新填写密钥后才能使用`)
       const sent = await found.kind.send(this.adapterChannel(c), await this.channelSecrets(c), message, { renderChartPng, logger: console, fetch: globalThis.fetch })
       c.lastSentAt = Date.now()
       delete c.lastError
-      this.recordDelivery({ channel: c, title: message.title, status: 'ok', results: resolved.length, images: sent?.images, source, sessionId, elapsedMs: Date.now() - started })
+      this.recordDelivery({ channel: c, title: message.title, status: 'ok', results: resolved.length, images: sent?.images, files: sent?.files, ...(sent?.warning ? { error: sent.warning } : {}), source, sessionId, elapsedMs: Date.now() - started })
       await this.save()
-      return { ok: true, channel: c.name, platform: c.platform, messageId: sent?.messageId ?? null, images: sent?.images ?? 0, results: resolved.length }
+      return { ok: true, channel: c.name, platform: c.platform, messageId: sent?.messageId ?? null, images: sent?.images ?? 0, files: sent?.files ?? 0, ...(sent?.warning ? { warning: sent.warning } : {}), results: resolved.length }
     } catch (error) {
       c.lastError = error.message
       this.recordDelivery({ channel: c, title: message.title, status: 'error', error: error.message, results: resolved.length, source, sessionId, elapsedMs: Date.now() - started })
@@ -336,8 +352,8 @@ export const methods = {
     }
   },
 
-  recordDelivery({ channel, title, status, error, results, images, source, sessionId, elapsedMs }) {
-    this.records.put('delivery', { id: newId('dl'), at: Date.now(), channelId: channel.id, channelName: channel.name, ...(channel.platform ? { platform: channel.platform } : {}), title, status, ...(error ? { error: String(error).slice(0, 500) } : {}), results, images: images ?? 0, source, sessionId: sessionId ?? null, elapsedMs })
+  recordDelivery({ channel, title, status, error, results, images, files, source, sessionId, elapsedMs }) {
+    this.records.put('delivery', { id: newId('dl'), at: Date.now(), channelId: channel.id, channelName: channel.name, ...(channel.platform ? { platform: channel.platform } : {}), title, status, ...(error ? { error: String(error).slice(0, 500) } : {}), results, images: images ?? 0, files: files ?? 0, source, sessionId: sessionId ?? null, elapsedMs })
   },
 
   listDeliveries(limit = 100) {

package/service/push.js

@@ -64,7 +64,7 @@ export const methods = {
   },
 
   /** Send the current dashboard (with saved global filters) to a channel. */
-  async sendDashboard({ channelId, title, filters, dashboardId, chartMode }) {
+  async sendDashboard({ channelId, title, filters, dashboardId, chartMode, attachFull }) {
     const results = []
     const d = dashboardId ? this.requireDashboard(dashboardId) : this.currentDashboardObj()
     for (const w of d.widgets.slice(0, 10)) {
@@ -76,7 +76,7 @@ export const methods = {
       }
     }
     if (!results.length) throw new UserError('nl2sql/invalid', '看板为空')
-    const sent = await this.sendToChannel({ channelId, title: title || d.name || '数据看板', results, source: 'dashboard', chartMode })
+    const sent = await this.sendToChannel({ channelId, title: title || d.name || '数据看板', results, source: 'dashboard', chartMode, attachFull })
     await this.saveDashboardSnapshot(d, results, 'push')
     return sent
   },

package/service/suggestions.js

@@ -14,7 +14,7 @@ import { admitSql, lexSql } from '../sql-guard.js'
 import { textGrams } from './schema.js'
 import { SENSITIVE_NAME, UserError, hashText } from './shared.js'
 
-/** Thresholds (see DEVELOPMENT.md). */
+/** Thresholds (see docs/development.md). */
 export const SUGGEST = {
   shapeMin: 2, // agent queries with the same literal-free shape
   metricMin: 3, // distinct successful queries using the same aggregate + filter on one table

package/README.en.md

@@ -1,63 +0,0 @@
-# dsh-nl2sql · Data Q&A
-
-An enterprise NL2SQL plugin for DeepSeek Harness: connect databases, ask in plain language, and the agent writes read-only SQL, runs it and charts the result in the conversation. Pin results to dashboards, push them to Lark / Feishu, or ask the bot directly in a Lark group.
-
-[中文](README.md)
-
-## Features
-
-- **Data sources:** MySQL 5.7+, PostgreSQL, Elasticsearch SQL (optionally over SSH), and CSV / Excel uploads stored as local SQLite.
-- **Schema analysis:** table roles, column semantics (time / measure / enum / sensitive), inferred relations, ER view.
-- **Asking:** in a Harness conversation through the `nl2sql_*` tools; results render as ECharts charts or tables.
-- **Drill-down and "why did it change":** split by a dimension, show rows, rank dimensions by their contribution to a change.
-- **Metric library and business knowledge:** glossary, governed metrics (`nl2sql_metric`), column notes, verified examples; import / export; accuracy evaluation against the examples.
-- **Dashboards:** multiple boards, global filters, drag-to-reorder, three widths, snapshots, Excel export.
-- **Channels:** Lark / Feishu, Telegram, WeCom and DingTalk through pluggable adapters ([docs/CHANNELS.md](docs/CHANNELS.md)); push Markdown and results (native charts, images or tables per platform), and @-mention bots that need no public URL.
-- **Automation:** scheduled reports, metric alerts (threshold / change / anomaly / missing), schema change detection.
-- **Safety:**
-  - read-only admission with per-dialect lexing, read-only transactions, server-side timeouts, streamed truncation;
-  - table / column access policies with masking, full-scan guard;
-  - member scopes, viewer role and row-level rules for Lark users;
-  - a setting for what result data reaches the model (column statistics by default); audit attributed to people.
-
-## Install
-
-Requires DeepSeek Harness (desktop or `dsh web`).
-
-```bash
-git clone <this repo> dsh-nl2sql && cd dsh-nl2sql
-pnpm install --ignore-scripts
-```
-
-Install the directory as a bundle (plugin manager `install_bundle` with the absolute path), or add `@hedeqiang/dsh-nl2sql` to your profile's `dsh.profile.bundles` with a `link:` dependency. Once published, install `@hedeqiang/dsh-nl2sql` from npm.
-
-## Use
-
-1. Sidebar → **Data Q&A** → **Manage → Data sources**: add a database (a read-only account is recommended) or upload a file.
-2. **Schema** → analyze.
-3. Ask on the **Ask** page, or pick the Data Q&A preset in any conversation.
-4. Pin useful charts to a **Dashboard**; add glossary terms, metrics and examples under **Knowledge**.
-5. **Manage → Push channels / Alerts & reports / Members & access** for Lark push, automation and the Lark bot.
-
-## Data and privacy
-
-- **Configuration:** `~/.dsh/nl2sql/state.json`.
-- **Results, cache and logs:** `~/.dsh/nl2sql/results.sqlite`.
-- **Uploaded files:** `~/.dsh/nl2sql/files/`.
-- **Passwords and app secrets:** the Harness credential store.
-- **What reaches the model:** **Settings → Data shown to the model** decides this; by default it is schema, enum values and per-column result statistics.
-- **Lark:** content pushed to Lark leaves the machine.
-
-## Development
-
-```bash
-npm test                       # whole suite; local MySQL / PostgreSQL optional (skipped when absent)
-npm test -- --fast             # unit + UI tests only
-node scripts/build-client.mjs  # regenerate lib/client.js after editing client-src/
-```
-
-See [DEVELOPMENT.md](DEVELOPMENT.md) and [eval/README.md](eval/README.md).
-
-## License
-
-[MIT](LICENSE)