dsh-lh-data 0.1.0

package/src/tools/datasource.ts

@@ -0,0 +1,378 @@
+/**
+ * `datasource_*`：数据源的列表 / 连通性测试 / 远端表浏览 / 导入。
+ *
+ * 与 `dataset_*` 的关系：数据源只是**数据来源**的另一种形态——导入完成后的数据集
+ * 与 Excel 导入的完全同权，之后一律用 `dataset_*` 工具操作。
+ *
+ * 脱敏红线：四个工具的返回值与渲染文本里都不出现密码、连接串与本地物理表名。
+ */
+
+import { connectSource, testSource } from '../datasource/connection'
+import { isDataSourceError } from '../datasource/errors'
+import { importRemoteTable, type RemoteImportResult } from '../datasource/importer'
+import type { DataSourceRecord, ConnectionTestResult } from '../datasource/types'
+import {
+  renderColumns,
+  renderConnectionTest,
+  renderSourceList,
+  renderSourceTables,
+  type SourceListItem,
+  type SourceTableItem,
+} from '../render'
+import type { DataServices } from '../store'
+import { toolDef, ToolError, type ToolDefinition, type ToolExec } from '../tooling'
+
+const SOURCE_PARAM = {
+  type: 'string',
+  required: true,
+  description: '数据源的 sourceId 或登记名（来自 datasource_list）。',
+} as const
+
+/** `DataSourceError` → `ToolError`（保留 code），其余错误原样上抛。 */
+function asToolFailure(error: unknown): unknown {
+  return isDataSourceError(error) ? new ToolError(error.message, error.code) : error
+}
+
+function toListItem(record: DataSourceRecord): SourceListItem {
+  return {
+    id: record.id,
+    name: record.name,
+    type: record.type,
+    host: record.host,
+    port: record.port,
+    database: record.database,
+    status: record.status,
+    lastError: record.lastError,
+    lastCheckedAt: record.lastCheckedAt,
+  }
+}
+
+// ── datasource_list ──────────────────────────────────────────────────────────
+
+const COLUMN_SCHEMA = {
+  type: 'array',
+  items: {
+    type: 'object',
+    additionalProperties: false,
+    properties: {
+      name: { type: 'string' },
+      type: { type: 'string' },
+      nullable: { type: 'boolean' },
+      description: { type: 'string' },
+    },
+  },
+} as const
+
+export function createSourceListTool(services: DataServices): ToolDefinition {
+  return toolDef({
+    name: 'datasource_list',
+    description: [
+      '列出已登记的数据库数据源：sourceId、登记名、类型（mysql / postgresql）、地址、库名、连通状态。',
+      '数据源是全局登记的，不按工作区隔离。返回内容不含密码等凭据。',
+      '只读工具，不修改任何数据。后续 datasource_* 工具都用这里的 sourceId 或登记名指代数据源。',
+    ].join(' '),
+    parameters: {},
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          count: { type: 'integer', description: '数据源数量' },
+          sources: {
+            type: 'array',
+            items: {
+              type: 'object',
+              additionalProperties: false,
+              properties: {
+                id: { type: 'string' },
+                name: { type: 'string' },
+                type: { type: 'string' },
+                host: { type: 'string' },
+                port: { type: 'integer' },
+                database: { type: 'string' },
+                status: { type: 'string', description: 'unknown / connected / error' },
+                lastError: { type: 'string' },
+                lastCheckedAt: { type: 'integer' },
+              },
+            },
+          },
+        },
+      },
+      render: (_args, value: { count: number; sources: SourceListItem[] }) => [
+        { type: 'text', text: renderSourceList(value.sources) },
+      ],
+    },
+    execute: async (): Promise<{ count: number; sources: SourceListItem[] }> => {
+      const records = await services.sources.list()
+      return { count: records.length, sources: records.map(toListItem) }
+    },
+  })
+}
+
+// ── datasource_test ──────────────────────────────────────────────────────────
+
+interface TestOutput {
+  sourceId: string
+  name: string
+  type: string
+  success: boolean
+  latency: number
+  version: string | null
+  error: string | null
+}
+
+export function createSourceTestTool(services: DataServices): ToolDefinition {
+  return toolDef({
+    name: 'datasource_test',
+    description: [
+      '测试一个已登记数据源的连通性，返回是否成功、延迟毫秒数与数据库版本。',
+      '失败时给出脱敏后的错误原因；检测结果会写回数据源的 status / lastError。',
+      '只读工具（不读业务数据），但会真正发起一次连接。',
+    ].join(' '),
+    parameters: { source: SOURCE_PARAM },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          sourceId: { type: 'string' },
+          name: { type: 'string' },
+          type: { type: 'string' },
+          success: { type: 'boolean' },
+          latency: { type: 'integer', description: '毫秒' },
+          version: { type: 'string' },
+          error: { type: 'string' },
+        },
+      },
+      render: (_args, value: TestOutput) => [
+        {
+          type: 'text',
+          text: renderConnectionTest({
+            name: value.name,
+            success: value.success,
+            latency: value.latency,
+            version: value.version,
+            error: value.error,
+          }),
+        },
+      ],
+    },
+    execute: async (args): Promise<TestOutput> => {
+      try {
+        const record = await services.sources.require(args.source)
+        const result: ConnectionTestResult = await testSource(services.cfg, record)
+        await services.sources.recordCheck(record.id, result.success, result.error)
+        return {
+          sourceId: record.id,
+          name: record.name,
+          type: record.type,
+          success: result.success,
+          latency: result.latency,
+          version: result.version,
+          error: result.error,
+        }
+      } catch (error: unknown) {
+        throw asToolFailure(error)
+      }
+    },
+  })
+}
+
+// ── datasource_tables ────────────────────────────────────────────────────────
+
+interface TablesOutput {
+  sourceId: string
+  name: string
+  schema: string | null
+  count: number
+  tables: {
+    tableName: string
+    schemaName: string | null
+    rowCount: number
+    primaryKey: string | null
+    columns: { name: string; type: string; nullable: boolean; description: string | null }[]
+  }[]
+}
+
+export function createSourceTablesTool(services: DataServices): ToolDefinition {
+  return toolDef({
+    name: 'datasource_tables',
+    description: [
+      '浏览一个数据源里可用的表：表名、估计行数、主键与列结构（列名 / 推断类型 / 是否可空 / 注释）。',
+      '可选 schema（PostgreSQL 默认 public，MySQL 无 schema 概念）；可选 q 按表名关键字过滤。',
+      '只读工具。看中某张表后用 datasource_import 把它导入当前工作区。',
+    ].join(' '),
+    parameters: {
+      source: SOURCE_PARAM,
+      schema: { type: 'string', description: '可选：schema 名；PostgreSQL 默认 public。' },
+      q: { type: 'string', description: '可选：按表名关键字过滤。' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          sourceId: { type: 'string' },
+          name: { type: 'string' },
+          schema: { type: 'string' },
+          count: { type: 'integer' },
+          tables: {
+            type: 'array',
+            items: {
+              type: 'object',
+              additionalProperties: false,
+              properties: {
+                tableName: { type: 'string' },
+                schemaName: { type: 'string' },
+                rowCount: { type: 'integer', description: '估计行数；-1 表示未知' },
+                primaryKey: { type: 'string' },
+                columns: COLUMN_SCHEMA,
+              },
+            },
+          },
+        },
+      },
+      render: (_args, value: TablesOutput) => {
+        const items: SourceTableItem[] = value.tables.map(table => ({
+          tableName: table.tableName,
+          schemaName: table.schemaName,
+          rowCount: table.rowCount,
+          columnCount: table.columns.length,
+          primaryKey: table.primaryKey,
+        }))
+        return [{ type: 'text', text: renderSourceTables(value.name, value.schema, items) }]
+      },
+    },
+    execute: async (args): Promise<TablesOutput> => {
+      try {
+        const record = await services.sources.require(args.source)
+        const connector = await connectSource(services.cfg, record)
+        const tables = await connector.getTables(args.schema)
+        const keyword = (args.q ?? '').trim().toLowerCase()
+        const matched = keyword.length === 0
+          ? tables
+          : tables.filter(table => table.tableName.toLowerCase().includes(keyword))
+        return {
+          sourceId: record.id,
+          name: record.name,
+          schema: args.schema ?? null,
+          count: matched.length,
+          tables: matched.map(table => ({
+            tableName: table.tableName,
+            schemaName: table.schemaName,
+            rowCount: table.rowCount,
+            primaryKey: table.primaryKey,
+            columns: table.columns.map(column => ({
+              name: column.name,
+              type: column.nativeType,
+              nullable: column.nullable,
+              description: column.comment,
+            })),
+          })),
+        }
+      } catch (error: unknown) {
+        throw asToolFailure(error)
+      }
+    },
+  })
+}
+
+// ── datasource_import ────────────────────────────────────────────────────────
+
+type ImportOutput = RemoteImportResult
+
+export function createSourceImportTool(services: DataServices): ToolDefinition {
+  return toolDef({
+    name: 'datasource_import',
+    description: [
+      '把远端数据源里的一张表全量导入当前工作区，产出标准数据集（之后用 dataset_* 工具操作）。',
+      'source 指定数据源，table 指定远端表名（PostgreSQL 可再加 schema）；name 指定登记名，缺省取表名。',
+      'limit 只导入前 N 行；大表会自动转后台任务并返回 jobId，完成后回注通知。',
+      '导入后的数据集与 Excel 导入的完全同权：dataset_query / dataset_insert 等照常使用。',
+      '这是写操作，会触发人工确认。',
+    ].join(' '),
+    parameters: {
+      source: SOURCE_PARAM,
+      table: { type: 'string', required: true, description: '远端表名（先用 datasource_tables 确认）。' },
+      schema: { type: 'string', description: '可选：PostgreSQL 的 schema 名。' },
+      name: { type: 'string', description: '可选：数据集登记名；默认取远端表名。' },
+      limit: { type: 'integer', description: '可选：只导入前 N 行。' },
+      background: { type: 'boolean', description: '可选：true 强制后台导入；默认大表自动转后台。' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          datasetId: { type: 'string' },
+          name: { type: 'string' },
+          rowCount: { type: 'integer' },
+          columnCount: { type: 'integer' },
+          status: { type: 'string', description: 'ready / running' },
+          jobId: { type: 'string' },
+          columns: {
+            type: 'array',
+            items: {
+              type: 'object',
+              additionalProperties: false,
+              properties: {
+                name: { type: 'string' },
+                sanitizedName: { type: 'string' },
+                type: { type: 'string' },
+                nullable: { type: 'boolean' },
+                description: { type: 'string' },
+                sample: { type: 'array', items: { type: 'json' } },
+              },
+            },
+          },
+        },
+      },
+      render: (_args, value: ImportOutput) => {
+        const head = value.status === 'running'
+          ? `已在后台开始导入 ${value.name}（datasetId: ${value.datasetId}，jobId: ${value.jobId ?? '-'}），完成后会回注通知。`
+          : `已导入 ${value.name}（datasetId: ${value.datasetId}）：${value.rowCount} 行 / ${value.columnCount} 列。`
+        return [{ type: 'text', text: [head, '', renderColumns(value.columns)].join('\n') }]
+      },
+    },
+    presentCall: args => ({
+      card: 'generic',
+      title: `导入远端表 ${args.table}`,
+      kind: 'write',
+    }),
+    execute: async (args, exec: ToolExec): Promise<ImportOutput> => {
+      if (services.cfg.readOnly) {
+        throw new ToolError('dsh-lh-data 处于只读模式（readOnly=true），已拒绝导入', 'READ_ONLY')
+      }
+      try {
+        const scope = await services.scopeOf(exec)
+        const record = await services.sources.require(args.source)
+        exec.signal.throwIfAborted()
+        const result = await importRemoteTable(services, {
+          scopeKey: scope.scopeKey,
+          source: record,
+          tableName: args.table,
+          schemaName: args.schema ?? null,
+          name: args.name ?? null,
+          limit: args.limit ?? null,
+        }, {
+          signal: exec.signal,
+          exec,
+          background: args.background === true,
+        })
+        return result
+      } catch (error: unknown) {
+        throw asToolFailure(error)
+      }
+    },
+  })
+}
+
+/** 供 registry 聚合。 */
+export function createDataSourceTools(services: DataServices): ToolDefinition[] {
+  return [
+    createSourceListTool(services),
+    createSourceTestTool(services),
+    createSourceTablesTool(services),
+    createSourceImportTool(services),
+  ]
+}

package/src/tools/import.ts

@@ -0,0 +1,282 @@
+/**
+ * `dataset_import`：解析工作区 Excel/CSV → 建表 → 批量插入 → 登记元数据。
+ *
+ * 大文件（≥ `backgroundThresholdRows`，或显式 `background: true`）走后台分支：
+ * 立即返回 `{ status: 'running', jobId }`，插入在 `ctx.jobs` 任务里跑完后回注通知
+ * （设计文档 §5.2）。任何失败都会删掉半截的物理表并把元数据标记为 `failed`。
+ */
+
+import { readFile } from 'node:fs/promises'
+import { basename, extname } from 'node:path'
+import { shortHash } from '../db'
+import { parseCSV, parseXLSX, type ColumnInfo, type ParsedFile } from '../parse'
+import { renderColumns } from '../render'
+import { assertReadableFile, resolveInputFile, type ScopeContext } from '../scope'
+import {
+  assertPhysicalTableName,
+  generateTableName,
+  makeDatasetId,
+  type DataServices,
+  type DatasetRecord,
+} from '../store'
+import { countRows, createDatasetTable, dropDatasetTable, insertRows } from '../table'
+import { ToolError, toolDef, PLUGIN_NAME, type ToolDefinition, type ToolExec } from '../tooling'
+
+export const IMPORT_JOB_KIND = 'dataset-import'
+
+export interface ImportOutput {
+  datasetId: string
+  name: string
+  rowCount: number
+  columnCount: number
+  status: string
+  jobId?: string
+  columns: ColumnInfo[]
+}
+
+/** 后台任务的终态（对应 dsh-jobs 的 `JobOutcome`）。 */
+interface JobOutcome {
+  status: 'completed' | 'killed' | 'failed'
+  detail?: string
+  output?: string
+}
+
+function parseFile(path: string, buffer: Buffer, sheet: string | undefined, sampleRows: number): ParsedFile {
+  return extname(path).toLowerCase() === '.csv'
+    ? parseCSV(buffer, sampleRows)
+    : parseXLSX(buffer, sheet, sampleRows)
+}
+
+/** 失败清理：删掉半截物理表 + 元数据标记 failed（后续工具会拒绝非 ready 的数据集）。 */
+async function cleanupFailedImport(
+  services: DataServices,
+  scope: ScopeContext,
+  record: DatasetRecord,
+  message: string,
+): Promise<void> {
+  try {
+    const db = await services.store.database(scope.scopeKey)
+    await dropDatasetTable(db, record.tableName)
+  } catch {
+    // 清理失败不应掩盖原始错误。
+  }
+  try {
+    await services.store.update(scope.scopeKey, record.id, {
+      status: 'failed',
+      error: message.slice(0, 500),
+      rowCount: 0,
+    })
+  } catch {
+    // 同上。
+  }
+}
+
+/** 后台导入：注册任务并返回 jobId；不可用时返回 undefined（降级为前台）。 */
+function startBackgroundImport(
+  services: DataServices,
+  exec: ToolExec,
+  scope: ScopeContext,
+  record: DatasetRecord,
+  columns: ColumnInfo[],
+  rows: Record<string, unknown>[],
+): string | undefined {
+  const jobs = services.jobs
+  if (jobs === undefined) return undefined
+
+  const controller = new AbortController()
+  const notify = (text: string): void => {
+    try {
+      exec.agent?.inject?.({
+        role: 'user',
+        content: [{ type: 'text', text }],
+        source: { kind: 'plugin', plugin: PLUGIN_NAME },
+      })
+    } catch {
+      // agent 已 dispose（会话结束）时静默跳过，不能让通知失败影响任务本身。
+    }
+  }
+
+  try {
+    return jobs.start({
+      kind: IMPORT_JOB_KIND,
+      label: `导入 ${record.name}（${rows.length} 行）`,
+      owner: exec.agent,
+      run: () => {
+        // 任务自有的取消信号：外层调用被取消不会终止已发布的工作。
+        const done = (async (): Promise<JobOutcome> => {
+          try {
+            const db = await services.store.database(scope.scopeKey)
+            const inserted = await insertRows(db, record.tableName, columns, rows, {
+              batchSize: services.cfg.batchSize,
+              signal: controller.signal,
+            })
+            await services.store.update(scope.scopeKey, record.id, { rowCount: inserted, status: 'ready' })
+            const text = `数据集导入完成：${record.name}（${inserted} 行，datasetId: ${record.id}）`
+            notify(text)
+            return { status: 'completed', detail: `${inserted} rows`, output: text }
+          } catch (error: unknown) {
+            const message = error instanceof Error ? error.message : String(error)
+            await cleanupFailedImport(services, scope, record, message)
+            const aborted = controller.signal.aborted
+            const text = `数据集导入${aborted ? '已取消' : '失败'}：${record.name} — ${message}`
+            notify(text)
+            return { status: aborted ? 'killed' : 'failed', detail: message.slice(0, 500), output: text }
+          }
+        })()
+        return {
+          cancel: (reason?: string) => controller.abort(reason),
+          done,
+        }
+      },
+    })
+  } catch {
+    // 任务注册表不可用/预检失败：降级为前台导入。
+    return undefined
+  }
+}
+
+export function createImportTool(services: DataServices): ToolDefinition {
+  return toolDef({
+    name: 'dataset_import',
+    description: [
+      '把工作区内的 Excel/CSV 导入本地表格库：解析文件、建表、批量插入并登记为数据集。',
+      'path 相对路径以当前工作区目录为基准，且必须位于工作区内（禁止 ../ 穿越）；仅支持 .xlsx / .xls / .csv。',
+      '可选：name 指定登记名（重复会自动加后缀）、sheet 指定工作表名（默认第一个）、limit 只导入前 N 行。',
+      '大文件会自动转后台任务并立即返回 jobId，完成后回注通知；小文件同步完成并返回行数与列信息。',
+      '这是写操作，会触发人工确认。导入后用 dataset_list / dataset_schema / dataset_query 查看。',
+    ].join(' '),
+    parameters: {
+      path: {
+        type: 'string',
+        required: true,
+        description: '工作区内的文件绝对路径，或相对当前工作区目录的相对路径。',
+      },
+      name: { type: 'string', description: '可选：数据集登记名；默认取文件名（不含扩展名）。' },
+      sheet: { type: 'string', description: '可选：Excel 工作表名；默认第一个工作表。' },
+      limit: { type: 'integer', description: '可选：只导入前 N 行。' },
+      background: { type: 'boolean', description: '可选：true 强制后台导入；默认大文件自动转后台。' },
+    },
+    output: {
+      schema: {
+        type: 'object',
+        additionalProperties: false,
+        properties: {
+          datasetId: { type: 'string' },
+          name: { type: 'string' },
+          rowCount: { type: 'integer' },
+          columnCount: { type: 'integer' },
+          status: { type: 'string', description: 'ready / running' },
+          jobId: { type: 'string' },
+          columns: {
+            type: 'array',
+            items: {
+              type: 'object',
+              additionalProperties: false,
+              properties: {
+                name: { type: 'string' },
+                sanitizedName: { type: 'string' },
+                type: { type: 'string' },
+                nullable: { type: 'boolean' },
+                description: { type: 'string' },
+                sample: { type: 'array', items: { type: 'json' } },
+              },
+            },
+          },
+        },
+      },
+      render: (_args, value: ImportOutput) => {
+        const head = value.status === 'running'
+          ? `已在后台开始导入 ${value.name}（datasetId: ${value.datasetId}，jobId: ${value.jobId ?? '-'}），完成后会回注通知。`
+          : `已导入 ${value.name}（datasetId: ${value.datasetId}）：${value.rowCount} 行 / ${value.columnCount} 列。`
+        return [
+          {
+            type: 'text',
+            text: [head, '', renderColumns(value.columns)].join('\n'),
+          },
+        ]
+      },
+    },
+    presentCall: args => ({
+      card: 'generic',
+      title: `导入 ${basename(args.path)}`,
+      kind: 'write',
+      locations: [{ path: args.path }],
+    }),
+    execute: async (args, exec: ToolExec): Promise<ImportOutput> => {
+      if (services.cfg.readOnly) {
+        throw new ToolError('dsh-lh-data 处于只读模式（readOnly=true），已拒绝导入', 'READ_ONLY')
+      }
+      const scope = await services.scopeOf(exec)
+      const absolutePath = resolveInputFile(scope.cwd, args.path)
+      await assertReadableFile(absolutePath, services.cfg.maxFileBytes)
+      exec.signal.throwIfAborted()
+
+      const buffer = await readFile(absolutePath)
+      exec.signal.throwIfAborted()
+      const parsed = parseFile(absolutePath, buffer, args.sheet, services.cfg.previewSampleRows)
+      if (parsed.columns.length === 0) throw new ToolError('文件没有可识别的列（表头为空？）', 'EMPTY_COLUMNS')
+      const rows = args.limit !== undefined && args.limit > 0 ? parsed.rows.slice(0, args.limit) : parsed.rows
+
+      const base = (args.name?.trim().length ?? 0) > 0 ? args.name!.trim() : basename(absolutePath, extname(absolutePath))
+      const name = await services.store.uniqueName(scope.scopeKey, base)
+      const tableName = generateTableName(base, shortHash(scope.scopeKey))
+      assertPhysicalTableName(tableName)
+
+      const now = Date.now()
+      const record: DatasetRecord = {
+        id: makeDatasetId(),
+        scopeKey: scope.scopeKey,
+        name,
+        tableName,
+        sourcePath: absolutePath,
+        sourceId: null,
+        sourceRef: null,
+        description: null,
+        rowCount: 0,
+        columns: parsed.columns,
+        status: 'importing',
+        error: null,
+        createdAt: now,
+        updatedAt: now,
+      }
+      await services.store.create(record)
+      const db = await services.store.database(scope.scopeKey)
+
+      try {
+        await createDatasetTable(db, tableName, parsed.columns)
+        const wantsBackground = args.background === true || rows.length >= services.cfg.backgroundThresholdRows
+        if (wantsBackground) {
+          const jobId = startBackgroundImport(services, exec, scope, record, parsed.columns, rows)
+          if (jobId !== undefined) {
+            return {
+              datasetId: record.id,
+              name,
+              rowCount: 0,
+              columnCount: parsed.columns.length,
+              status: 'running',
+              jobId,
+              columns: parsed.columns,
+            }
+          }
+        }
+        const inserted = await insertRows(db, tableName, parsed.columns, rows, {
+          batchSize: services.cfg.batchSize,
+          signal: exec.signal,
+        })
+        const rowCount = await countRows(db, tableName)
+        await services.store.update(scope.scopeKey, record.id, { rowCount: inserted, status: 'ready' })
+        return {
+          datasetId: record.id,
+          name,
+          rowCount: Number.isFinite(rowCount) ? rowCount : inserted,
+          columnCount: parsed.columns.length,
+          status: 'ready',
+          columns: parsed.columns,
+        }
+      } catch (error: unknown) {
+        await cleanupFailedImport(services, scope, record, error instanceof Error ? error.message : String(error))
+        throw error
+      }
+    },
+  })
+}