dsh-lh-data 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +332 -0
- package/cordis.patch.yml +7 -0
- package/lib/admin-contract.d.ts +274 -0
- package/lib/admin-http.d.ts +31 -0
- package/lib/admin-validate.d.ts +87 -0
- package/lib/admin.d.ts +60 -0
- package/lib/client.js +2969 -0
- package/lib/client.js.map +1 -0
- package/lib/datasource/columns.d.ts +13 -0
- package/lib/datasource/connection.d.ts +33 -0
- package/lib/datasource/connector/base.d.ts +21 -0
- package/lib/datasource/connector/mysql.d.ts +23 -0
- package/lib/datasource/connector/postgresql.d.ts +23 -0
- package/lib/datasource/crypto.d.ts +13 -0
- package/lib/datasource/driver.d.ts +39 -0
- package/lib/datasource/errors.d.ts +25 -0
- package/lib/datasource/importer.d.ts +38 -0
- package/lib/datasource/source-sql.d.ts +11 -0
- package/lib/datasource/source-store.d.ts +28 -0
- package/lib/datasource/types.d.ts +119 -0
- package/lib/db.d.ts +91 -0
- package/lib/http-common.d.ts +31 -0
- package/lib/http.d.ts +13 -0
- package/lib/index.d.ts +50 -0
- package/lib/index.js +5735 -0
- package/lib/parse.d.ts +41 -0
- package/lib/preview.d.ts +75 -0
- package/lib/render.d.ts +100 -0
- package/lib/scope-registry.d.ts +47 -0
- package/lib/scope.d.ts +56 -0
- package/lib/sql.d.ts +133 -0
- package/lib/store.d.ts +174 -0
- package/lib/table.d.ts +38 -0
- package/lib/tooling.d.ts +267 -0
- package/lib/tools/datasource.d.ts +16 -0
- package/lib/tools/import.d.ts +21 -0
- package/lib/tools/read.d.ts +77 -0
- package/lib/tools/registry.d.ts +12 -0
- package/lib/tools/write.d.ts +37 -0
- package/lib/view.d.ts +138 -0
- package/package.json +61 -0
- package/src/admin-contract.ts +351 -0
- package/src/admin-http.ts +254 -0
- package/src/admin-validate.ts +393 -0
- package/src/admin.ts +558 -0
- package/src/client/index.ts +403 -0
- package/src/client/settings/CreateForm.tsx +186 -0
- package/src/client/settings/DataSourceForm.tsx +278 -0
- package/src/client/settings/DataSourcesPanel.tsx +301 -0
- package/src/client/settings/DatasetEditor.tsx +245 -0
- package/src/client/settings/DatasetTable.tsx +110 -0
- package/src/client/settings/DatasetsPanel.tsx +207 -0
- package/src/client/settings/RowsPanel.tsx +196 -0
- package/src/client/settings/Section.tsx +41 -0
- package/src/client/settings/SourceTablesPanel.tsx +226 -0
- package/src/client/settings/api.ts +154 -0
- package/src/client/settings/styles.ts +125 -0
- package/src/datasource/columns.ts +56 -0
- package/src/datasource/connection.ts +124 -0
- package/src/datasource/connector/base.ts +54 -0
- package/src/datasource/connector/mysql.ts +187 -0
- package/src/datasource/connector/postgresql.ts +212 -0
- package/src/datasource/crypto.ts +61 -0
- package/src/datasource/driver.ts +108 -0
- package/src/datasource/errors.ts +69 -0
- package/src/datasource/importer.ts +262 -0
- package/src/datasource/index.ts +62 -0
- package/src/datasource/source-sql.ts +64 -0
- package/src/datasource/source-store.ts +152 -0
- package/src/datasource/types.ts +133 -0
- package/src/db.ts +277 -0
- package/src/http-common.ts +91 -0
- package/src/http.ts +130 -0
- package/src/index.ts +486 -0
- package/src/parse.ts +213 -0
- package/src/preview.ts +198 -0
- package/src/render.ts +294 -0
- package/src/scope-registry.ts +111 -0
- package/src/scope.ts +159 -0
- package/src/sql.ts +491 -0
- package/src/store.ts +412 -0
- package/src/table.ts +160 -0
- package/src/tooling.ts +551 -0
- package/src/tools/datasource.ts +378 -0
- package/src/tools/import.ts +282 -0
- package/src/tools/read.ts +536 -0
- package/src/tools/registry.ts +56 -0
- package/src/tools/write.ts +241 -0
- package/src/view.ts +371 -0
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 远端表 → 本地数据集的导入。
|
|
3
|
+
*
|
|
4
|
+
* 落库完全复用文件导入的那一套(`createDatasetTable` / `insertRows` / `DatasetStore`),
|
|
5
|
+
* 因此导进来的数据集与 Excel 导入的**结构完全一致**,八个 `dataset_*` 工具零改动可用。
|
|
6
|
+
* 远端表名只出现在发往远端的 SQL 里,本地表名仍走 `generateTableName` 白名单。
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { shortHash, type Database } from '../db'
|
|
10
|
+
import { fillSamples, buildColumns, mapRowKeys } from './columns'
|
|
11
|
+
import { connectSource } from './connection'
|
|
12
|
+
import type { DatabaseConnector } from './connector/base'
|
|
13
|
+
import { DataSourceError } from './errors'
|
|
14
|
+
import type { ColumnInfo } from '../parse'
|
|
15
|
+
import {
|
|
16
|
+
assertPhysicalTableName,
|
|
17
|
+
generateTableName,
|
|
18
|
+
makeDatasetId,
|
|
19
|
+
type DataServices,
|
|
20
|
+
type DatasetRecord,
|
|
21
|
+
} from '../store'
|
|
22
|
+
import { countRows, createDatasetTable, dropDatasetTable, insertRows } from '../table'
|
|
23
|
+
import { PLUGIN_NAME, type ToolExec } from '../tooling'
|
|
24
|
+
import type { DataSourceRecord } from './types'
|
|
25
|
+
|
|
26
|
+
export const REMOTE_IMPORT_JOB_KIND = 'datasource-import'
|
|
27
|
+
|
|
28
|
+
/** 采样式预览:只为了给设置页与 `dataset_schema` 填样例值。 */
|
|
29
|
+
const SAMPLE_ROWS = 5
|
|
30
|
+
|
|
31
|
+
export interface RemoteImportParams {
|
|
32
|
+
scopeKey: string
|
|
33
|
+
source: DataSourceRecord
|
|
34
|
+
tableName: string
|
|
35
|
+
schemaName?: string | null
|
|
36
|
+
/** 数据集登记名;缺省取远端表名。 */
|
|
37
|
+
name?: string | null
|
|
38
|
+
/** 只导入前 N 行;配合 `datasourceMaxImportRows` 双重夹紧。 */
|
|
39
|
+
limit?: number | null
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface RemoteImportContext {
|
|
43
|
+
signal: AbortSignal
|
|
44
|
+
/** 有 agent 才能回注后台任务的完成通知;设置页导入时传 undefined。 */
|
|
45
|
+
exec?: ToolExec
|
|
46
|
+
background?: boolean
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface RemoteImportResult {
|
|
50
|
+
datasetId: string
|
|
51
|
+
name: string
|
|
52
|
+
rowCount: number
|
|
53
|
+
columnCount: number
|
|
54
|
+
columns: ColumnInfo[]
|
|
55
|
+
status: 'ready' | 'running'
|
|
56
|
+
jobId?: string
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
interface PreparedImport {
|
|
60
|
+
scopeKey: string
|
|
61
|
+
record: DatasetRecord
|
|
62
|
+
db: Database
|
|
63
|
+
columns: ColumnInfo[]
|
|
64
|
+
connector: DatabaseConnector
|
|
65
|
+
remoteTable: string
|
|
66
|
+
remoteSchema: string | null
|
|
67
|
+
/** 0 表示不限。 */
|
|
68
|
+
maxRows: number
|
|
69
|
+
estimate: number
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function resolveMaxRows(services: DataServices, limit?: number | null): number {
|
|
73
|
+
const ceiling = services.cfg.datasourceMaxImportRows
|
|
74
|
+
const requested = limit !== undefined && limit !== null && limit > 0 ? Math.trunc(limit) : 0
|
|
75
|
+
if (requested === 0) return ceiling > 0 ? ceiling : 0
|
|
76
|
+
return ceiling > 0 ? Math.min(requested, ceiling) : requested
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** 远端的脱敏定位串:`<schema>.<table>`(MySQL 无 schema 时为 `<table>`)。 */
|
|
80
|
+
function sourceRefOf(params: RemoteImportParams): string {
|
|
81
|
+
const schema = params.schemaName?.trim() ?? ''
|
|
82
|
+
return schema.length > 0 ? `${schema}.${params.tableName}` : params.tableName
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function prepareImport(services: DataServices, params: RemoteImportParams): Promise<PreparedImport> {
|
|
86
|
+
const connector = await connectSource(services.cfg, params.source)
|
|
87
|
+
const schema = params.schemaName?.trim() || undefined
|
|
88
|
+
const tables = await connector.getTables(schema)
|
|
89
|
+
const remote = tables.find(table => table.tableName === params.tableName)
|
|
90
|
+
if (remote === undefined) {
|
|
91
|
+
throw new DataSourceError('NOT_FOUND', `数据源「${params.source.name}」里没有表 ${params.tableName}`)
|
|
92
|
+
}
|
|
93
|
+
if (remote.columns.length === 0) {
|
|
94
|
+
throw new DataSourceError('BAD_REQUEST', `表 ${params.tableName} 没有可导入的列`)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const columns = buildColumns(remote.columns)
|
|
98
|
+
const preview = await connector.getTableData(params.tableName, remote.schemaName ?? undefined, { limit: SAMPLE_ROWS })
|
|
99
|
+
fillSamples(columns, preview)
|
|
100
|
+
|
|
101
|
+
const base = params.name?.trim() || params.tableName
|
|
102
|
+
const now = Date.now()
|
|
103
|
+
const record: DatasetRecord = {
|
|
104
|
+
id: makeDatasetId(),
|
|
105
|
+
scopeKey: params.scopeKey,
|
|
106
|
+
name: await services.store.uniqueName(params.scopeKey, base),
|
|
107
|
+
tableName: generateTableName(base, shortHash(params.scopeKey)),
|
|
108
|
+
sourcePath: `db:${params.source.name}`,
|
|
109
|
+
sourceId: params.source.id,
|
|
110
|
+
sourceRef: sourceRefOf(params),
|
|
111
|
+
description: null,
|
|
112
|
+
rowCount: 0,
|
|
113
|
+
columns,
|
|
114
|
+
status: 'importing',
|
|
115
|
+
error: null,
|
|
116
|
+
createdAt: now,
|
|
117
|
+
updatedAt: now,
|
|
118
|
+
}
|
|
119
|
+
assertPhysicalTableName(record.tableName)
|
|
120
|
+
await services.store.create(record)
|
|
121
|
+
const db = await services.store.database(params.scopeKey)
|
|
122
|
+
await createDatasetTable(db, record.tableName, columns)
|
|
123
|
+
return {
|
|
124
|
+
scopeKey: params.scopeKey,
|
|
125
|
+
record,
|
|
126
|
+
db,
|
|
127
|
+
columns,
|
|
128
|
+
connector,
|
|
129
|
+
remoteTable: params.tableName,
|
|
130
|
+
remoteSchema: remote.schemaName,
|
|
131
|
+
maxRows: resolveMaxRows(services, params.limit),
|
|
132
|
+
estimate: remote.rowCount,
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** 分块拉取 → 键改写 → 批量插入,直到拉空或达到上限。 */
|
|
137
|
+
async function pullAndInsert(services: DataServices, prepared: PreparedImport, signal: AbortSignal): Promise<number> {
|
|
138
|
+
const fetchSize = Math.max(1, services.cfg.datasourceFetchBatchSize)
|
|
139
|
+
let inserted = 0
|
|
140
|
+
let offset = 0
|
|
141
|
+
for (;;) {
|
|
142
|
+
signal.throwIfAborted()
|
|
143
|
+
const remaining = prepared.maxRows > 0 ? prepared.maxRows - inserted : fetchSize
|
|
144
|
+
const limit = Math.min(fetchSize, remaining > 0 ? remaining : 0)
|
|
145
|
+
if (limit <= 0) break
|
|
146
|
+
const rows = await prepared.connector.getTableData(prepared.remoteTable, prepared.remoteSchema ?? undefined, { limit, offset })
|
|
147
|
+
if (rows.length === 0) break
|
|
148
|
+
await insertRows(prepared.db, prepared.record.tableName, prepared.columns, mapRowKeys(rows, prepared.columns), {
|
|
149
|
+
batchSize: services.cfg.batchSize,
|
|
150
|
+
signal,
|
|
151
|
+
})
|
|
152
|
+
inserted += rows.length
|
|
153
|
+
offset += rows.length
|
|
154
|
+
if (rows.length < limit) break
|
|
155
|
+
}
|
|
156
|
+
return inserted
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function finalize(services: DataServices, prepared: PreparedImport, inserted: number): Promise<number> {
|
|
160
|
+
const rowCount = await countRows(prepared.db, prepared.record.tableName)
|
|
161
|
+
const total = Number.isFinite(rowCount) ? rowCount : inserted
|
|
162
|
+
await services.store.update(prepared.scopeKey, prepared.record.id, { rowCount: total, status: 'ready' })
|
|
163
|
+
return total
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async function cleanupFailedImport(services: DataServices, prepared: PreparedImport, message: string): Promise<void> {
|
|
167
|
+
try {
|
|
168
|
+
await dropDatasetTable(prepared.db, prepared.record.tableName)
|
|
169
|
+
} catch {
|
|
170
|
+
// 清理失败不应掩盖原始错误。
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
await services.store.update(prepared.scopeKey, prepared.record.id, {
|
|
174
|
+
status: 'failed',
|
|
175
|
+
error: message.slice(0, 500),
|
|
176
|
+
rowCount: 0,
|
|
177
|
+
})
|
|
178
|
+
} catch {
|
|
179
|
+
// 同上。
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function startBackgroundImport(services: DataServices, exec: ToolExec, prepared: PreparedImport): string | undefined {
|
|
184
|
+
const jobs = services.jobs
|
|
185
|
+
if (jobs === undefined) return undefined
|
|
186
|
+
const controller = new AbortController()
|
|
187
|
+
const notify = (text: string): void => {
|
|
188
|
+
try {
|
|
189
|
+
exec.agent?.inject?.({ role: 'user', content: [{ type: 'text', text }], source: { kind: 'plugin', plugin: PLUGIN_NAME } })
|
|
190
|
+
} catch {
|
|
191
|
+
// agent 已 dispose 时静默跳过。
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
try {
|
|
195
|
+
return jobs.start({
|
|
196
|
+
kind: REMOTE_IMPORT_JOB_KIND,
|
|
197
|
+
label: `导入远端表 ${prepared.remoteTable} → ${prepared.record.name}`,
|
|
198
|
+
owner: exec.agent,
|
|
199
|
+
run: () => {
|
|
200
|
+
const done = (async (): Promise<{ status: string; detail: string; output: string }> => {
|
|
201
|
+
try {
|
|
202
|
+
const inserted = await pullAndInsert(services, prepared, controller.signal)
|
|
203
|
+
const total = await finalize(services, prepared, inserted)
|
|
204
|
+
const text = `远端表导入完成:${prepared.record.name}(${total} 行,datasetId: ${prepared.record.id})`
|
|
205
|
+
notify(text)
|
|
206
|
+
return { status: 'completed', detail: `${total} rows`, output: text }
|
|
207
|
+
} catch (error: unknown) {
|
|
208
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
209
|
+
await cleanupFailedImport(services, prepared, message)
|
|
210
|
+
const aborted = controller.signal.aborted
|
|
211
|
+
const text = `远端表导入${aborted ? '已取消' : '失败'}:${prepared.record.name} — ${message}`
|
|
212
|
+
notify(text)
|
|
213
|
+
return { status: aborted ? 'killed' : 'failed', detail: message.slice(0, 500), output: text }
|
|
214
|
+
}
|
|
215
|
+
})()
|
|
216
|
+
return { cancel: (reason?: string) => controller.abort(reason), done }
|
|
217
|
+
},
|
|
218
|
+
})
|
|
219
|
+
} catch {
|
|
220
|
+
return undefined
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
export async function importRemoteTable(
|
|
225
|
+
services: DataServices,
|
|
226
|
+
params: RemoteImportParams,
|
|
227
|
+
context: RemoteImportContext,
|
|
228
|
+
): Promise<RemoteImportResult> {
|
|
229
|
+
const prepared = await prepareImport(services, params)
|
|
230
|
+
try {
|
|
231
|
+
const wantsBackground = context.background === true
|
|
232
|
+
|| (context.exec !== undefined && prepared.estimate >= services.cfg.backgroundThresholdRows)
|
|
233
|
+
if (wantsBackground && context.exec !== undefined) {
|
|
234
|
+
const jobId = startBackgroundImport(services, context.exec, prepared)
|
|
235
|
+
if (jobId !== undefined) {
|
|
236
|
+
return {
|
|
237
|
+
datasetId: prepared.record.id,
|
|
238
|
+
name: prepared.record.name,
|
|
239
|
+
rowCount: 0,
|
|
240
|
+
columnCount: prepared.columns.length,
|
|
241
|
+
columns: prepared.columns,
|
|
242
|
+
status: 'running',
|
|
243
|
+
jobId,
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
const inserted = await pullAndInsert(services, prepared, context.signal)
|
|
248
|
+
const total = await finalize(services, prepared, inserted)
|
|
249
|
+
return {
|
|
250
|
+
datasetId: prepared.record.id,
|
|
251
|
+
name: prepared.record.name,
|
|
252
|
+
rowCount: total,
|
|
253
|
+
columnCount: prepared.columns.length,
|
|
254
|
+
columns: prepared.columns,
|
|
255
|
+
status: 'ready',
|
|
256
|
+
}
|
|
257
|
+
} catch (error: unknown) {
|
|
258
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
259
|
+
await cleanupFailedImport(services, prepared, message)
|
|
260
|
+
throw error
|
|
261
|
+
}
|
|
262
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 数据源模块对外入口。
|
|
3
|
+
*
|
|
4
|
+
* 主机侧(工具 / 管理接口)只从这里导入;连接器与驱动加载属于内部实现,不直接暴露。
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
// 错误
|
|
8
|
+
export { DataSourceError, dataSourceErrorStatus, isDataSourceError, type DataSourceErrorCode } from './errors'
|
|
9
|
+
|
|
10
|
+
// 类型
|
|
11
|
+
export type {
|
|
12
|
+
ConnectionConfig,
|
|
13
|
+
ConnectionTestResult,
|
|
14
|
+
DataSourceConfig,
|
|
15
|
+
DataSourceInput,
|
|
16
|
+
DataSourcePatch,
|
|
17
|
+
DataSourceRecord,
|
|
18
|
+
DataSourceStatus,
|
|
19
|
+
DataSourceType,
|
|
20
|
+
FetchRange,
|
|
21
|
+
ImportRequest,
|
|
22
|
+
RemoteColumn,
|
|
23
|
+
RemoteTable,
|
|
24
|
+
SchemaSummary,
|
|
25
|
+
} from './types'
|
|
26
|
+
|
|
27
|
+
// 加解密
|
|
28
|
+
export { decryptPassword, encryptPassword, isValidEncryptedFormat, resolveEncryptKey, usingDefaultEncryptKey } from './crypto'
|
|
29
|
+
|
|
30
|
+
// 驱动
|
|
31
|
+
export { DRIVER_INSTALL_HINTS } from './driver'
|
|
32
|
+
|
|
33
|
+
// 连接
|
|
34
|
+
export {
|
|
35
|
+
closeAllConnectors,
|
|
36
|
+
closeConnector,
|
|
37
|
+
connectSource,
|
|
38
|
+
createConnector,
|
|
39
|
+
DEFAULT_PORTS,
|
|
40
|
+
testDraft,
|
|
41
|
+
testSource,
|
|
42
|
+
type ConnectionDraft,
|
|
43
|
+
} from './connection'
|
|
44
|
+
|
|
45
|
+
// 连接器基类
|
|
46
|
+
export { DatabaseConnector } from './connector/base'
|
|
47
|
+
|
|
48
|
+
// 持久化
|
|
49
|
+
export { createSourceStore, DataSourceStore } from './source-store'
|
|
50
|
+
export { makeSourceId, SOURCE_SCHEMA_SQL } from './source-sql'
|
|
51
|
+
|
|
52
|
+
// 列映射
|
|
53
|
+
export { buildColumns, fillSamples, mapRowKeys } from './columns'
|
|
54
|
+
|
|
55
|
+
// 导入
|
|
56
|
+
export {
|
|
57
|
+
importRemoteTable,
|
|
58
|
+
REMOTE_IMPORT_JOB_KIND,
|
|
59
|
+
type RemoteImportContext,
|
|
60
|
+
type RemoteImportParams,
|
|
61
|
+
type RemoteImportResult,
|
|
62
|
+
} from './importer'
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `lh_data_sources` 的 DDL 与行映射。
|
|
3
|
+
*
|
|
4
|
+
* 数据源是**全局**的(catalog 库,跨工作区共享),与 `dataset_scopes` 同库。
|
|
5
|
+
* DDL 与行映射单独放这里,`source-store.ts` 只留 CRUD,避免类膨胀。
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { Row } from '../db'
|
|
9
|
+
import type { DataSourceRecord, DataSourceStatus, DataSourceType } from './types'
|
|
10
|
+
|
|
11
|
+
export const SOURCE_SCHEMA_SQL = `
|
|
12
|
+
CREATE TABLE IF NOT EXISTS lh_data_sources (
|
|
13
|
+
id TEXT PRIMARY KEY,
|
|
14
|
+
name TEXT NOT NULL,
|
|
15
|
+
type TEXT NOT NULL,
|
|
16
|
+
host TEXT NOT NULL,
|
|
17
|
+
port INTEGER NOT NULL,
|
|
18
|
+
database_name TEXT NOT NULL,
|
|
19
|
+
username TEXT NOT NULL,
|
|
20
|
+
password_enc TEXT NOT NULL,
|
|
21
|
+
ssl_mode TEXT,
|
|
22
|
+
pool_max INTEGER,
|
|
23
|
+
description TEXT,
|
|
24
|
+
status TEXT NOT NULL DEFAULT 'unknown',
|
|
25
|
+
last_error TEXT,
|
|
26
|
+
last_checked_at INTEGER,
|
|
27
|
+
created_at INTEGER NOT NULL,
|
|
28
|
+
updated_at INTEGER NOT NULL
|
|
29
|
+
);
|
|
30
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_lh_data_sources_name ON lh_data_sources(name);
|
|
31
|
+
`
|
|
32
|
+
|
|
33
|
+
export function makeSourceId(): string {
|
|
34
|
+
return `dsrc_${Date.now().toString(36)}${Math.random().toString(36).slice(2, 8)}`
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function nullableString(value: unknown): string | null {
|
|
38
|
+
return value === null || value === undefined ? null : String(value)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function nullableNumber(value: unknown): number | null {
|
|
42
|
+
return value === null || value === undefined ? null : Number(value)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function rowToSource(row: Row): DataSourceRecord {
|
|
46
|
+
return {
|
|
47
|
+
id: String(row.id),
|
|
48
|
+
name: String(row.name),
|
|
49
|
+
type: String(row.type ?? 'mysql') as DataSourceType,
|
|
50
|
+
host: String(row.host ?? ''),
|
|
51
|
+
port: Number(row.port ?? 0),
|
|
52
|
+
database: String(row.database_name ?? ''),
|
|
53
|
+
username: String(row.username ?? ''),
|
|
54
|
+
passwordEnc: String(row.password_enc ?? ''),
|
|
55
|
+
sslMode: nullableString(row.ssl_mode),
|
|
56
|
+
poolMax: nullableNumber(row.pool_max),
|
|
57
|
+
description: nullableString(row.description),
|
|
58
|
+
status: String(row.status ?? 'unknown') as DataSourceStatus,
|
|
59
|
+
lastError: nullableString(row.last_error),
|
|
60
|
+
lastCheckedAt: nullableNumber(row.last_checked_at),
|
|
61
|
+
createdAt: Number(row.created_at ?? 0),
|
|
62
|
+
updatedAt: Number(row.updated_at ?? 0),
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 数据源的持久化:catalog 库 `lh_data_sources` 的 CRUD。
|
|
3
|
+
*
|
|
4
|
+
* 密码在这一层加密落库;改 / 删数据源会顺带关掉对应的连接池,避免缓存里残留
|
|
5
|
+
* 旧凭据。`list()` / `find()` 返回的是含密文的完整记录——**绝不能直接序列化
|
|
6
|
+
* 给前端或模型**,边界处必须过 `toView()` / 工具侧的脱敏。
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { resolveCatalogDatabase, type Database, type DbConfig } from '../db'
|
|
10
|
+
import { closeConnector } from './connection'
|
|
11
|
+
import { encryptPassword, resolveEncryptKey } from './crypto'
|
|
12
|
+
import { DataSourceError } from './errors'
|
|
13
|
+
import { makeSourceId, rowToSource, SOURCE_SCHEMA_SQL } from './source-sql'
|
|
14
|
+
import type { DataSourceConfig, DataSourceInput, DataSourcePatch, DataSourceRecord } from './types'
|
|
15
|
+
|
|
16
|
+
export class DataSourceStore {
|
|
17
|
+
private initialized = false
|
|
18
|
+
|
|
19
|
+
constructor(private readonly cfg: DataSourceConfig & DbConfig) {}
|
|
20
|
+
|
|
21
|
+
private async db(): Promise<Database> {
|
|
22
|
+
const database = resolveCatalogDatabase(this.cfg)
|
|
23
|
+
if (!this.initialized) {
|
|
24
|
+
await database.exec(SOURCE_SCHEMA_SQL)
|
|
25
|
+
this.initialized = true
|
|
26
|
+
}
|
|
27
|
+
return database
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async list(): Promise<DataSourceRecord[]> {
|
|
31
|
+
const rows = await (await this.db()).prepare('SELECT * FROM lh_data_sources ORDER BY name ASC').all()
|
|
32
|
+
return rows.map(rowToSource)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** 按 id 或登记名查找;找不到返回 undefined(不抛错)。 */
|
|
36
|
+
async find(reference: string): Promise<DataSourceRecord | undefined> {
|
|
37
|
+
const key = reference.trim()
|
|
38
|
+
if (key.length === 0) return undefined
|
|
39
|
+
const row = await (await this.db())
|
|
40
|
+
.prepare('SELECT * FROM lh_data_sources WHERE id = ? OR name = ? LIMIT 1')
|
|
41
|
+
.get(key, key)
|
|
42
|
+
return row === undefined ? undefined : rowToSource(row)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async require(reference: string): Promise<DataSourceRecord> {
|
|
46
|
+
const record = await this.find(reference)
|
|
47
|
+
if (record === undefined) {
|
|
48
|
+
throw new DataSourceError('NOT_FOUND', `未找到数据源:${reference}(先用 datasource_list 查看可用数据源)`)
|
|
49
|
+
}
|
|
50
|
+
return record
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** 全局唯一的名字:已存在则追加 `_2` / `_3`。 */
|
|
54
|
+
async uniqueName(name: string): Promise<string> {
|
|
55
|
+
const base = name.trim().length > 0 ? name.trim() : 'source'
|
|
56
|
+
const taken = new Set((await this.list()).map(record => record.name))
|
|
57
|
+
if (!taken.has(base)) return base
|
|
58
|
+
for (let index = 2; index < 1000; index += 1) {
|
|
59
|
+
if (!taken.has(`${base}_${index}`)) return `${base}_${index}`
|
|
60
|
+
}
|
|
61
|
+
throw new DataSourceError('BAD_REQUEST', `无法为数据源生成唯一名称:${base}`)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async create(input: DataSourceInput): Promise<DataSourceRecord> {
|
|
65
|
+
const key = resolveEncryptKey(this.cfg.datasourceEncryptKey)
|
|
66
|
+
const now = Date.now()
|
|
67
|
+
const record: DataSourceRecord = {
|
|
68
|
+
id: makeSourceId(),
|
|
69
|
+
name: await this.uniqueName(input.name),
|
|
70
|
+
type: input.type,
|
|
71
|
+
host: input.host,
|
|
72
|
+
port: input.port,
|
|
73
|
+
database: input.database,
|
|
74
|
+
username: input.username,
|
|
75
|
+
passwordEnc: encryptPassword(input.password, key),
|
|
76
|
+
sslMode: input.sslMode ?? null,
|
|
77
|
+
poolMax: input.poolMax ?? null,
|
|
78
|
+
description: input.description ?? null,
|
|
79
|
+
status: 'unknown',
|
|
80
|
+
lastError: null,
|
|
81
|
+
lastCheckedAt: null,
|
|
82
|
+
createdAt: now,
|
|
83
|
+
updatedAt: now,
|
|
84
|
+
}
|
|
85
|
+
await this.insert(record)
|
|
86
|
+
return record
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async update(id: string, patch: DataSourcePatch): Promise<DataSourceRecord> {
|
|
90
|
+
const current = await this.require(id)
|
|
91
|
+
const assignments: string[] = ['updated_at = ?']
|
|
92
|
+
const params: unknown[] = [Date.now()]
|
|
93
|
+
const assign = (column: string, value: unknown): void => {
|
|
94
|
+
assignments.push(`${column} = ?`)
|
|
95
|
+
params.push(value)
|
|
96
|
+
}
|
|
97
|
+
if (patch.name !== undefined) assign('name', patch.name)
|
|
98
|
+
if (patch.type !== undefined) assign('type', patch.type)
|
|
99
|
+
if (patch.host !== undefined) assign('host', patch.host)
|
|
100
|
+
if (patch.port !== undefined) assign('port', patch.port)
|
|
101
|
+
if (patch.database !== undefined) assign('database_name', patch.database)
|
|
102
|
+
if (patch.username !== undefined) assign('username', patch.username)
|
|
103
|
+
if (typeof patch.password === 'string') {
|
|
104
|
+
assign('password_enc', encryptPassword(patch.password, resolveEncryptKey(this.cfg.datasourceEncryptKey)))
|
|
105
|
+
}
|
|
106
|
+
if (patch.sslMode !== undefined) assign('ssl_mode', patch.sslMode)
|
|
107
|
+
if (patch.poolMax !== undefined) assign('pool_max', patch.poolMax)
|
|
108
|
+
if (patch.description !== undefined) assign('description', patch.description)
|
|
109
|
+
if (patch.status !== undefined) assign('status', patch.status)
|
|
110
|
+
if (patch.lastError !== undefined) assign('last_error', patch.lastError)
|
|
111
|
+
if (patch.lastCheckedAt !== undefined) assign('last_checked_at', patch.lastCheckedAt)
|
|
112
|
+
params.push(current.id)
|
|
113
|
+
await (await this.db()).prepare(`UPDATE lh_data_sources SET ${assignments.join(', ')} WHERE id = ?`).run(...params)
|
|
114
|
+
// 连接参数可能已变,缓存里的旧连接池必须作废。
|
|
115
|
+
await closeConnector(current.id)
|
|
116
|
+
return await this.require(current.id)
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** 记录一次连通性检测的结果。 */
|
|
120
|
+
async recordCheck(id: string, success: boolean, error: string | null): Promise<void> {
|
|
121
|
+
await this.update(id, {
|
|
122
|
+
status: success ? 'connected' : 'error',
|
|
123
|
+
lastError: success ? null : error,
|
|
124
|
+
lastCheckedAt: Date.now(),
|
|
125
|
+
})
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async remove(id: string): Promise<void> {
|
|
129
|
+
const current = await this.require(id)
|
|
130
|
+
await closeConnector(current.id)
|
|
131
|
+
await (await this.db()).prepare('DELETE FROM lh_data_sources WHERE id = ?').run(current.id)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
private async insert(record: DataSourceRecord): Promise<void> {
|
|
135
|
+
await (await this.db())
|
|
136
|
+
.prepare(
|
|
137
|
+
`INSERT INTO lh_data_sources
|
|
138
|
+
(id, name, type, host, port, database_name, username, password_enc, ssl_mode, pool_max,
|
|
139
|
+
description, status, last_error, last_checked_at, created_at, updated_at)
|
|
140
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
141
|
+
)
|
|
142
|
+
.run(
|
|
143
|
+
record.id, record.name, record.type, record.host, record.port, record.database, record.username,
|
|
144
|
+
record.passwordEnc, record.sslMode, record.poolMax, record.description, record.status,
|
|
145
|
+
record.lastError, record.lastCheckedAt, record.createdAt, record.updatedAt,
|
|
146
|
+
)
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
export function createSourceStore(cfg: DataSourceConfig & DbConfig): DataSourceStore {
|
|
151
|
+
return new DataSourceStore(cfg)
|
|
152
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 数据源(DataSource)的类型定义。
|
|
3
|
+
*
|
|
4
|
+
* 数据源 = 一条远程关系型数据库的连接配置,全局登记在 catalog 库,跨工作区共享;
|
|
5
|
+
* 导入产出的数据集仍落在**当前工作区**的 scope 库里。
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { ColumnType } from '../parse'
|
|
9
|
+
|
|
10
|
+
/** 支持的关系型数据库类型。 */
|
|
11
|
+
export type DataSourceType = 'mysql' | 'postgresql'
|
|
12
|
+
|
|
13
|
+
/** 连通状态:从未检测 / 最近一次成功 / 最近一次失败。 */
|
|
14
|
+
export type DataSourceStatus = 'unknown' | 'connected' | 'error'
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* 与 `DataConfig` 结构同源的最小视图。
|
|
18
|
+
* 只声明本模块真正读取的键,避免 `datasource/*` → `store.ts` 的循环依赖。
|
|
19
|
+
*/
|
|
20
|
+
export interface DataSourceConfig {
|
|
21
|
+
datasourceEnabled: boolean
|
|
22
|
+
datasourceFetchBatchSize: number
|
|
23
|
+
datasourceConnectTimeoutMs: number
|
|
24
|
+
datasourceMaxImportRows: number
|
|
25
|
+
datasourceEncryptKey: string
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** catalog 库 `lh_data_sources` 的一行。`passwordEnc` 恒为密文。 */
|
|
29
|
+
export interface DataSourceRecord {
|
|
30
|
+
id: string
|
|
31
|
+
name: string
|
|
32
|
+
type: DataSourceType
|
|
33
|
+
host: string
|
|
34
|
+
port: number
|
|
35
|
+
database: string
|
|
36
|
+
username: string
|
|
37
|
+
passwordEnc: string
|
|
38
|
+
sslMode: string | null
|
|
39
|
+
poolMax: number | null
|
|
40
|
+
description: string | null
|
|
41
|
+
status: DataSourceStatus
|
|
42
|
+
lastError: string | null
|
|
43
|
+
lastCheckedAt: number | null
|
|
44
|
+
createdAt: number
|
|
45
|
+
updatedAt: number
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** 新建一条数据源需要的字段(密码为明文,落库前加密)。 */
|
|
49
|
+
export interface DataSourceInput {
|
|
50
|
+
name: string
|
|
51
|
+
type: DataSourceType
|
|
52
|
+
host: string
|
|
53
|
+
port: number
|
|
54
|
+
database: string
|
|
55
|
+
username: string
|
|
56
|
+
password: string
|
|
57
|
+
sslMode?: string | null
|
|
58
|
+
poolMax?: number | null
|
|
59
|
+
description?: string | null
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** 改一条数据源:字段缺省表示不改;`password` 给空串表示清空。 */
|
|
63
|
+
export interface DataSourcePatch {
|
|
64
|
+
name?: string
|
|
65
|
+
type?: DataSourceType
|
|
66
|
+
host?: string
|
|
67
|
+
port?: number
|
|
68
|
+
database?: string
|
|
69
|
+
username?: string
|
|
70
|
+
password?: string | null
|
|
71
|
+
sslMode?: string | null
|
|
72
|
+
poolMax?: number | null
|
|
73
|
+
description?: string | null
|
|
74
|
+
status?: DataSourceStatus
|
|
75
|
+
lastError?: string | null
|
|
76
|
+
lastCheckedAt?: number | null
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** 明文连接参数:只在内存里流转,绝不落库、绝不回显给前端或模型。 */
|
|
80
|
+
export interface ConnectionConfig {
|
|
81
|
+
host: string
|
|
82
|
+
port: number
|
|
83
|
+
database: string
|
|
84
|
+
username: string
|
|
85
|
+
password: string
|
|
86
|
+
sslMode: string | null
|
|
87
|
+
poolMax: number | null
|
|
88
|
+
connectTimeoutMs: number
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** 连接器返回的列元数据(方言无关,已把远端类型映射成本项目的列类型)。 */
|
|
92
|
+
export interface RemoteColumn {
|
|
93
|
+
name: string
|
|
94
|
+
nativeType: ColumnType
|
|
95
|
+
nullable: boolean
|
|
96
|
+
comment: string | null
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export interface RemoteTable {
|
|
100
|
+
tableName: string
|
|
101
|
+
schemaName: string | null
|
|
102
|
+
/** 远端的估计行数(MySQL `TABLE_ROWS` / PG `reltuples`),可能为 -1(未知)。 */
|
|
103
|
+
rowCount: number
|
|
104
|
+
primaryKey: string | null
|
|
105
|
+
columns: RemoteColumn[]
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export interface SchemaSummary {
|
|
109
|
+
schemaName: string
|
|
110
|
+
tableCount: number
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export interface ConnectionTestResult {
|
|
114
|
+
success: boolean
|
|
115
|
+
latency: number
|
|
116
|
+
version: string | null
|
|
117
|
+
error: string | null
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** 分块拉取的区间。 */
|
|
121
|
+
export interface FetchRange {
|
|
122
|
+
limit?: number
|
|
123
|
+
offset?: number
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** 导入一张远端表的请求(`datasource_import` 与设置页共用)。 */
|
|
127
|
+
export interface ImportRequest {
|
|
128
|
+
scopeKey: string
|
|
129
|
+
tableName: string
|
|
130
|
+
schemaName?: string | null
|
|
131
|
+
name?: string | null
|
|
132
|
+
limit?: number | null
|
|
133
|
+
}
|