dsh-limix-data 0.0.0-stage → 0.3.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +73 -0
- package/LICENSING.md +19 -0
- package/THIRD-PARTY-NOTICES.md +46 -0
- package/assets/icon.png +0 -0
- package/cordis.patch.yml +4 -0
- package/lib/client.js +368 -0
- package/lib/index.js +841 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +70 -4
- package/src/CLAUDE.md +10 -0
- package/src/client/CLAUDE.md +8 -0
- package/src/client/DatasourceSettings.tsx +175 -0
- package/src/client/index.tsx +18 -0
- package/src/core/CLAUDE.md +13 -0
- package/src/core/cells.ts +67 -0
- package/src/core/csv.ts +13 -0
- package/src/core/drivers.ts +250 -0
- package/src/core/guard.ts +138 -0
- package/src/core/materialize.ts +46 -0
- package/src/core/store.ts +97 -0
- package/src/core/types.ts +68 -0
- package/src/host/CLAUDE.md +11 -0
- package/src/host/define-tool.ts +38 -0
- package/src/index.ts +193 -0
- package/tsdown.config.ts +62 -0
- package/README.md +0 -3
package/locale/en.json
ADDED
package/locale/zh.json
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,72 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-limix-data",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"description": "
|
|
6
|
-
|
|
3
|
+
"version": "0.3.11",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "配置数据库连接,查看库表结构,执行只读查询并导出 CSV,供 LimiX 预测分析使用。",
|
|
6
|
+
"main": "lib/index.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"default": "./lib/index.js"
|
|
10
|
+
},
|
|
11
|
+
"./client": {
|
|
12
|
+
"default": "./lib/client.js"
|
|
13
|
+
},
|
|
14
|
+
"./package.json": "./package.json",
|
|
15
|
+
"./locale/*.json": "./locale/*.json",
|
|
16
|
+
"./icon": "./assets/icon.png"
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
"lib/index.js",
|
|
20
|
+
"lib/client.js",
|
|
21
|
+
"cordis.patch.yml",
|
|
22
|
+
"locale/*.json",
|
|
23
|
+
"assets/icon.png",
|
|
24
|
+
"src",
|
|
25
|
+
"tsdown.config.ts",
|
|
26
|
+
"LICENSE",
|
|
27
|
+
"LICENSING.md",
|
|
28
|
+
"THIRD-PARTY-NOTICES.md"
|
|
29
|
+
],
|
|
30
|
+
"dsh": {
|
|
31
|
+
"client": {
|
|
32
|
+
"platform": "web",
|
|
33
|
+
"inject": [
|
|
34
|
+
"@deepseek-ai/dsh-client-runtime",
|
|
35
|
+
"@deepseek-ai/dsh-client-ui-settings"
|
|
36
|
+
]
|
|
37
|
+
},
|
|
38
|
+
"bundle": {
|
|
39
|
+
"patch": "./cordis.patch.yml"
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
"scripts": {
|
|
43
|
+
"build": "tsdown",
|
|
44
|
+
"test": "node --test tests/*.test.mjs"
|
|
45
|
+
},
|
|
46
|
+
"dependencies": {
|
|
47
|
+
"mysql2": "^3.23.2",
|
|
48
|
+
"pg": "^8.22.0"
|
|
49
|
+
},
|
|
50
|
+
"devDependencies": {
|
|
51
|
+
"@types/react": "~18.3.1",
|
|
52
|
+
"tsdown": "^0.15.7",
|
|
53
|
+
"typescript": "^5.9.3"
|
|
54
|
+
},
|
|
55
|
+
"engines": {
|
|
56
|
+
"dsh": ">=0.2.1-rc.5"
|
|
57
|
+
},
|
|
58
|
+
"publishConfig": {
|
|
59
|
+
"access": "public",
|
|
60
|
+
"registry": "https://registry.npmjs.org/"
|
|
61
|
+
},
|
|
62
|
+
"repository": {
|
|
63
|
+
"type": "git",
|
|
64
|
+
"url": "git+https://github.com/Limitless2023/limix-dsh.git",
|
|
65
|
+
"directory": "packages/data"
|
|
66
|
+
},
|
|
67
|
+
"homepage": "https://github.com/Limitless2023/limix-dsh#readme",
|
|
68
|
+
"bugs": {
|
|
69
|
+
"url": "https://github.com/Limitless2023/limix-dsh/issues"
|
|
70
|
+
},
|
|
71
|
+
"license": "PolyForm-Noncommercial-1.0.0"
|
|
72
|
+
}
|
package/src/CLAUDE.md
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* [INPUT]: 依赖 react(宿主 external);数据面走同源路由 GET/POST /limix-datasource/config、POST /test
|
|
3
|
+
* [OUTPUT]: 对外提供 DatasourceSettings:连接清单、演示标识、增删改表单、测试连接、保存
|
|
4
|
+
* [POS]: limix-datasource 的设置 UI(对位 Next.js 版 /datasources 管理页的 DSH 形态);
|
|
5
|
+
* 样式对齐宿主 token(主色 rgb(65,118,230)、胶囊、幽灵钮、13px)
|
|
6
|
+
* [PROTOCOL]: 变更时更新此头部,然后检查 CLAUDE.md
|
|
7
|
+
*/
|
|
8
|
+
import { useEffect, useState } from "react";
|
|
9
|
+
|
|
10
|
+
const ACCENT = "rgb(65, 118, 230)";
|
|
11
|
+
|
|
12
|
+
interface Source {
|
|
13
|
+
id?: string;
|
|
14
|
+
name: string;
|
|
15
|
+
kind: "postgres" | "mysql";
|
|
16
|
+
host: string;
|
|
17
|
+
port: number;
|
|
18
|
+
database: string;
|
|
19
|
+
schema?: string | null;
|
|
20
|
+
username: string;
|
|
21
|
+
password: string;
|
|
22
|
+
isDemo?: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const EMPTY: Source = {
|
|
26
|
+
name: "", kind: "postgres", host: "localhost", port: 5432,
|
|
27
|
+
database: "", schema: "", username: "", password: "",
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
const ui = {
|
|
31
|
+
wrap: { padding: "8px 4px", fontSize: 13, maxWidth: 760 },
|
|
32
|
+
h: { fontSize: 15, fontWeight: 700, margin: "0 0 4px" },
|
|
33
|
+
sub: { opacity: 0.55, marginBottom: 14, lineHeight: 1.6 },
|
|
34
|
+
card: {
|
|
35
|
+
border: "1px solid rgba(0,0,0,.09)", borderRadius: 12, padding: 14, marginBottom: 12,
|
|
36
|
+
},
|
|
37
|
+
row: { display: "flex", gap: 8, flexWrap: "wrap" as const, marginBottom: 8 },
|
|
38
|
+
field: { display: "flex", flexDirection: "column" as const, gap: 3 },
|
|
39
|
+
label: { fontSize: 11, opacity: 0.55 },
|
|
40
|
+
input: {
|
|
41
|
+
padding: "5px 9px", borderRadius: 8, border: "1px solid rgba(0,0,0,.15)",
|
|
42
|
+
fontSize: 13, background: "Canvas", color: "CanvasText", minWidth: 0,
|
|
43
|
+
},
|
|
44
|
+
btn: (primary: boolean) => ({
|
|
45
|
+
fontSize: 12.5, cursor: "pointer", padding: "5px 14px", borderRadius: 999,
|
|
46
|
+
border: primary ? "none" : "1px solid rgba(0,0,0,.15)",
|
|
47
|
+
background: primary ? ACCENT : "transparent",
|
|
48
|
+
color: primary ? "#fff" : "inherit",
|
|
49
|
+
}),
|
|
50
|
+
tag: (ok: boolean) => ({
|
|
51
|
+
fontSize: 12, marginLeft: 8, color: ok ? "#16a34a" : "#dc2626",
|
|
52
|
+
}),
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
export function DatasourceSettings(): JSX.Element {
|
|
56
|
+
const [sources, setSources] = useState<Source[]>([]);
|
|
57
|
+
const [loaded, setLoaded] = useState(false);
|
|
58
|
+
const [dirty, setDirty] = useState(false);
|
|
59
|
+
const [msg, setMsg] = useState<{ ok: boolean; text: string } | null>(null);
|
|
60
|
+
const [testing, setTesting] = useState<number | null>(null);
|
|
61
|
+
const [testResult, setTestResult] = useState<Record<number, { ok: boolean; text: string }>>({});
|
|
62
|
+
|
|
63
|
+
useEffect(() => {
|
|
64
|
+
fetch("/limix-datasource/config")
|
|
65
|
+
.then((r) => r.json())
|
|
66
|
+
.then((d) => { setSources(d.sources ?? []); setLoaded(true); })
|
|
67
|
+
.catch(() => { setMsg({ ok: false, text: "读取配置失败(Node 半路由不可达)" }); setLoaded(true); });
|
|
68
|
+
}, []);
|
|
69
|
+
|
|
70
|
+
function patch(i: number, k: keyof Source, v: unknown) {
|
|
71
|
+
setSources((ss) => ss.map((s, j) => (j === i ? { ...s, [k]: v } : s)));
|
|
72
|
+
setDirty(true);
|
|
73
|
+
setMsg(null);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async function test(i: number) {
|
|
77
|
+
setTesting(i);
|
|
78
|
+
try {
|
|
79
|
+
const r = await fetch("/limix-datasource/test", {
|
|
80
|
+
method: "POST", headers: { "content-type": "application/json" },
|
|
81
|
+
body: JSON.stringify({ source: sources[i] }),
|
|
82
|
+
}).then((x) => x.json());
|
|
83
|
+
setTestResult((m) => ({ ...m, [i]: r.ok ? { ok: true, text: `连接成功 ${r.version ?? ""}` } : { ok: false, text: r.error ?? "失败" } }));
|
|
84
|
+
} catch (e) {
|
|
85
|
+
setTestResult((m) => ({ ...m, [i]: { ok: false, text: String(e) } }));
|
|
86
|
+
} finally {
|
|
87
|
+
setTesting(null);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function save() {
|
|
92
|
+
try {
|
|
93
|
+
const r = await fetch("/limix-datasource/config", {
|
|
94
|
+
method: "POST", headers: { "content-type": "application/json" },
|
|
95
|
+
body: JSON.stringify({ sources }),
|
|
96
|
+
}).then((x) => x.json());
|
|
97
|
+
if (r.ok) { setSources(r.sources); setDirty(false); setMsg({ ok: true, text: "已保存,agent 侧即时生效" }); }
|
|
98
|
+
else setMsg({ ok: false, text: r.error ?? "保存失败" });
|
|
99
|
+
} catch (e) {
|
|
100
|
+
setMsg({ ok: false, text: String(e) });
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const F = (i: number, k: keyof Source, label: string, width: number, type = "text") => (
|
|
105
|
+
<span style={ui.field}>
|
|
106
|
+
<span style={ui.label}>{label}</span>
|
|
107
|
+
<input style={{ ...ui.input, width }} type={type} value={String(sources[i][k] ?? "")}
|
|
108
|
+
onChange={(e) => patch(i, k, k === "port" ? Number(e.target.value) : e.target.value)} />
|
|
109
|
+
</span>
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
return (
|
|
113
|
+
<div style={ui.wrap}>
|
|
114
|
+
<h3 style={ui.h}>数据源</h3>
|
|
115
|
+
<div style={ui.sub}>
|
|
116
|
+
配置客户数据库连接(postgres / mysql),LimiX 分析师即可对话取数:看表结构 → 生成 SQL(先展示确认)→
|
|
117
|
+
预览 → 物化成 CSV 进文件区 → 预测。全链路只读三闸防护;<b>建议使用只读账号</b>。
|
|
118
|
+
凭据以 0600 权限明文存于 DSH_HOME/limix-datasource.json(本地单用户环境取舍)。
|
|
119
|
+
</div>
|
|
120
|
+
|
|
121
|
+
{!loaded && <div style={{ opacity: 0.5 }}>加载中…</div>}
|
|
122
|
+
{loaded && sources.length === 0 && <div style={{ opacity: 0.6, marginBottom: 12 }}>还没有数据源,点下方"新增"。</div>}
|
|
123
|
+
|
|
124
|
+
{sources.map((s, i) => (
|
|
125
|
+
<div key={s.id ?? `new-${i}`} style={ui.card}>
|
|
126
|
+
<div style={ui.row}>
|
|
127
|
+
{F(i, "name", "名称", 170)}
|
|
128
|
+
<span style={ui.field}>
|
|
129
|
+
<span style={ui.label}>类型</span>
|
|
130
|
+
<select style={{ ...ui.input, width: 110 }} value={s.kind}
|
|
131
|
+
onChange={(e) => patch(i, "kind", e.target.value)}>
|
|
132
|
+
<option value="postgres">postgres</option>
|
|
133
|
+
<option value="mysql">mysql</option>
|
|
134
|
+
</select>
|
|
135
|
+
</span>
|
|
136
|
+
{F(i, "host", "主机", 150)}
|
|
137
|
+
{F(i, "port", "端口", 70)}
|
|
138
|
+
</div>
|
|
139
|
+
<div style={ui.row}>
|
|
140
|
+
{F(i, "database", "数据库", 130)}
|
|
141
|
+
{F(i, "schema", "schema(PG 可选)", 130)}
|
|
142
|
+
{F(i, "username", "用户名", 120)}
|
|
143
|
+
{F(i, "password", "密码", 140, "password")}
|
|
144
|
+
</div>
|
|
145
|
+
<div style={{ display: "flex", alignItems: "center", gap: 8 }}>
|
|
146
|
+
<label style={{ display: "flex", alignItems: "center", gap: 5, marginRight: "auto", fontSize: 12 }}>
|
|
147
|
+
<input type="checkbox" checked={!!s.isDemo} onChange={(e) => patch(i, "isDemo", e.target.checked)} />
|
|
148
|
+
演示数据(模拟数据源)
|
|
149
|
+
</label>
|
|
150
|
+
<span role="button" style={ui.btn(false)} onClick={() => void test(i)}>
|
|
151
|
+
{testing === i ? "测试中…" : "测试连接"}
|
|
152
|
+
</span>
|
|
153
|
+
<span role="button" style={{ ...ui.btn(false), color: "#dc2626" }}
|
|
154
|
+
onClick={() => { setSources((ss) => ss.filter((_, j) => j !== i)); setDirty(true); }}>
|
|
155
|
+
删除
|
|
156
|
+
</span>
|
|
157
|
+
{testResult[i] && <span style={ui.tag(testResult[i].ok)}>{testResult[i].text}</span>}
|
|
158
|
+
</div>
|
|
159
|
+
</div>
|
|
160
|
+
))}
|
|
161
|
+
|
|
162
|
+
<div style={{ display: "flex", gap: 10, alignItems: "center", marginTop: 4 }}>
|
|
163
|
+
<span role="button" style={ui.btn(false)}
|
|
164
|
+
onClick={() => { setSources((ss) => [...ss, { ...EMPTY }]); setDirty(true); }}>
|
|
165
|
+
+ 新增数据源
|
|
166
|
+
</span>
|
|
167
|
+
<span role="button" style={{ ...ui.btn(true), opacity: dirty ? 1 : 0.5 }}
|
|
168
|
+
onClick={() => { if (dirty) void save(); }}>
|
|
169
|
+
保存
|
|
170
|
+
</span>
|
|
171
|
+
{msg && <span style={ui.tag(msg.ok)}>{msg.text}</span>}
|
|
172
|
+
</div>
|
|
173
|
+
</div>
|
|
174
|
+
);
|
|
175
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* [INPUT]: 依赖宿主 slots 服务;组件见 ./DatasourceSettings
|
|
3
|
+
* [OUTPUT]: 浏览器半 Cordis 插件:settings.section 挂"数据源"整页(官方 ui-settings-models 同款注册法)
|
|
4
|
+
* [POS]: limix-datasource 的设置入口;数据经同源路由 /limix-datasource/* 与 Node 半交互
|
|
5
|
+
* [PROTOCOL]: 变更时更新此头部,然后检查 CLAUDE.md
|
|
6
|
+
*/
|
|
7
|
+
import { DatasourceSettings } from "./DatasourceSettings.tsx";
|
|
8
|
+
|
|
9
|
+
export const inject = ["slots"];
|
|
10
|
+
|
|
11
|
+
export function apply(ctx: any): void {
|
|
12
|
+
ctx.slots.inject("settings.section", () =>
|
|
13
|
+
ctx.slots.register(
|
|
14
|
+
{ name: "settings.section", id: "limix-datasource", order: 30, label: () => "数据源" },
|
|
15
|
+
DatasourceSettings,
|
|
16
|
+
),
|
|
17
|
+
);
|
|
18
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# packages/data/src/core/
|
|
2
|
+
|
|
3
|
+
> L2 | 父级: ../CLAUDE.md
|
|
4
|
+
|
|
5
|
+
drivers.ts - PostgreSQL/MySQL 只读短连接;MySQL DATE 原样保留日历日期。
|
|
6
|
+
types.ts - 数据源、表结构、查询结果与驱动接口。
|
|
7
|
+
store.ts - 本地配置读写与连接转换;凭据文件权限 0600。
|
|
8
|
+
guard.ts - 只读 SQL 校验与行数限制。
|
|
9
|
+
materialize.ts - 预览与 CSV 物化;调用驱动和 SQL 校验。
|
|
10
|
+
cells.ts - 列名去重与单元格序列化。
|
|
11
|
+
csv.ts - CSV 引号、逗号与换行转义。
|
|
12
|
+
|
|
13
|
+
[PROTOCOL]: 变更时更新此头部,然后检查 CLAUDE.md
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* [INPUT]: 无运行时依赖(纯函数, client/server/test 均可 import — 刻意不带 server-only)
|
|
3
|
+
* [OUTPUT]: 对外提供 sanitizeColumns(列名清洗)/cellToText(单元格→CSV 值)
|
|
4
|
+
* [POS]: lib/datasource 的物化纯逻辑层, 被 materialize.ts 消费并再导出;
|
|
5
|
+
* 与 guard.ts 同属"可单测的纯函数"一档 —— 三处审查修复(时区/撞名/公式注入)的行为契约锁在 cells.test.ts
|
|
6
|
+
* [PROTOCOL]: 变更时更新此头部,然后检查 CLAUDE.md
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** 列名清洗: 去首尾空白、内部空白/特殊字符换下划线 — LimiX/CSV 下游友好 */
|
|
10
|
+
export function sanitizeColumns(columns: string[]): string[] {
|
|
11
|
+
const seen = new Set<string>();
|
|
12
|
+
return columns.map((c, i) => {
|
|
13
|
+
let name = (c || `col_${i + 1}`).trim().replace(/[\s,;"'`]+/g, "_");
|
|
14
|
+
if (!name) name = `col_${i + 1}`;
|
|
15
|
+
// 撞名逐级避让并登记生成名本身, 防 ["x","x","x_2"] 撞出两个 x_2(审查发现 F4)
|
|
16
|
+
if (seen.has(name)) {
|
|
17
|
+
let k = 2;
|
|
18
|
+
while (seen.has(`${name}_${k}`)) k++;
|
|
19
|
+
name = `${name}_${k}`;
|
|
20
|
+
}
|
|
21
|
+
seen.add(name);
|
|
22
|
+
return name;
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** 两位补零 */
|
|
27
|
+
function p2(n: number): string {
|
|
28
|
+
return n < 10 ? `0${n}` : String(n);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** 合法数字字面量(含负号/正号/小数/科学计数) — 这类必须原样, 否则毁掉下游数值解析 */
|
|
32
|
+
const NUMERIC_LITERAL = /^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$/;
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* CSV 公式注入中和(审查发现 F6): Excel/WPS 打开 CSV 时会把 = + - @ 开头(及前导 Tab/CR)的单元格当公式执行。
|
|
36
|
+
* 数据来自客户库、理论上可被上游污染 → 加前导单引号让表格软件当纯文本。
|
|
37
|
+
* ⚠️ 关键例外: **负数必须放行** —— PG 的 numeric/decimal 经驱动返回的是字符串(保精度),
|
|
38
|
+
* 若把 "-5.5" 变成 "'-5.5", LimiX 侧数值解析直接崩(与时区 bug 同类的静默数据污染)。
|
|
39
|
+
*/
|
|
40
|
+
function neutralizeFormula(s: string): string {
|
|
41
|
+
if (!/^[=+\-@\t\r]/.test(s)) return s;
|
|
42
|
+
if (NUMERIC_LITERAL.test(s)) return s;
|
|
43
|
+
return `'${s}`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* 单元格 → CSV 值。
|
|
48
|
+
* 时间戳按**本地字段**拼串: 驱动把 DB 的 timestamp/DATETIME(无时区语义)按进程本地时区解析成 Date,
|
|
49
|
+
* 若再 toISOString() 转 UTC 会平移时区差(实测东八区平移 8 小时, 时序预测的时间轴整体错位)。
|
|
50
|
+
* 取本地字段即把驱动的解析原样还原成 DB 里的墙钟时间(审查发现 P0)。
|
|
51
|
+
*/
|
|
52
|
+
export function cellToText(v: unknown): string | number {
|
|
53
|
+
if (v === null || v === undefined) return "";
|
|
54
|
+
if (typeof v === "number") return v;
|
|
55
|
+
if (typeof v === "bigint") return v.toString();
|
|
56
|
+
if (typeof v === "boolean") return v ? "true" : "false";
|
|
57
|
+
if (v instanceof Date) {
|
|
58
|
+
if (Number.isNaN(v.getTime())) return "";
|
|
59
|
+
const d = `${v.getFullYear()}-${p2(v.getMonth() + 1)}-${p2(v.getDate())}`;
|
|
60
|
+
const t = `${p2(v.getHours())}:${p2(v.getMinutes())}:${p2(v.getSeconds())}`;
|
|
61
|
+
return `${d} ${t}`;
|
|
62
|
+
}
|
|
63
|
+
// 二进制列(MySQL BLOB / PG bytea)转 hex, 免 JSON.stringify 出 {"type":"Buffer",...} 垃圾(审查发现 F5)
|
|
64
|
+
if (v instanceof Uint8Array) return `0x${Buffer.from(v).toString("hex")}`;
|
|
65
|
+
if (typeof v === "object") return neutralizeFormula(JSON.stringify(v));
|
|
66
|
+
return neutralizeFormula(String(v));
|
|
67
|
+
}
|
package/src/core/csv.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* [INPUT]: 无依赖(自 Next.js 版 lib/limix/csv.ts 摘取 toCsv 一个函数)
|
|
3
|
+
* [OUTPUT]: 对外提供 toCsv(含引号/逗号/换行转义)
|
|
4
|
+
* [POS]: core 的 CSV 序列化原子,materialize 消费
|
|
5
|
+
* [PROTOCOL]: 变更时更新此头部,然后检查 CLAUDE.md
|
|
6
|
+
*/
|
|
7
|
+
export function toCsv(columns: string[], rows: (string | number)[][]): string {
|
|
8
|
+
const esc = (v: string | number) => {
|
|
9
|
+
const s = v == null ? "" : String(v);
|
|
10
|
+
return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
|
11
|
+
};
|
|
12
|
+
return [columns.map(esc).join(","), ...rows.map((r) => r.map(esc).join(","))].join("\n");
|
|
13
|
+
}
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
|
|
2
|
+
/**
|
|
3
|
+
* [INPUT]: 依赖 pg(Pool)/mysql2(纯JS, 懒加载) 驱动、./types 的统一接口
|
|
4
|
+
* [OUTPUT]: 对外提供 getDriver(kind);MySQL DATE 保留日历字符串,不经过时区换算。
|
|
5
|
+
* [POS]: lib/datasource 的驱动适配层 — 每次调用建短连接即用即弃(serverless 友好, 不留池);
|
|
6
|
+
* 只读三闸之二: 会话级 read-only + 语句超时(第一闸 guard 在上游, 第三闸=客户只读账号)
|
|
7
|
+
* [PROTOCOL]: 变更时更新此头部,然后检查 CLAUDE.md
|
|
8
|
+
*/
|
|
9
|
+
import type { ColumnInfo, DataSourceConn, DatasourceDriver, QueryResult, TableInfo } from "./types";
|
|
10
|
+
|
|
11
|
+
const CONNECT_TIMEOUT_MS = 8_000;
|
|
12
|
+
|
|
13
|
+
// ============================================================
|
|
14
|
+
// PostgreSQL(兼容人大金仓) — 走 pg, 短连接 Client
|
|
15
|
+
// ============================================================
|
|
16
|
+
const postgresDriver: DatasourceDriver = {
|
|
17
|
+
async testConnection(conn) {
|
|
18
|
+
try {
|
|
19
|
+
const version = await withPgClient(conn, 5_000, async (c) => {
|
|
20
|
+
const r = await c.query("SELECT version() AS v");
|
|
21
|
+
return String(r.rows[0]?.v || "");
|
|
22
|
+
});
|
|
23
|
+
return { ok: true, version };
|
|
24
|
+
} catch (e) {
|
|
25
|
+
return { ok: false, error: e instanceof Error ? e.message : String(e) };
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
|
|
29
|
+
async listTables(conn) {
|
|
30
|
+
return withPgClient(conn, 15_000, async (c) => {
|
|
31
|
+
const schema = conn.schema || "public";
|
|
32
|
+
const r = await c.query(
|
|
33
|
+
`SELECT t.table_name AS name, obj_description(pc.oid) AS comment
|
|
34
|
+
FROM information_schema.tables t
|
|
35
|
+
JOIN pg_class pc ON pc.relname = t.table_name
|
|
36
|
+
JOIN pg_namespace pn ON pn.oid = pc.relnamespace AND pn.nspname = t.table_schema
|
|
37
|
+
WHERE t.table_schema = $1 AND t.table_type = 'BASE TABLE'
|
|
38
|
+
ORDER BY t.table_name`,
|
|
39
|
+
[schema],
|
|
40
|
+
);
|
|
41
|
+
return r.rows.map((row) => ({ name: String(row.name), comment: row.comment ? String(row.comment) : null }));
|
|
42
|
+
});
|
|
43
|
+
},
|
|
44
|
+
|
|
45
|
+
async describeTable(conn, table, sampleN = 5) {
|
|
46
|
+
assertIdent(table);
|
|
47
|
+
return withPgClient(conn, 20_000, async (c) => {
|
|
48
|
+
const schema = conn.schema || "public";
|
|
49
|
+
const qualified = `${qIdent(schema)}.${qIdent(table)}`;
|
|
50
|
+
const n = Math.min(sampleN, 20);
|
|
51
|
+
// 列结构 / 表注释 / 采样行**一次往返取回** —— 三条分开发就是三个 RTT(跨区往返约 100ms/次,
|
|
52
|
+
// 点一次表要等 300ms 纯属白等)。子查询各自 json 聚合, 结果一行三列。
|
|
53
|
+
const r = await c.query(
|
|
54
|
+
`SELECT
|
|
55
|
+
(SELECT json_agg(x ORDER BY x.attnum) FROM (
|
|
56
|
+
SELECT a.attnum,
|
|
57
|
+
a.attname AS name,
|
|
58
|
+
format_type(a.atttypid, a.atttypmod) AS "dataType",
|
|
59
|
+
col_description(a.attrelid, a.attnum) AS comment
|
|
60
|
+
FROM pg_attribute a
|
|
61
|
+
WHERE a.attrelid = $1::regclass AND a.attnum > 0 AND NOT a.attisdropped
|
|
62
|
+
) x) AS columns,
|
|
63
|
+
obj_description($1::regclass) AS comment,
|
|
64
|
+
(SELECT json_agg(s) FROM (SELECT * FROM ${qualified} LIMIT ${n}) s) AS sample`,
|
|
65
|
+
[qualified],
|
|
66
|
+
);
|
|
67
|
+
const row = r.rows[0] || {};
|
|
68
|
+
const columns: ColumnInfo[] = ((row.columns as { name: string; dataType: string; comment: string | null }[] | null) ?? []).map(
|
|
69
|
+
(x) => ({ name: String(x.name), dataType: String(x.dataType), comment: x.comment ? String(x.comment) : null }),
|
|
70
|
+
);
|
|
71
|
+
return {
|
|
72
|
+
name: table,
|
|
73
|
+
comment: row.comment ? String(row.comment) : null,
|
|
74
|
+
columns,
|
|
75
|
+
sampleRows: ((row.sample as Record<string, unknown>[] | null) ?? []) as Record<string, unknown>[],
|
|
76
|
+
} satisfies TableInfo;
|
|
77
|
+
});
|
|
78
|
+
},
|
|
79
|
+
|
|
80
|
+
async runQuery(conn, sql, opts) {
|
|
81
|
+
return withPgClient(conn, opts.timeoutMs, async (c) => {
|
|
82
|
+
const r = await c.query({ text: sql, rowMode: "array" });
|
|
83
|
+
const columns = (r.fields || []).map((f) => f.name);
|
|
84
|
+
const rows = (r.rows || []) as unknown[][];
|
|
85
|
+
const truncated = rows.length >= opts.maxRows;
|
|
86
|
+
return { columns, rows: rows.slice(0, opts.maxRows), rowCount: Math.min(rows.length, opts.maxRows), truncated } satisfies QueryResult;
|
|
87
|
+
});
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
async function withPgClient<T>(conn: DataSourceConn, timeoutMs: number, fn: (c: import("pg").Client) => Promise<T>): Promise<T> {
|
|
92
|
+
const { Client } = await import("pg");
|
|
93
|
+
// schema 限定的源: search_path 走**启动参数**(零往返) —— 否则 listTables 列得出表、
|
|
94
|
+
// 裸 SQL `FROM 该表` 却报"不存在"(演示源就在 demo_source schema 里, agent 按表清单写的 SQL 必踩)。
|
|
95
|
+
// schema 名已过 assertIdent 白名单, 拼进 options 安全。
|
|
96
|
+
const startupOptions = conn.schema ? `-c search_path=${assertIdentAndReturn(conn.schema)},public` : undefined;
|
|
97
|
+
const client = new Client({
|
|
98
|
+
host: conn.host,
|
|
99
|
+
port: conn.port,
|
|
100
|
+
database: conn.database,
|
|
101
|
+
user: conn.username,
|
|
102
|
+
password: conn.password,
|
|
103
|
+
connectionTimeoutMillis: CONNECT_TIMEOUT_MS,
|
|
104
|
+
options: startupOptions,
|
|
105
|
+
// 本地容器无 TLS, 云库(Supabase 等)要 TLS — prefer 语义: 先试 TLS 失败落明文由服务端决定;
|
|
106
|
+
// pg 无内置 prefer, 用宽松校验 + 允许降级: 云端主机名默认开 TLS, localhost/127 不开
|
|
107
|
+
ssl: isLocalHost(conn.host) ? undefined : { rejectUnauthorized: false },
|
|
108
|
+
});
|
|
109
|
+
await client.connect();
|
|
110
|
+
try {
|
|
111
|
+
// 三闸之二: 会话级只读 + 语句超时(guard 之外的纵深)。
|
|
112
|
+
// ⚠️ 只读**必须走显式 SET**, 不可挪进上面的启动参数 —— 实测 Supabase 连接池会过滤部分
|
|
113
|
+
// 启动参数(search_path 生效、default_transaction_read_only 被丢弃, SHOW 出来是 off),
|
|
114
|
+
// 而只读是最后防线, 静默失效等于裸奔。两条合并成一次往返(无参数走 simple protocol, 可多语句)。
|
|
115
|
+
await client.query(
|
|
116
|
+
`SET default_transaction_read_only = on; SET statement_timeout = ${Math.max(1_000, Math.floor(timeoutMs))}`,
|
|
117
|
+
);
|
|
118
|
+
return await fn(client);
|
|
119
|
+
} finally {
|
|
120
|
+
void client.end().catch(() => {});
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// ============================================================
|
|
125
|
+
// MySQL — 走 mysql2/promise, 短连接
|
|
126
|
+
// ============================================================
|
|
127
|
+
const mysqlDriver: DatasourceDriver = {
|
|
128
|
+
async testConnection(conn) {
|
|
129
|
+
try {
|
|
130
|
+
const version = await withMysql(conn, 5_000, async (c) => {
|
|
131
|
+
const [rows] = await c.query("SELECT VERSION() AS v");
|
|
132
|
+
return String((rows as { v?: unknown }[])[0]?.v || "");
|
|
133
|
+
});
|
|
134
|
+
return { ok: true, version };
|
|
135
|
+
} catch (e) {
|
|
136
|
+
return { ok: false, error: e instanceof Error ? e.message : String(e) };
|
|
137
|
+
}
|
|
138
|
+
},
|
|
139
|
+
|
|
140
|
+
async listTables(conn) {
|
|
141
|
+
return withMysql(conn, 15_000, async (c) => {
|
|
142
|
+
const [rows] = await c.query(
|
|
143
|
+
`SELECT TABLE_NAME AS name, TABLE_COMMENT AS comment
|
|
144
|
+
FROM information_schema.TABLES
|
|
145
|
+
WHERE TABLE_SCHEMA = ? AND TABLE_TYPE = 'BASE TABLE'
|
|
146
|
+
ORDER BY TABLE_NAME`,
|
|
147
|
+
[conn.database],
|
|
148
|
+
);
|
|
149
|
+
return (rows as { name: unknown; comment: unknown }[]).map((r) => ({
|
|
150
|
+
name: String(r.name),
|
|
151
|
+
comment: r.comment ? String(r.comment) : null,
|
|
152
|
+
}));
|
|
153
|
+
});
|
|
154
|
+
},
|
|
155
|
+
|
|
156
|
+
async describeTable(conn, table, sampleN = 5) {
|
|
157
|
+
assertIdent(table);
|
|
158
|
+
return withMysql(conn, 20_000, async (c) => {
|
|
159
|
+
const [cols] = await c.query(
|
|
160
|
+
`SELECT COLUMN_NAME AS name, COLUMN_TYPE AS data_type, COLUMN_COMMENT AS comment
|
|
161
|
+
FROM information_schema.COLUMNS
|
|
162
|
+
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
|
|
163
|
+
ORDER BY ORDINAL_POSITION`,
|
|
164
|
+
[conn.database, table],
|
|
165
|
+
);
|
|
166
|
+
const columns: ColumnInfo[] = (cols as { name: unknown; data_type: unknown; comment: unknown }[]).map((r) => ({
|
|
167
|
+
name: String(r.name),
|
|
168
|
+
dataType: String(r.data_type),
|
|
169
|
+
comment: r.comment ? String(r.comment) : null,
|
|
170
|
+
}));
|
|
171
|
+
// 表级注释也带上 — 它是 agent NL2SQL 视野的一部分(审查发现 P3)
|
|
172
|
+
const [tc] = await c.query(
|
|
173
|
+
`SELECT TABLE_COMMENT AS comment FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?`,
|
|
174
|
+
[conn.database, table],
|
|
175
|
+
);
|
|
176
|
+
const tcRow = (tc as { comment?: unknown }[])[0];
|
|
177
|
+
const [sample] = await c.query(`SELECT * FROM \`${table}\` LIMIT ${Math.min(sampleN, 20)}`);
|
|
178
|
+
return {
|
|
179
|
+
name: table,
|
|
180
|
+
comment: tcRow?.comment ? String(tcRow.comment) : null,
|
|
181
|
+
columns,
|
|
182
|
+
sampleRows: sample as Record<string, unknown>[],
|
|
183
|
+
} satisfies TableInfo;
|
|
184
|
+
});
|
|
185
|
+
},
|
|
186
|
+
|
|
187
|
+
async runQuery(conn, sql, opts) {
|
|
188
|
+
return withMysql(conn, opts.timeoutMs, async (c) => {
|
|
189
|
+
const [rows, fields] = await c.query({ sql, rowsAsArray: true, timeout: opts.timeoutMs });
|
|
190
|
+
const columns = (fields || []).map((f) => f.name);
|
|
191
|
+
const arr = rows as unknown[][];
|
|
192
|
+
const truncated = arr.length >= opts.maxRows;
|
|
193
|
+
return { columns, rows: arr.slice(0, opts.maxRows), rowCount: Math.min(arr.length, opts.maxRows), truncated } satisfies QueryResult;
|
|
194
|
+
});
|
|
195
|
+
},
|
|
196
|
+
};
|
|
197
|
+
|
|
198
|
+
async function withMysql<T>(
|
|
199
|
+
conn: DataSourceConn,
|
|
200
|
+
timeoutMs: number,
|
|
201
|
+
fn: (c: import("mysql2/promise").Connection) => Promise<T>,
|
|
202
|
+
): Promise<T> {
|
|
203
|
+
const mysql = await import("mysql2/promise");
|
|
204
|
+
const c = await mysql.createConnection({
|
|
205
|
+
host: conn.host,
|
|
206
|
+
port: conn.port,
|
|
207
|
+
database: conn.database,
|
|
208
|
+
user: conn.username,
|
|
209
|
+
password: conn.password,
|
|
210
|
+
connectTimeout: CONNECT_TIMEOUT_MS,
|
|
211
|
+
dateStrings: ['DATE'], // 营业日期不代表某个时刻,不能转 UTC 后改变日期。
|
|
212
|
+
ssl: isLocalHost(conn.host) ? undefined : { rejectUnauthorized: false },
|
|
213
|
+
});
|
|
214
|
+
try {
|
|
215
|
+
await c.query("SET SESSION TRANSACTION READ ONLY");
|
|
216
|
+
await c.query(`SET SESSION MAX_EXECUTION_TIME = ${Math.max(1_000, Math.floor(timeoutMs))}`); // 毫秒, 仅 SELECT 生效
|
|
217
|
+
return await fn(c);
|
|
218
|
+
} finally {
|
|
219
|
+
void c.end().catch(() => {});
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ============================================================
|
|
224
|
+
// 公共小件
|
|
225
|
+
// ============================================================
|
|
226
|
+
function isLocalHost(host: string): boolean {
|
|
227
|
+
return host === "localhost" || host === "127.0.0.1" || host === "::1" || host === "host.docker.internal";
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** 标识符白名单(表名/schema名): 字母数字下划线中文, 防注入拼接 */
|
|
231
|
+
function assertIdent(name: string): void {
|
|
232
|
+
if (!/^[\w一-龥]+$/.test(name)) throw new Error(`非法标识符: ${name}`);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** 同上, 校验通过即回传 —— 供拼接处一行完成"校验+使用", 免漏校验 */
|
|
236
|
+
function assertIdentAndReturn(name: string): string {
|
|
237
|
+
assertIdent(name);
|
|
238
|
+
return name;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function qIdent(name: string): string {
|
|
242
|
+
assertIdent(name);
|
|
243
|
+
return `"${name}"`;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export function getDriver(kind: DataSourceConn["kind"]): DatasourceDriver {
|
|
247
|
+
if (kind === "postgres") return postgresDriver;
|
|
248
|
+
if (kind === "mysql") return mysqlDriver;
|
|
249
|
+
throw new Error(`不支持的数据源类型: ${kind}`);
|
|
250
|
+
}
|