@nsyan/db 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -0
- package/index.ts +929 -0
- package/package.json +57 -0
- package/src/config.ts +220 -0
- package/src/core/export.ts +14 -0
- package/src/core/index.ts +7 -0
- package/src/core/policy.ts +11 -0
- package/src/core/scan/candidates.ts +286 -0
- package/src/core/scan/parsers.ts +149 -0
- package/src/core/scan/placeholders.ts +24 -0
- package/src/core/scan/scoring.ts +25 -0
- package/src/core/scan/spring.ts +73 -0
- package/src/core/scan/walker.ts +47 -0
- package/src/core/sql-text.ts +72 -0
- package/src/core/types.ts +88 -0
- package/src/core/whitelist.ts +4 -0
- package/src/dialects/bigdata-dialect.ts +197 -0
- package/src/dialects/dialect.ts +38 -0
- package/src/dialects/dm.ts +159 -0
- package/src/dialects/elasticsearch.ts +219 -0
- package/src/dialects/hive.ts +124 -0
- package/src/dialects/index.ts +12 -0
- package/src/dialects/kv-dialect.ts +127 -0
- package/src/dialects/mysql.ts +158 -0
- package/src/dialects/oracle.ts +177 -0
- package/src/dialects/postgresql.ts +168 -0
- package/src/dialects/redis.ts +190 -0
- package/src/dialects/relational-dialect.ts +76 -0
- package/src/dialects/search-dialect.ts +128 -0
- package/src/dialects/spark.ts +120 -0
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// src/dialects/postgresql.ts
|
|
2
|
+
import pg from "pg";
|
|
3
|
+
import type { ConnConfig, DbConnection, ExecOpts, ParsedTarget,
|
|
4
|
+
ListTablesResult, DescribeTableResult } from "../core/types.js";
|
|
5
|
+
import { RelationalDialect } from "./relational-dialect.js";
|
|
6
|
+
import { register, filterTables, type Fingerprints } from "./dialect.js";
|
|
7
|
+
|
|
8
|
+
// ── URL 解析(JDBC + 原生 URI 双形态,Spec §7)───
|
|
9
|
+
|
|
10
|
+
const JDBC_RE = /^jdbc:postgresql:\/\/([^:/?#]+)(?::(\d+))?\/([^?#]+).*$/;
|
|
11
|
+
const NATIVE_RE = /^postgresql:\/\/(?:([^:/?#@]+)(?::([^/?#@]*))?@)?([^:/?#]+)(?::(\d+))?\/([^?#]+).*$/;
|
|
12
|
+
|
|
13
|
+
function parsePostgresUrl(url: string): ParsedTarget | null {
|
|
14
|
+
const clean = url.split("?")[0];
|
|
15
|
+
const m = clean.match(JDBC_RE);
|
|
16
|
+
if (m) {
|
|
17
|
+
const host = m[1];
|
|
18
|
+
const port = m[2] ? parseInt(m[2], 10) : 5432;
|
|
19
|
+
const database = m[3];
|
|
20
|
+
if (!host || !database) return null;
|
|
21
|
+
return { host, port, database };
|
|
22
|
+
}
|
|
23
|
+
const n = clean.match(NATIVE_RE);
|
|
24
|
+
if (n) {
|
|
25
|
+
const host = n[3];
|
|
26
|
+
const port = n[4] ? parseInt(n[4], 10) : 5432;
|
|
27
|
+
const database = n[5];
|
|
28
|
+
if (!host || !database) return null;
|
|
29
|
+
const out: ParsedTarget = { host, port, database };
|
|
30
|
+
if (n[1] !== undefined) out.username = decodeURIComponent(n[1]);
|
|
31
|
+
if (n[2] !== undefined) out.password = decodeURIComponent(n[2]);
|
|
32
|
+
return out;
|
|
33
|
+
}
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// ── PostgreSQL 方言(逻辑从 src/db.ts 原样搬入)───
|
|
38
|
+
|
|
39
|
+
class PostgresqlDialect extends RelationalDialect {
|
|
40
|
+
id = "postgresql" as const;
|
|
41
|
+
label = "PostgreSQL";
|
|
42
|
+
family = "relational" as const;
|
|
43
|
+
defaultPort = 5432;
|
|
44
|
+
fingerprints: Fingerprints = {
|
|
45
|
+
urlPatterns: [/^jdbc:postgresql:\/\//, /^postgresql:\/\//],
|
|
46
|
+
configKeys: ["spring.datasource.url"],
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
parseUrl(url: string): ParsedTarget | null {
|
|
50
|
+
return parsePostgresUrl(url);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
displayUrl(config: ConnConfig): string {
|
|
54
|
+
return `jdbc:postgresql://${config.host}:${config.port}/${config.database}`;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
protected async doConnect(config: ConnConfig, _timeoutMs: number): Promise<DbConnection> {
|
|
58
|
+
const client = new pg.Client({
|
|
59
|
+
host: config.host,
|
|
60
|
+
port: config.port,
|
|
61
|
+
user: config.username,
|
|
62
|
+
password: config.password,
|
|
63
|
+
database: config.database,
|
|
64
|
+
});
|
|
65
|
+
await client.connect();
|
|
66
|
+
return {
|
|
67
|
+
type: "postgresql",
|
|
68
|
+
client,
|
|
69
|
+
async close() { await client.end(); },
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
protected async doExecute(client: unknown, stmt: string, opts: ExecOpts): Promise<{ columns: string[]; rows: unknown[][]; rowCount: number }> {
|
|
74
|
+
const pgClient = client as pg.Client;
|
|
75
|
+
const timeoutMs = opts.timeoutSec * 1000;
|
|
76
|
+
const maxRows = opts.maxRows;
|
|
77
|
+
// 设置 statement_timeout
|
|
78
|
+
await pgClient.query(`SET statement_timeout = '${timeoutMs}'`);
|
|
79
|
+
const res = await pgClient.query(stmt);
|
|
80
|
+
if (res.fields && res.fields.length > 0) {
|
|
81
|
+
const columns = res.fields.map((f: any) => f.name);
|
|
82
|
+
const rows = res.rows.slice(0, maxRows).map((r: any) => columns.map((col: string) => r[col]));
|
|
83
|
+
return { columns, rows, rowCount: res.rows.length };
|
|
84
|
+
}
|
|
85
|
+
return { columns: [], rows: [], rowCount: res.rowCount ?? 0 };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async versionQuery(conn: DbConnection): Promise<string> {
|
|
89
|
+
const pgClient = conn.client as pg.Client;
|
|
90
|
+
const res = await pgClient.query("SELECT version()");
|
|
91
|
+
return res.rows[0].version.split(",")[0].trim();
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async listTables(config: ConnConfig, pattern?: string): Promise<ListTablesResult> {
|
|
95
|
+
try {
|
|
96
|
+
const tables = await this.withConnection(config, async (conn) => {
|
|
97
|
+
const pgClient = conn.client as pg.Client;
|
|
98
|
+
const res = await pgClient.query(`
|
|
99
|
+
SELECT schemaname, tablename, obj_description(c.oid) AS description
|
|
100
|
+
FROM pg_catalog.pg_tables t
|
|
101
|
+
JOIN pg_catalog.pg_class c ON c.relname = t.tablename AND c.relnamespace = (
|
|
102
|
+
SELECT oid FROM pg_catalog.pg_namespace WHERE nspname = t.schemaname
|
|
103
|
+
)
|
|
104
|
+
WHERE schemaname NOT IN ('pg_catalog', 'information_schema')
|
|
105
|
+
ORDER BY schemaname, tablename
|
|
106
|
+
`);
|
|
107
|
+
const all = res.rows.map((r: any) => ({
|
|
108
|
+
schema: r.schemaname,
|
|
109
|
+
name: r.tablename,
|
|
110
|
+
type: "TABLE",
|
|
111
|
+
description: r.description || "",
|
|
112
|
+
}));
|
|
113
|
+
return filterTables(all, pattern);
|
|
114
|
+
});
|
|
115
|
+
return { success: true, tables, count: tables.length };
|
|
116
|
+
} catch (err: unknown) {
|
|
117
|
+
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async describeTable(config: ConnConfig, table: string): Promise<DescribeTableResult> {
|
|
122
|
+
if (!table.trim()) {
|
|
123
|
+
return { success: false, error: "表名不能为空" };
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
const columns = await this.withConnection(config, async (conn) => {
|
|
127
|
+
const pgClient = conn.client as pg.Client;
|
|
128
|
+
const parts = table.split(".");
|
|
129
|
+
const schema = parts.length === 2 ? parts[0] : "public";
|
|
130
|
+
const tableName = parts.length === 2 ? parts[1] : table;
|
|
131
|
+
const res = await pgClient.query(`
|
|
132
|
+
SELECT
|
|
133
|
+
a.attname AS name,
|
|
134
|
+
pg_catalog.format_type(a.atttypid, a.atttypmod) AS type,
|
|
135
|
+
NOT a.attnotnull AS nullable,
|
|
136
|
+
COALESCE(pg_catalog.pg_get_expr(ad.adbin, ad.adrelid), '') AS default_val,
|
|
137
|
+
COALESCE(ct.contype = 'p', FALSE) AS primary_key,
|
|
138
|
+
COALESCE(cd.description, '') AS comment
|
|
139
|
+
FROM pg_catalog.pg_attribute a
|
|
140
|
+
LEFT JOIN pg_catalog.pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum
|
|
141
|
+
LEFT JOIN pg_catalog.pg_description cd ON a.attrelid = cd.objoid AND a.attnum = cd.objsubid
|
|
142
|
+
LEFT JOIN pg_catalog.pg_constraint ct ON a.attrelid = ct.conrelid
|
|
143
|
+
AND ct.contype = 'p' AND a.attnum = ANY(ct.conkey)
|
|
144
|
+
WHERE a.attrelid = (
|
|
145
|
+
SELECT c.oid FROM pg_catalog.pg_class c
|
|
146
|
+
JOIN pg_catalog.pg_namespace n ON c.relnamespace = n.oid
|
|
147
|
+
WHERE c.relname = $1 AND n.nspname = $2
|
|
148
|
+
) AND a.attnum > 0 AND NOT a.attisdropped
|
|
149
|
+
ORDER BY a.attnum
|
|
150
|
+
`, [tableName, schema]);
|
|
151
|
+
return res.rows.map((r: any) => ({
|
|
152
|
+
name: r.name,
|
|
153
|
+
type: r.type,
|
|
154
|
+
nullable: r.nullable,
|
|
155
|
+
default: r.default_val || null,
|
|
156
|
+
primaryKey: r.primary_key,
|
|
157
|
+
comment: r.comment,
|
|
158
|
+
}));
|
|
159
|
+
});
|
|
160
|
+
return { success: true, columns, count: columns.length };
|
|
161
|
+
} catch (err: unknown) {
|
|
162
|
+
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export const postgresqlDialect = new PostgresqlDialect();
|
|
168
|
+
register(postgresqlDialect);
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// dialects/redis.ts —— Redis 方言(KvDialect + ioredis)
|
|
2
|
+
import Redis from "ioredis";
|
|
3
|
+
import type { ConnConfig, DbConnection, ParsedTarget,
|
|
4
|
+
ListTablesResult, DescribeTableResult, TableInfo } from "../core/types.js";
|
|
5
|
+
import { KvDialect } from "./kv-dialect.js";
|
|
6
|
+
import { register, filterTables, type Fingerprints } from "./dialect.js";
|
|
7
|
+
|
|
8
|
+
// ── URL 解析:redis(s)://[:password@]host:port[/db] ───
|
|
9
|
+
const REDIS_RE = /^(rediss?):\/\/(?:[^/@]*@)?([^:/?#@]+)(?::(\d+))?(?:\/(\d+))?$/;
|
|
10
|
+
|
|
11
|
+
function parseRedisUrl(url: string): ParsedTarget | null {
|
|
12
|
+
const clean = url.split("?")[0];
|
|
13
|
+
const m = clean.match(REDIS_RE);
|
|
14
|
+
if (!m) return null;
|
|
15
|
+
const ssl = m[1] === "rediss";
|
|
16
|
+
// 密码段(:password@ 或 user:password@)只取 @ 前冒号后部分
|
|
17
|
+
let password: string | undefined;
|
|
18
|
+
const atIdx = clean.indexOf("@");
|
|
19
|
+
if (atIdx >= 0) {
|
|
20
|
+
const auth = clean.slice(clean.indexOf("://") + 3, atIdx);
|
|
21
|
+
const colonIdx = auth.indexOf(":");
|
|
22
|
+
password = colonIdx >= 0 ? auth.slice(colonIdx + 1) : undefined;
|
|
23
|
+
if (password === "") password = undefined;
|
|
24
|
+
}
|
|
25
|
+
const host = m[2];
|
|
26
|
+
const port = m[3] ? parseInt(m[3], 10) : 6379;
|
|
27
|
+
const dbIndex = m[4] !== undefined ? parseInt(m[4], 10) : undefined;
|
|
28
|
+
if (!host) return null;
|
|
29
|
+
return { host, port, password, dbIndex, ssl };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
class RedisDialect extends KvDialect {
|
|
33
|
+
id = "redis" as const;
|
|
34
|
+
label = "Redis";
|
|
35
|
+
family = "kv" as const;
|
|
36
|
+
defaultPort = 6379;
|
|
37
|
+
fingerprints: Fingerprints = {
|
|
38
|
+
urlPatterns: [/^rediss?:\/\//],
|
|
39
|
+
configKeys: ["spring.data.redis.host", "spring.redis.host", "redisson.address"],
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
parseUrl(url: string): ParsedTarget | null {
|
|
43
|
+
return parseRedisUrl(url);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
displayUrl(config: ConnConfig): string {
|
|
47
|
+
const scheme = "redis";
|
|
48
|
+
const port = config.port ?? 6379;
|
|
49
|
+
const db = config.dbIndex ?? 0;
|
|
50
|
+
return `${scheme}://${config.host}:${port}/${db}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
protected async doConnect(config: ConnConfig, timeoutMs: number): Promise<DbConnection> {
|
|
54
|
+
const client = new Redis({
|
|
55
|
+
host: config.host ?? "localhost",
|
|
56
|
+
port: config.port ?? 6379,
|
|
57
|
+
password: config.password || undefined,
|
|
58
|
+
db: config.dbIndex ?? 0,
|
|
59
|
+
lazyConnect: true,
|
|
60
|
+
connectTimeout: timeoutMs,
|
|
61
|
+
maxRetriesPerRequest: 1,
|
|
62
|
+
});
|
|
63
|
+
await client.connect();
|
|
64
|
+
if (config.dbIndex !== undefined && config.dbIndex !== 0) {
|
|
65
|
+
await client.select(config.dbIndex);
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
type: "redis",
|
|
69
|
+
client,
|
|
70
|
+
async close() { client.disconnect(); },
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
protected async doSendCommand(client: unknown, args: string[]): Promise<unknown> {
|
|
75
|
+
const redis = client as Redis;
|
|
76
|
+
return redis.sendCommand(new Redis.Command(args[0], args.slice(1)));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async versionQuery(conn: DbConnection): Promise<string> {
|
|
80
|
+
const redis = conn.client as Redis;
|
|
81
|
+
const info = await redis.info("server");
|
|
82
|
+
const m = /^redis_version:(.+)$/m.exec(info ?? "");
|
|
83
|
+
return m ? m[1].trim() : "unknown";
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Redis 无表概念 → keyspace 概览:DBSIZE + SCAN 采样≤200 统计各类型 key 数量
|
|
87
|
+
async listTables(config: ConnConfig, pattern?: string): Promise<ListTablesResult> {
|
|
88
|
+
try {
|
|
89
|
+
const tables: TableInfo[] = await this.withConnection(config, async (conn) => {
|
|
90
|
+
const redis = conn.client as Redis;
|
|
91
|
+
const total = await redis.dbsize();
|
|
92
|
+
const typeCounts = new Map<string, number>();
|
|
93
|
+
let sampled = 0;
|
|
94
|
+
let cursor = "0";
|
|
95
|
+
// LIKE(%/_)→ SCAN 通配(*/?),全局替换(String.replace 单次替换是 bug)
|
|
96
|
+
const matchArgs = pattern ? ["MATCH", pattern.split("").map((ch) => ch === "%" ? "*" : ch === "_" ? "?" : ch).join("")] : [];
|
|
97
|
+
do {
|
|
98
|
+
const [next, keys] = await redis.scan(cursor, "COUNT", 100, ...matchArgs);
|
|
99
|
+
cursor = next;
|
|
100
|
+
for (const key of keys) {
|
|
101
|
+
if (sampled >= 200) break;
|
|
102
|
+
const t = await redis.type(key);
|
|
103
|
+
typeCounts.set(t, (typeCounts.get(t) ?? 0) + 1);
|
|
104
|
+
sampled++;
|
|
105
|
+
}
|
|
106
|
+
} while (cursor !== "0" && sampled < 200);
|
|
107
|
+
const rows: TableInfo[] = [...typeCounts.entries()].map(([type, n]) => ({
|
|
108
|
+
schema: "",
|
|
109
|
+
name: type,
|
|
110
|
+
type: "KEYSPACE",
|
|
111
|
+
description: `采样 ${sampled} 个 key 中该类型约 ${n} 个(共 ${total} 个 key,Redis 无表,用 SCAN 浏览 key)`,
|
|
112
|
+
}));
|
|
113
|
+
return rows;
|
|
114
|
+
});
|
|
115
|
+
return { success: true, tables, count: tables.length };
|
|
116
|
+
} catch (err: unknown) {
|
|
117
|
+
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// describeTable(key) → TYPE + 长度 + TTL + MEMORY USAGE(≥4.0 失败跳过)+ OBJECT ENCODING + 值预览
|
|
122
|
+
async describeTable(config: ConnConfig, key: string): Promise<DescribeTableResult> {
|
|
123
|
+
if (!key.trim()) {
|
|
124
|
+
return { success: false, error: "key 不能为空" };
|
|
125
|
+
}
|
|
126
|
+
try {
|
|
127
|
+
const columns = await this.withConnection(config, async (conn) => {
|
|
128
|
+
const redis = conn.client as Redis;
|
|
129
|
+
const exists = await redis.exists(key);
|
|
130
|
+
if (!exists) throw new Error(`key 不存在: ${key}`);
|
|
131
|
+
const type = await redis.type(key);
|
|
132
|
+
const ttl = await redis.ttl(key);
|
|
133
|
+
// 长度命令按类型分发
|
|
134
|
+
const lengthCmd: Record<string, string[]> = {
|
|
135
|
+
string: ["STRLEN", key],
|
|
136
|
+
list: ["LLEN", key],
|
|
137
|
+
set: ["SCARD", key],
|
|
138
|
+
hash: ["HLEN", key],
|
|
139
|
+
zset: ["ZCARD", key],
|
|
140
|
+
};
|
|
141
|
+
let length = "";
|
|
142
|
+
if (lengthCmd[type]) {
|
|
143
|
+
try {
|
|
144
|
+
length = String(await redis.sendCommand(new Redis.Command(lengthCmd[type][0], [key])));
|
|
145
|
+
} catch { /* ignore */ }
|
|
146
|
+
}
|
|
147
|
+
// MEMORY USAGE 需 ≥4.0,低版本失败时降级跳过
|
|
148
|
+
let memory = "";
|
|
149
|
+
try {
|
|
150
|
+
memory = String(await redis.sendCommand(new Redis.Command("MEMORY", ["USAGE", key])));
|
|
151
|
+
} catch { /* ignore */ }
|
|
152
|
+
// OBJECT ENCODING 服务 key 探测
|
|
153
|
+
let encoding = "";
|
|
154
|
+
try {
|
|
155
|
+
encoding = String(await redis.sendCommand(new Redis.Command("OBJECT", ["ENCODING", key])));
|
|
156
|
+
} catch { /* ignore */ }
|
|
157
|
+
// 值预览(截断)
|
|
158
|
+
let preview = "";
|
|
159
|
+
try {
|
|
160
|
+
if (type === "string") {
|
|
161
|
+
preview = String(await redis.get(key) ?? "").slice(0, 200);
|
|
162
|
+
} else if (type === "hash") {
|
|
163
|
+
preview = JSON.stringify(await redis.hgetall(key)).slice(0, 200);
|
|
164
|
+
} else if (type === "list") {
|
|
165
|
+
preview = JSON.stringify(await redis.lrange(key, 0, 9)).slice(0, 200);
|
|
166
|
+
} else if (type === "set") {
|
|
167
|
+
preview = JSON.stringify(await redis.smembers(key)).slice(0, 200);
|
|
168
|
+
} else if (type === "zset") {
|
|
169
|
+
preview = JSON.stringify(await redis.zrange(key, 0, 9, "WITHSCORES")).slice(0, 200);
|
|
170
|
+
}
|
|
171
|
+
} catch { /* ignore */ }
|
|
172
|
+
return [
|
|
173
|
+
{ name: "key", type: "string", nullable: false, default: null, primaryKey: true, comment: "" },
|
|
174
|
+
{ name: "type", type: "string", nullable: false, default: type, primaryKey: false, comment: "" },
|
|
175
|
+
{ name: "length", type: "integer", nullable: true, default: length || null, primaryKey: false, comment: "按类型取 STRLEN/LLEN/SCARD/HLEN/ZCARD" },
|
|
176
|
+
{ name: "ttl", type: "integer", nullable: false, default: String(ttl), primaryKey: false, comment: "-1=持久 -2=不存在(此处已确认存在)" },
|
|
177
|
+
{ name: "memory_usage", type: "integer", nullable: true, default: memory || null, primaryKey: false, comment: "需 Redis ≥4.0,低版本为空" },
|
|
178
|
+
{ name: "encoding", type: "string", nullable: true, default: encoding || null, primaryKey: false, comment: "OBJECT ENCODING" },
|
|
179
|
+
{ name: "preview", type: "string", nullable: true, default: preview || null, primaryKey: false, comment: "值预览(截断 200 字符)" },
|
|
180
|
+
];
|
|
181
|
+
});
|
|
182
|
+
return { success: true, columns, count: columns.length };
|
|
183
|
+
} catch (err: unknown) {
|
|
184
|
+
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export const redisDialect = new RedisDialect();
|
|
190
|
+
register(redisDialect);
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// src/dialects/relational-dialect.ts
|
|
2
|
+
import type { ConnConfig, DbConnection, ExecOpts, ParsedTarget } from "../core/types.js";
|
|
3
|
+
import type { QueryResult } from "../core/types.js";
|
|
4
|
+
import { splitStatements, isWriteStatement, isDropStatement } from "../core/sql-text.js";
|
|
5
|
+
import type { Dialect, Verdict, Fingerprints } from "./dialect.js";
|
|
6
|
+
|
|
7
|
+
export abstract class RelationalDialect implements Dialect {
|
|
8
|
+
abstract id: Dialect["id"];
|
|
9
|
+
abstract label: string;
|
|
10
|
+
abstract family: Dialect["family"];
|
|
11
|
+
abstract defaultPort: number;
|
|
12
|
+
abstract fingerprints: Fingerprints;
|
|
13
|
+
abstract parseUrl(url: string): ParsedTarget | null;
|
|
14
|
+
abstract displayUrl(config: ConnConfig): string;
|
|
15
|
+
abstract versionQuery(conn: DbConnection): Promise<string>;
|
|
16
|
+
protected abstract doConnect(config: ConnConfig, timeoutMs: number): Promise<DbConnection>;
|
|
17
|
+
// 注:doExecute 直接收驱动 client(unknown),各方言内部收窄为私有连接类型
|
|
18
|
+
protected abstract doExecute(client: unknown, stmt: string, opts: ExecOpts): Promise<{ columns: string[]; rows: unknown[][]; rowCount: number }>;
|
|
19
|
+
|
|
20
|
+
async withConnection<T>(config: ConnConfig, fn: (conn: DbConnection) => Promise<T>, timeoutMs = 10_000): Promise<T> {
|
|
21
|
+
const conn = await this.doConnect(config, timeoutMs);
|
|
22
|
+
try { return await fn(conn); }
|
|
23
|
+
finally { await conn.close(); }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// testConnection 复用 withConnection(连接泄漏防护);versionQuery 由各方言实现
|
|
27
|
+
|
|
28
|
+
isAllowed(sql: string, readonly: boolean): Verdict {
|
|
29
|
+
const stmts = splitStatements(sql);
|
|
30
|
+
if (stmts.length === 0) return { ok: false, reason: "SQL 语句为空" };
|
|
31
|
+
for (const s of stmts) {
|
|
32
|
+
if (isDropStatement(s)) {
|
|
33
|
+
return { ok: false, reason: `禁止执行 DROP 操作:${s.slice(0, 80)}`, summary: `DROP(硬限制): ${s.slice(0, 60)}` };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
const first = stmts[0];
|
|
37
|
+
const kind = /^\s*(\w+)/.exec(first)?.[1]?.toUpperCase() ?? "SQL";
|
|
38
|
+
const targets = [...first.matchAll(/\b(?:FROM|INTO|UPDATE|TABLE)\s+([A-Za-z0-9_."]+)/gi)].map((m) => m[1]).slice(0, 3).join(", ");
|
|
39
|
+
const summary = `${kind}${targets ? " " + targets : ""}(共 ${stmts.length} 条语句)`;
|
|
40
|
+
if (readonly) {
|
|
41
|
+
for (const s of stmts) {
|
|
42
|
+
if (isWriteStatement(s)) {
|
|
43
|
+
return { ok: false, reason: `只读模式下不允许执行非查询语句:${s.slice(0, 80)}`, isWrite: true, summary };
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
const isWrite = stmts.some(isWriteStatement);
|
|
48
|
+
return { ok: true, isWrite, summary };
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async executeOn(config: ConnConfig, sql: string, opts: ExecOpts): Promise<QueryResult> {
|
|
52
|
+
const start = Date.now();
|
|
53
|
+
const stmts = splitStatements(sql);
|
|
54
|
+
try {
|
|
55
|
+
let last = { columns: [] as string[], rows: [] as unknown[][], rowCount: 0 };
|
|
56
|
+
await this.withConnection(config, async (conn) => {
|
|
57
|
+
for (const s of stmts) last = await this.doExecute(conn.client, s, opts);
|
|
58
|
+
}, opts.timeoutSec * 1000);
|
|
59
|
+
return { success: true, ...last, duration: `${Date.now() - start}ms` };
|
|
60
|
+
} catch (err: unknown) {
|
|
61
|
+
return { success: false, error: err instanceof Error ? err.message : String(err), duration: `${Date.now() - start}ms` };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async testConnection(config: ConnConfig): Promise<import("../core/types.js").TestConnectionResult> {
|
|
66
|
+
const start = Date.now();
|
|
67
|
+
try {
|
|
68
|
+
const version = await this.withConnection(config, (conn) => this.versionQuery(conn));
|
|
69
|
+
return { success: true, version, latency: `${Date.now() - start}ms` };
|
|
70
|
+
} catch (err: unknown) {
|
|
71
|
+
return { success: false, error: err instanceof Error ? err.message : String(err), latency: `${Date.now() - start}ms` };
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
abstract listTables(config: ConnConfig, pattern?: string): Promise<import("../core/types.js").ListTablesResult>;
|
|
75
|
+
abstract describeTable(config: ConnConfig, target: string): Promise<import("../core/types.js").DescribeTableResult>;
|
|
76
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
// dialects/search-dialect.ts —— 搜索家族基类(DSL 解析、端点白名单 verdict)
|
|
2
|
+
// 注:搜索交互是"发 DSL JSON、拿文档结果",与关系型/KV 的语句·命令语义不同,独立成基类
|
|
3
|
+
|
|
4
|
+
import type { ConnConfig, DbConnection, ExecOpts, ParsedTarget,
|
|
5
|
+
QueryResult } from "../core/types.js";
|
|
6
|
+
import type { Dialect, Verdict, Fingerprints } from "./dialect.js";
|
|
7
|
+
|
|
8
|
+
// ── DSL 解封(Spec §4.3 DSL 信封约定)───
|
|
9
|
+
// JSON 顶层 key 映射端点:query/count→读(_search/_count);mget/mget_docs→读(_mget);
|
|
10
|
+
// bulk/doc_write/delete/update/mapping/settings→写;非 JSON 纯字符串→query_string 简化搜索
|
|
11
|
+
export type DslKind =
|
|
12
|
+
| { type: "read"; endpoint: "_search" | "_count" | "_mget"; detail: string }
|
|
13
|
+
| { type: "write"; endpoint: string; detail: string }
|
|
14
|
+
| { type: "query_string"; text: string };
|
|
15
|
+
|
|
16
|
+
const READ_KEYS: Record<string, "_search" | "_count" | "_mget"> = {
|
|
17
|
+
query: "_search",
|
|
18
|
+
count: "_count",
|
|
19
|
+
mget: "_mget",
|
|
20
|
+
mget_docs: "_mget",
|
|
21
|
+
};
|
|
22
|
+
|
|
23
|
+
const WRITE_KEYS = ["bulk", "doc_write", "delete", "update", "mapping", "settings"];
|
|
24
|
+
|
|
25
|
+
// `DELETE <index>` 纯字符串形式(删索引)恒拒——大小写不敏感、前导空白容忍
|
|
26
|
+
const DELETE_INDEX_RE = /^\s*DELETE\s+\S+/i;
|
|
27
|
+
|
|
28
|
+
export function parseDsl(input: string): DslKind {
|
|
29
|
+
const trimmed = input.trim();
|
|
30
|
+
try {
|
|
31
|
+
const body = JSON.parse(trimmed) as Record<string, unknown>;
|
|
32
|
+
if (body === null || typeof body !== "object" || Array.isArray(body)) {
|
|
33
|
+
return { type: "query_string", text: trimmed };
|
|
34
|
+
}
|
|
35
|
+
const keys = Object.keys(body);
|
|
36
|
+
for (const k of keys) {
|
|
37
|
+
const ep = READ_KEYS[k.toLowerCase()];
|
|
38
|
+
if (ep) return { type: "read", endpoint: ep, detail: describeDetail(body) };
|
|
39
|
+
}
|
|
40
|
+
for (const k of keys) {
|
|
41
|
+
if (WRITE_KEYS.includes(k.toLowerCase())) {
|
|
42
|
+
return { type: "write", endpoint: k, detail: describeDetail(body) };
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
// JSON 但无已知信封 key——保守按写处理(未知操作的写意图不可排除)
|
|
46
|
+
return { type: "write", endpoint: keys[0] ?? "unknown", detail: describeDetail(body) };
|
|
47
|
+
} catch {
|
|
48
|
+
return { type: "query_string", text: trimmed };
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function describeDetail(body: Record<string, unknown>): string {
|
|
53
|
+
const keys = Object.keys(body);
|
|
54
|
+
const first = keys[0] ?? "";
|
|
55
|
+
const sub = body[first];
|
|
56
|
+
if (sub !== null && typeof sub === "object" && !Array.isArray(sub)) {
|
|
57
|
+
const subKeys = Object.keys(sub as Record<string, unknown>);
|
|
58
|
+
if (subKeys.length > 0) return `${first}/${subKeys[0]}`;
|
|
59
|
+
}
|
|
60
|
+
return first;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export abstract class SearchDialect implements Dialect {
|
|
64
|
+
abstract id: Dialect["id"];
|
|
65
|
+
abstract label: string;
|
|
66
|
+
abstract family: Dialect["family"];
|
|
67
|
+
abstract defaultPort: number;
|
|
68
|
+
abstract fingerprints: Fingerprints;
|
|
69
|
+
abstract parseUrl(url: string): ParsedTarget | null;
|
|
70
|
+
abstract displayUrl(config: ConnConfig): string;
|
|
71
|
+
abstract versionQuery(conn: DbConnection): Promise<string>;
|
|
72
|
+
protected abstract doConnect(config: ConnConfig, timeoutMs: number): Promise<DbConnection>;
|
|
73
|
+
// 注:doSearch 直接收驱动 client(unknown),各方言内部收窄为私有客户端类型
|
|
74
|
+
protected abstract doSearch(client: unknown, config: ConnConfig, sql: string, kind: DslKind, opts: ExecOpts): Promise<{ columns: string[]; rows: unknown[][]; rowCount: number }>;
|
|
75
|
+
|
|
76
|
+
isAllowed(sql: string, readonly: boolean): Verdict {
|
|
77
|
+
const trimmed = sql.trim();
|
|
78
|
+
if (!trimmed) return { ok: false, reason: "查询内容为空" };
|
|
79
|
+
// DELETE <index> 纯字符串形式恒拒(与只读开关无关)
|
|
80
|
+
if (DELETE_INDEX_RE.test(trimmed)) {
|
|
81
|
+
return { ok: false, reason: `禁止删除索引:${trimmed.slice(0, 80)}`, isWrite: true, summary: `DELETE 索引(硬限制): ${trimmed.slice(0, 40)}` };
|
|
82
|
+
}
|
|
83
|
+
const kind = parseDsl(trimmed);
|
|
84
|
+
if (kind.type === "query_string") {
|
|
85
|
+
const summary = `SEARCH(query_string 简化搜索)`;
|
|
86
|
+
return { ok: true, isWrite: false, summary };
|
|
87
|
+
}
|
|
88
|
+
if (kind.type === "read") {
|
|
89
|
+
const summary = `SEARCH(${kind.endpoint}/${kind.detail || "match"})`;
|
|
90
|
+
return { ok: true, isWrite: false, summary };
|
|
91
|
+
}
|
|
92
|
+
// 写端点
|
|
93
|
+
const summary = `SEARCH(${kind.endpoint}/${kind.detail || "write"})`;
|
|
94
|
+
if (readonly) {
|
|
95
|
+
return { ok: false, reason: `只读模式下不允许执行写端点:${kind.endpoint}`, isWrite: true, summary };
|
|
96
|
+
}
|
|
97
|
+
return { ok: true, isWrite: true, summary };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async executeOn(config: ConnConfig, sql: string, opts: ExecOpts): Promise<QueryResult> {
|
|
101
|
+
const start = Date.now();
|
|
102
|
+
const kind = parseDsl(sql.trim());
|
|
103
|
+
try {
|
|
104
|
+
const last = await this.withConnection(config, (conn) => this.doSearch(conn.client, config, sql, kind, opts), opts.timeoutSec * 1000);
|
|
105
|
+
return { success: true, ...last, duration: `${Date.now() - start}ms` };
|
|
106
|
+
} catch (err: unknown) {
|
|
107
|
+
return { success: false, error: err instanceof Error ? err.message : String(err), duration: `${Date.now() - start}ms` };
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async withConnection<T>(config: ConnConfig, fn: (conn: DbConnection) => Promise<T>, timeoutMs = 10_000): Promise<T> {
|
|
112
|
+
const conn = await this.doConnect(config, timeoutMs);
|
|
113
|
+
try { return await fn(conn); }
|
|
114
|
+
finally { await conn.close(); }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async testConnection(config: ConnConfig): Promise<import("../core/types.js").TestConnectionResult> {
|
|
118
|
+
const start = Date.now();
|
|
119
|
+
try {
|
|
120
|
+
const version = await this.withConnection(config, (conn) => this.versionQuery(conn));
|
|
121
|
+
return { success: true, version, latency: `${Date.now() - start}ms` };
|
|
122
|
+
} catch (err: unknown) {
|
|
123
|
+
return { success: false, error: err instanceof Error ? err.message : String(err), latency: `${Date.now() - start}ms` };
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
abstract listTables(config: ConnConfig, pattern?: string): Promise<import("../core/types.js").ListTablesResult>;
|
|
127
|
+
abstract describeTable(config: ConnConfig, target: string): Promise<import("../core/types.js").DescribeTableResult>;
|
|
128
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// src/dialects/spark.ts —— Spark 方言(Spark Thrift Server,同 HS2 协议栈)
|
|
2
|
+
// 与 Hive 共享 BigDataDialect 基类;差异:写关键字 +CACHE|REFRESH|UNCACHE、
|
|
3
|
+
// 会话变量前缀 spark.、DESCRIBE 输出列差异适配。Spark Connect(DataFrame/gRPC)
|
|
4
|
+
// 明确不支持(Spec §1 非目标)。
|
|
5
|
+
// 注:hive-driver 无 default export(见 hive.ts),用 namespace import。
|
|
6
|
+
import * as hive from "hive-driver";
|
|
7
|
+
import type { ConnConfig, DbConnection, ParsedTarget,
|
|
8
|
+
DescribeTableResult, ColumnInfo } from "../core/types.js";
|
|
9
|
+
import { BigDataDialect, type Hs2Session } from "./bigdata-dialect.js";
|
|
10
|
+
import { register, type Fingerprints } from "./dialect.js";
|
|
11
|
+
|
|
12
|
+
const HIVE2_RE = /^jdbc:hive2:\/\/([^:/?#]+)(?::(\d+))?\/([^?#]*)$/;
|
|
13
|
+
const DEFAULT_PORT = 10000; // Spark Thrift Server 默认 10000(部分发行版 10015,可配)
|
|
14
|
+
|
|
15
|
+
function parseSparkUrl(url: string): ParsedTarget | null {
|
|
16
|
+
const clean = url.split("?")[0];
|
|
17
|
+
const m = clean.match(HIVE2_RE);
|
|
18
|
+
if (!m) return null;
|
|
19
|
+
const host = m[1];
|
|
20
|
+
const port = m[2] ? parseInt(m[2], 10) : DEFAULT_PORT;
|
|
21
|
+
const database = m[3] || undefined;
|
|
22
|
+
if (!host) return null;
|
|
23
|
+
return { host, port, database };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const { TCLIService, TCLIService_types } = hive.thrift as unknown as {
|
|
27
|
+
TCLIService: object; TCLIService_types: { TProtocolVersion: Record<string, number> };
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
export function buildSparkSessionConfig(config: ConnConfig): Record<string, string> {
|
|
31
|
+
const vars: Record<string, string> = {};
|
|
32
|
+
for (const [k, v] of Object.entries(config.options ?? {})) {
|
|
33
|
+
const key = k.startsWith("spark.") ? k : `spark.${k}`;
|
|
34
|
+
vars[key] = v;
|
|
35
|
+
}
|
|
36
|
+
return vars;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
class SparkDialect extends BigDataDialect {
|
|
40
|
+
id = "spark" as const;
|
|
41
|
+
label = "Spark";
|
|
42
|
+
family = "bigdata" as const;
|
|
43
|
+
defaultPort = DEFAULT_PORT;
|
|
44
|
+
fingerprints: Fingerprints = {
|
|
45
|
+
urlPatterns: [/^jdbc:hive2:\/\//],
|
|
46
|
+
configKeys: ["spring.datasource.url"],
|
|
47
|
+
};
|
|
48
|
+
// Hive 集 + SparkSQL 特有:CACHE|REFRESH|UNCACHE
|
|
49
|
+
protected writeKeywords = /\b(INSERT\s+(INTO|OVERWRITE)|CREATE\s+TABLE(\s+AS)?|LOAD\s+DATA|MSCK|ALTER|DROP|CACHE\s+TABLE|UNCACHE\s+TABLE|REFRESH(\s+TABLE|\s+RESOURCE|\s+CACHE)?)\b/i;
|
|
50
|
+
protected sessionPrefix = "spark.";
|
|
51
|
+
|
|
52
|
+
parseUrl(url: string): ParsedTarget | null {
|
|
53
|
+
return parseSparkUrl(url);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
displayUrl(config: ConnConfig): string {
|
|
57
|
+
return `jdbc:hive2://${config.host}:${config.port ?? DEFAULT_PORT}/${config.database ?? "default"}`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
protected async doConnect(config: ConnConfig, timeoutMs: number): Promise<DbConnection> {
|
|
61
|
+
const client = new hive.HiveClient(TCLIService, TCLIService_types);
|
|
62
|
+
await client.connect(
|
|
63
|
+
{ host: config.host ?? "localhost", port: config.port ?? DEFAULT_PORT },
|
|
64
|
+
new hive.connections.TcpConnection(),
|
|
65
|
+
config.password
|
|
66
|
+
? new hive.auth.PlainTcpAuthentication({ username: config.username ?? "", password: config.password })
|
|
67
|
+
: new hive.auth.NoSaslAuthentication(),
|
|
68
|
+
);
|
|
69
|
+
const configuration = buildSparkSessionConfig(config);
|
|
70
|
+
if (config.database) configuration["spark.sql.currentDb"] = config.database;
|
|
71
|
+
const session = await client.openSession({
|
|
72
|
+
client_protocol: TCLIService_types.TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10,
|
|
73
|
+
username: config.username || undefined,
|
|
74
|
+
password: config.password || undefined,
|
|
75
|
+
configuration,
|
|
76
|
+
});
|
|
77
|
+
void timeoutMs;
|
|
78
|
+
const hs2session = session as unknown as Hs2Session;
|
|
79
|
+
return {
|
|
80
|
+
type: "spark",
|
|
81
|
+
client: hs2session,
|
|
82
|
+
async close() {
|
|
83
|
+
try { await session.close(); } catch { /* ignore */ }
|
|
84
|
+
client.close();
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async versionQuery(conn: DbConnection): Promise<string> {
|
|
90
|
+
const session = conn.client as Hs2Session;
|
|
91
|
+
return this.fetchVersion(session);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// DESCRIBE TABLE → col_name/data_type/comment;Spark 输出含 # Partition Information 等段,遇 # 段即停
|
|
95
|
+
async describeTable(config: ConnConfig, table: string): Promise<DescribeTableResult> {
|
|
96
|
+
if (!table.trim()) {
|
|
97
|
+
return { success: false, error: "表名不能为空" };
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
const columns: ColumnInfo[] = await this.withConnection(config, async (conn) => {
|
|
101
|
+
const session = conn.client as Hs2Session;
|
|
102
|
+
const r = await this.runStatement(session, `DESCRIBE TABLE ${table}`,
|
|
103
|
+
{ readonly: true, maxRows: 500, timeoutSec: 30 });
|
|
104
|
+
const cols: ColumnInfo[] = [];
|
|
105
|
+
for (const row of r.rows) {
|
|
106
|
+
const [name, type, comment] = row.map((v) => String(v ?? "").trim());
|
|
107
|
+
if (!name || name.startsWith("#")) break;
|
|
108
|
+
cols.push({ name, type, nullable: true, default: null, primaryKey: false, comment: comment ?? "" });
|
|
109
|
+
}
|
|
110
|
+
return cols;
|
|
111
|
+
});
|
|
112
|
+
return { success: true, columns, count: columns.length };
|
|
113
|
+
} catch (err: unknown) {
|
|
114
|
+
return { success: false, error: err instanceof Error ? err.message : String(err) };
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export const sparkDialect = new SparkDialect();
|
|
120
|
+
register(sparkDialect);
|