@compforge/harness-toolbox 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/dist/chunk-0rs1aacd.js +127 -0
- package/dist/chunk-1at7zs0z.js +104 -0
- package/dist/chunk-1vfwvgkj.js +104 -0
- package/dist/chunk-205a4vma.js +85 -0
- package/dist/chunk-3dt25p98.js +59 -0
- package/dist/chunk-6fnhzhdb.js +103 -0
- package/dist/chunk-8nm9f0hh.js +92 -0
- package/dist/chunk-ajhdd4hp.js +267 -0
- package/dist/chunk-c6z919qb.js +98 -0
- package/dist/chunk-djk4kmxb.js +0 -0
- package/dist/chunk-hqx7j9xe.js +14 -0
- package/dist/chunk-hv2kx47y.js +92 -0
- package/dist/chunk-j46x7j7c.js +73 -0
- package/dist/chunk-jr9y0ghr.js +157 -0
- package/dist/chunk-mkhyty75.js +166 -0
- package/dist/chunk-mp88s7r7.js +12 -0
- package/dist/chunk-pfar9cm3.js +0 -0
- package/dist/chunk-qq77x9ek.js +34 -0
- package/dist/chunk-s06hs7a8.js +79 -0
- package/dist/chunk-s1sjyem7.js +41 -0
- package/dist/chunk-v3hxvnrc.js +47 -0
- package/dist/chunk-w3a4p6jm.js +124 -0
- package/dist/chunk-xy4g56g4.js +50 -0
- package/dist/chunk-ztenxgjv.js +27 -0
- package/dist/client-manager.d.ts +14 -0
- package/dist/client-manager.js +6 -0
- package/dist/client.d.ts +5 -0
- package/dist/client.js +0 -0
- package/dist/concurrency.d.ts +7 -0
- package/dist/concurrency.js +6 -0
- package/dist/datasource.d.ts +26 -0
- package/dist/datasource.js +10 -0
- package/dist/kubernetes/client-node-pod-log.d.ts +31 -0
- package/dist/kubernetes/client-node-pod-log.js +425 -0
- package/dist/kubernetes/client.d.ts +17 -0
- package/dist/kubernetes/client.js +92 -0
- package/dist/kubernetes/endpoint.d.ts +4 -0
- package/dist/kubernetes/endpoint.js +0 -0
- package/dist/kubernetes/executor.d.ts +45 -0
- package/dist/kubernetes/executor.js +13 -0
- package/dist/kubernetes/log-capture-file.d.ts +12 -0
- package/dist/kubernetes/log-capture-file.js +6 -0
- package/dist/kubernetes/log-capture-plan.d.ts +29 -0
- package/dist/kubernetes/log-capture-plan.js +9 -0
- package/dist/kubernetes/log-timestamp.d.ts +2 -0
- package/dist/kubernetes/log-timestamp.js +6 -0
- package/dist/kubernetes/pod-log-client.d.ts +25 -0
- package/dist/kubernetes/pod-log-client.js +12 -0
- package/dist/kubernetes/pod-log-datasource.d.ts +17 -0
- package/dist/kubernetes/pod-log-datasource.js +31 -0
- package/dist/kubernetes/pod-log.d.ts +52 -0
- package/dist/kubernetes/pod-log.js +121 -0
- package/dist/kubernetes/pod.d.ts +61 -0
- package/dist/kubernetes/pod.js +8 -0
- package/dist/kubernetes/port-forward.d.ts +42 -0
- package/dist/kubernetes/port-forward.js +10 -0
- package/dist/kubernetes/service-port-forward.d.ts +29 -0
- package/dist/kubernetes/service-port-forward.js +11 -0
- package/dist/kubernetes/service.d.ts +31 -0
- package/dist/kubernetes/service.js +17 -0
- package/dist/mysql/client.d.ts +36 -0
- package/dist/mysql/client.js +13 -0
- package/dist/mysql/index.d.ts +3 -0
- package/dist/mysql/index.js +18 -0
- package/dist/mysql/pod.d.ts +7 -0
- package/dist/mysql/pod.js +8 -0
- package/dist/mysql/target.d.ts +12 -0
- package/dist/mysql/target.js +6 -0
- package/dist/mysql/types.d.ts +16 -0
- package/dist/mysql/types.js +1 -0
- package/dist/opensearch/client.d.ts +62 -0
- package/dist/opensearch/client.js +12 -0
- package/dist/opensearch/endpoint.d.ts +16 -0
- package/dist/opensearch/endpoint.js +6 -0
- package/dist/opensearch/index.d.ts +3 -0
- package/dist/opensearch/index.js +17 -0
- package/dist/opensearch/types.d.ts +8 -0
- package/dist/opensearch/types.js +1 -0
- package/dist/process/index.d.ts +13 -0
- package/dist/process/index.js +8 -0
- package/dist/redis/client.d.ts +90 -0
- package/dist/redis/client.js +14 -0
- package/dist/redis/index.d.ts +3 -0
- package/dist/redis/index.js +24 -0
- package/dist/redis/key-stats.d.ts +27 -0
- package/dist/redis/key-stats.js +8 -0
- package/dist/redis/topology.d.ts +23 -0
- package/dist/redis/topology.js +6 -0
- package/dist/transport/index.d.ts +40 -0
- package/dist/transport/index.js +12 -0
- package/package.json +96 -0
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// src/redis/key-stats.ts
|
|
2
|
+
var LENGTH_COMMANDS = {
|
|
3
|
+
string: "STRLEN",
|
|
4
|
+
list: "LLEN",
|
|
5
|
+
set: "SCARD",
|
|
6
|
+
hash: "HLEN",
|
|
7
|
+
zset: "ZCARD",
|
|
8
|
+
stream: "XLEN"
|
|
9
|
+
};
|
|
10
|
+
var RANDOM_SAMPLE_MAX_FRACTION = 0.1;
|
|
11
|
+
var RANDOM_SAMPLE_DRAW_MULTIPLIER = 1.25;
|
|
12
|
+
function text(value) {
|
|
13
|
+
return Buffer.isBuffer(value) ? value.toString("utf8") : String(value ?? "");
|
|
14
|
+
}
|
|
15
|
+
function integer(value, fallback) {
|
|
16
|
+
if (value instanceof Error || value === null || value === undefined)
|
|
17
|
+
return fallback;
|
|
18
|
+
const parsed = Number(value);
|
|
19
|
+
return Number.isFinite(parsed) ? parsed : fallback;
|
|
20
|
+
}
|
|
21
|
+
function sleep(ms) {
|
|
22
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
23
|
+
}
|
|
24
|
+
async function throttle(startedAt, processed, maxPerSecond) {
|
|
25
|
+
const delay = processed / maxPerSecond * 1000 - (performance.now() - startedAt);
|
|
26
|
+
if (delay > 0)
|
|
27
|
+
await sleep(delay);
|
|
28
|
+
}
|
|
29
|
+
function redisGroupedSampleSizes(totalKeys, targetSamples) {
|
|
30
|
+
const samples = Math.min(Math.max(0, totalKeys), Math.max(0, targetSamples));
|
|
31
|
+
if (samples === 0)
|
|
32
|
+
return [];
|
|
33
|
+
const baseSize = Math.floor(totalKeys / samples);
|
|
34
|
+
const largerGroups = totalKeys % samples;
|
|
35
|
+
return Array.from({ length: samples }, (_, index) => baseSize + (index < largerGroups ? 1 : 0));
|
|
36
|
+
}
|
|
37
|
+
async function selectKeysByScan(client, totalKeys, limit, scanCount, maxKeysPerSecond, onProgress) {
|
|
38
|
+
const startedAt = performance.now();
|
|
39
|
+
if (limit >= totalKeys) {
|
|
40
|
+
const uniqueKeys = new Set;
|
|
41
|
+
let cursor2 = "0";
|
|
42
|
+
let visited2 = 0;
|
|
43
|
+
do {
|
|
44
|
+
const page = await client.scan(cursor2, scanCount);
|
|
45
|
+
cursor2 = page.cursor;
|
|
46
|
+
visited2 += page.keys.length;
|
|
47
|
+
for (const key of page.keys)
|
|
48
|
+
uniqueKeys.add(key);
|
|
49
|
+
onProgress?.(uniqueKeys.size, limit);
|
|
50
|
+
await throttle(startedAt, visited2, maxKeysPerSecond);
|
|
51
|
+
} while (cursor2 !== "0");
|
|
52
|
+
return { keys: [...uniqueKeys], visited: visited2 };
|
|
53
|
+
}
|
|
54
|
+
const groupSizes = redisGroupedSampleSizes(totalKeys, limit);
|
|
55
|
+
const keys = [];
|
|
56
|
+
let groupIndex = 0;
|
|
57
|
+
let groupPosition = 0;
|
|
58
|
+
let selectedPosition = groupSizes.length ? Math.floor(Math.random() * groupSizes[0]) : -1;
|
|
59
|
+
let cursor = "0";
|
|
60
|
+
let visited = 0;
|
|
61
|
+
do {
|
|
62
|
+
const page = await client.scan(cursor, scanCount);
|
|
63
|
+
cursor = page.cursor;
|
|
64
|
+
for (const key of page.keys) {
|
|
65
|
+
visited += 1;
|
|
66
|
+
if (groupIndex >= groupSizes.length)
|
|
67
|
+
continue;
|
|
68
|
+
if (groupPosition === selectedPosition)
|
|
69
|
+
keys.push(key);
|
|
70
|
+
groupPosition += 1;
|
|
71
|
+
if (groupPosition >= groupSizes[groupIndex]) {
|
|
72
|
+
groupIndex += 1;
|
|
73
|
+
groupPosition = 0;
|
|
74
|
+
selectedPosition = groupIndex < groupSizes.length ? Math.floor(Math.random() * groupSizes[groupIndex]) : -1;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
onProgress?.(keys.length, limit);
|
|
78
|
+
await throttle(startedAt, visited, maxKeysPerSecond);
|
|
79
|
+
} while (cursor !== "0");
|
|
80
|
+
return { keys, visited };
|
|
81
|
+
}
|
|
82
|
+
async function selectKeysByRandomDraw(client, limit, pipelineKeys, maxKeysPerSecond, onProgress) {
|
|
83
|
+
const keys = new Set;
|
|
84
|
+
const maxDraws = Math.ceil(limit * RANDOM_SAMPLE_DRAW_MULTIPLIER);
|
|
85
|
+
const startedAt = performance.now();
|
|
86
|
+
let visited = 0;
|
|
87
|
+
while (keys.size < limit && visited < maxDraws) {
|
|
88
|
+
const batchSize = Math.min(pipelineKeys, maxDraws - visited);
|
|
89
|
+
const replies = await client.pipeline(Array.from({ length: batchSize }, () => ["RANDOMKEY"]));
|
|
90
|
+
visited += batchSize;
|
|
91
|
+
for (const reply of replies) {
|
|
92
|
+
if (!(reply instanceof Error) && reply !== null && reply !== undefined)
|
|
93
|
+
keys.add(text(reply));
|
|
94
|
+
if (keys.size >= limit)
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
onProgress?.(keys.size, limit);
|
|
98
|
+
await throttle(startedAt, visited, maxKeysPerSecond);
|
|
99
|
+
}
|
|
100
|
+
return { keys: [...keys].slice(0, limit), visited };
|
|
101
|
+
}
|
|
102
|
+
async function selectKeys(client, totalKeys, limit, options) {
|
|
103
|
+
if (limit === 0)
|
|
104
|
+
return { keys: [], visited: 0 };
|
|
105
|
+
if (limit / totalKeys < RANDOM_SAMPLE_MAX_FRACTION) {
|
|
106
|
+
return selectKeysByRandomDraw(client, limit, options.pipelineKeys, options.maxKeysPerSecond, options.onSelectionProgress);
|
|
107
|
+
}
|
|
108
|
+
return selectKeysByScan(client, totalKeys, limit, options.scanCount, options.maxKeysPerSecond, options.onSelectionProgress);
|
|
109
|
+
}
|
|
110
|
+
async function inspectKeys(client, keys, memorySamples) {
|
|
111
|
+
const typeReplies = await client.pipeline(keys.map((key) => ["TYPE", key]));
|
|
112
|
+
const types = typeReplies.map((value) => value instanceof Error ? "unknown" : text(value));
|
|
113
|
+
const commands = [];
|
|
114
|
+
const replyOffsets = [];
|
|
115
|
+
for (let index = 0;index < keys.length; index += 1) {
|
|
116
|
+
const key = keys[index];
|
|
117
|
+
const type = types[index];
|
|
118
|
+
const ttl = commands.push(["PTTL", key]) - 1;
|
|
119
|
+
const memory = commands.push(["MEMORY", "USAGE", key, "SAMPLES", String(memorySamples)]) - 1;
|
|
120
|
+
const lengthCommand = LENGTH_COMMANDS[type];
|
|
121
|
+
const length = lengthCommand ? commands.push([lengthCommand, key]) - 1 : undefined;
|
|
122
|
+
replyOffsets.push({ ttl, memory, length });
|
|
123
|
+
}
|
|
124
|
+
const replies = commands.length ? await client.pipeline(commands) : [];
|
|
125
|
+
return keys.map((key, index) => {
|
|
126
|
+
const offsets = replyOffsets[index];
|
|
127
|
+
return {
|
|
128
|
+
key,
|
|
129
|
+
type: types[index],
|
|
130
|
+
ttlMs: integer(replies[offsets.ttl], -3),
|
|
131
|
+
memoryBytes: integer(replies[offsets.memory], 0),
|
|
132
|
+
length: offsets.length === undefined ? null : integer(replies[offsets.length], 0)
|
|
133
|
+
};
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
async function collectRedisKeyStats(client, options) {
|
|
137
|
+
const totalKeys = Math.max(0, Number(await client.dbSize()));
|
|
138
|
+
const limit = Math.min(totalKeys, Math.max(0, options.maxKeys));
|
|
139
|
+
const selected = await selectKeys(client, totalKeys, limit, options);
|
|
140
|
+
const result = [];
|
|
141
|
+
const startedAt = performance.now();
|
|
142
|
+
for (let offset = 0;offset < selected.keys.length; offset += options.pipelineKeys) {
|
|
143
|
+
const rows = await inspectKeys(client, selected.keys.slice(offset, offset + options.pipelineKeys), options.memorySamples);
|
|
144
|
+
result.push(...rows);
|
|
145
|
+
await throttle(startedAt, result.length, options.maxKeysPerSecond);
|
|
146
|
+
options.onProgress?.(result.length, limit);
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
totalKeys,
|
|
150
|
+
visitedKeys: selected.visited,
|
|
151
|
+
inspectedKeys: result.length,
|
|
152
|
+
scanComplete: limit >= totalKeys,
|
|
153
|
+
keys: result
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
export { redisGroupedSampleSizes, collectRedisKeyStats };
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ConcurrencyPool
|
|
3
|
+
} from "./chunk-v3hxvnrc.js";
|
|
4
|
+
import {
|
|
5
|
+
queryMysqlViaPod
|
|
6
|
+
} from "./chunk-s06hs7a8.js";
|
|
7
|
+
import {
|
|
8
|
+
isConnectionNetworkError
|
|
9
|
+
} from "./chunk-xy4g56g4.js";
|
|
10
|
+
|
|
11
|
+
// src/mysql/client.ts
|
|
12
|
+
import { createConnection } from "mysql2/promise";
|
|
13
|
+
class MysqlDatabase {
|
|
14
|
+
transports;
|
|
15
|
+
options;
|
|
16
|
+
connect;
|
|
17
|
+
#queries = new ConcurrencyPool(1);
|
|
18
|
+
#connections = new Map;
|
|
19
|
+
constructor(transports, options, connect = createConnection) {
|
|
20
|
+
this.transports = transports;
|
|
21
|
+
this.options = options;
|
|
22
|
+
this.connect = connect;
|
|
23
|
+
}
|
|
24
|
+
query(target, sql, values) {
|
|
25
|
+
return this.#queries.run(() => this.#query(target, sql, values), this.options.signal);
|
|
26
|
+
}
|
|
27
|
+
initialize(target) {
|
|
28
|
+
return this.#queries.run(async () => {
|
|
29
|
+
await this.#session(target);
|
|
30
|
+
}, this.options.signal);
|
|
31
|
+
}
|
|
32
|
+
#session(target) {
|
|
33
|
+
this.options.signal?.throwIfAborted();
|
|
34
|
+
const key = [target.host, target.port, target.database, target.user, target.password].join("\x00");
|
|
35
|
+
let pending = this.#connections.get(key);
|
|
36
|
+
if (!pending) {
|
|
37
|
+
pending = this.#open(target);
|
|
38
|
+
this.#connections.set(key, pending);
|
|
39
|
+
pending.catch(() => {
|
|
40
|
+
if (this.#connections.get(key) === pending)
|
|
41
|
+
this.#connections.delete(key);
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return pending;
|
|
45
|
+
}
|
|
46
|
+
async#query(target, sql, values) {
|
|
47
|
+
this.options.signal?.throwIfAborted();
|
|
48
|
+
const key = [target.host, target.port, target.database, target.user, target.password].join("\x00");
|
|
49
|
+
const pending = this.#session(target);
|
|
50
|
+
let session;
|
|
51
|
+
try {
|
|
52
|
+
session = await pending;
|
|
53
|
+
this.options.signal?.throwIfAborted();
|
|
54
|
+
if (session.kind === "python")
|
|
55
|
+
return await queryMysqlViaPod(session.transport, target, sql, values, this.options);
|
|
56
|
+
const [rows] = await session.connection.execute({ sql, values: [...values], timeout: this.options.queryTimeoutMs });
|
|
57
|
+
return rows;
|
|
58
|
+
} catch (error) {
|
|
59
|
+
if (this.#connections.get(key) === pending)
|
|
60
|
+
this.#connections.delete(key);
|
|
61
|
+
if (session?.kind === "tcp")
|
|
62
|
+
session.connection.destroy();
|
|
63
|
+
throw error;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
async#open(target) {
|
|
67
|
+
let reason;
|
|
68
|
+
for (let i = 0;i < this.transports.length; i++) {
|
|
69
|
+
const transport = this.transports[i];
|
|
70
|
+
this.options.signal?.throwIfAborted();
|
|
71
|
+
if (transport.kind === "python") {
|
|
72
|
+
this.options.onRoute?.({ transport: transport.name, reason });
|
|
73
|
+
return { kind: "python", transport };
|
|
74
|
+
}
|
|
75
|
+
try {
|
|
76
|
+
const endpoint = await transport.connect(target);
|
|
77
|
+
const connection = await this.connect({
|
|
78
|
+
host: endpoint.host,
|
|
79
|
+
port: endpoint.port,
|
|
80
|
+
user: target.user,
|
|
81
|
+
password: target.password,
|
|
82
|
+
database: target.database,
|
|
83
|
+
connectTimeout: this.options.connectTimeoutMs,
|
|
84
|
+
dateStrings: true,
|
|
85
|
+
supportBigNumbers: true,
|
|
86
|
+
bigNumberStrings: true
|
|
87
|
+
});
|
|
88
|
+
this.options.onRoute?.({ transport: endpoint.host === target.host && endpoint.port === target.port ? "direct" : transport.name, reason });
|
|
89
|
+
return { kind: "tcp", connection };
|
|
90
|
+
} catch (error) {
|
|
91
|
+
if (!isConnectionNetworkError(error) || i === this.transports.length - 1)
|
|
92
|
+
throw error;
|
|
93
|
+
reason = String(error.code);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
throw new Error("MySQL DataSource has no transport");
|
|
97
|
+
}
|
|
98
|
+
async queryOne(target, sql, values) {
|
|
99
|
+
return (await this.query(target, sql, values))[0];
|
|
100
|
+
}
|
|
101
|
+
close() {
|
|
102
|
+
return this.#queries.run(async () => {
|
|
103
|
+
const pending = [...this.#connections.values()];
|
|
104
|
+
this.#connections.clear();
|
|
105
|
+
for (const result of await Promise.allSettled(pending)) {
|
|
106
|
+
if (result.status === "fulfilled" && result.value.kind === "tcp")
|
|
107
|
+
result.value.connection.destroy();
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
class MysqlClient {
|
|
114
|
+
source;
|
|
115
|
+
options;
|
|
116
|
+
#controller = new AbortController;
|
|
117
|
+
signal;
|
|
118
|
+
#initialization;
|
|
119
|
+
#disposal;
|
|
120
|
+
#database;
|
|
121
|
+
#target;
|
|
122
|
+
constructor(source, options) {
|
|
123
|
+
this.source = source;
|
|
124
|
+
this.options = options;
|
|
125
|
+
this.signal = options.signal ? AbortSignal.any([options.signal, this.#controller.signal]) : this.#controller.signal;
|
|
126
|
+
}
|
|
127
|
+
initialize() {
|
|
128
|
+
this.signal.throwIfAborted();
|
|
129
|
+
return this.#initialization ??= (async () => {
|
|
130
|
+
this.#target = await this.source.resolve();
|
|
131
|
+
this.signal.throwIfAborted();
|
|
132
|
+
this.#database = new MysqlDatabase(this.source.transports, { ...this.options, signal: this.signal });
|
|
133
|
+
await this.#database.initialize(this.#target);
|
|
134
|
+
})();
|
|
135
|
+
}
|
|
136
|
+
get database() {
|
|
137
|
+
if (!this.#database)
|
|
138
|
+
throw new Error("MySQL client is not initialized");
|
|
139
|
+
return this.#database;
|
|
140
|
+
}
|
|
141
|
+
get target() {
|
|
142
|
+
if (!this.#target)
|
|
143
|
+
throw new Error("MySQL client is not initialized");
|
|
144
|
+
return this.#target;
|
|
145
|
+
}
|
|
146
|
+
dispose() {
|
|
147
|
+
return this.#disposal ??= (async () => {
|
|
148
|
+
this.#controller.abort(new Error("MySQL client disposed"));
|
|
149
|
+
await this.#initialization?.catch(() => {});
|
|
150
|
+
await this.#database?.close();
|
|
151
|
+
})();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
async function openMysql(source, options) {
|
|
155
|
+
const client = new MysqlClient(source, options);
|
|
156
|
+
try {
|
|
157
|
+
await client.initialize();
|
|
158
|
+
} catch (error) {
|
|
159
|
+
await client.dispose();
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
options.onDispose?.(() => client.dispose());
|
|
163
|
+
return client;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export { MysqlDatabase, MysqlClient, openMysql };
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// src/kubernetes/log-timestamp.ts
|
|
2
|
+
function logTimestampNanos(value) {
|
|
3
|
+
const parts = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/.exec(value);
|
|
4
|
+
if (!parts)
|
|
5
|
+
return;
|
|
6
|
+
const seconds = Date.parse(`${parts[1]}${parts[3]}`);
|
|
7
|
+
if (!Number.isFinite(seconds))
|
|
8
|
+
return;
|
|
9
|
+
return BigInt(seconds) * 1000000n + BigInt((parts[2] ?? "").padEnd(9, "0"));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export { logTimestampNanos };
|
|
File without changes
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// src/mysql/target.ts
|
|
2
|
+
function parseMysqlEnvTarget(raw, options) {
|
|
3
|
+
const env = new Map;
|
|
4
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
5
|
+
const index = line.indexOf("=");
|
|
6
|
+
if (index > 0)
|
|
7
|
+
env.set(line.slice(0, index), line.slice(index + 1));
|
|
8
|
+
}
|
|
9
|
+
const { label, prefix, fallback, defaultPort = 3306 } = options;
|
|
10
|
+
const host = env.get(`${prefix}_HOST`);
|
|
11
|
+
const database = env.get(`${prefix}_DATABASE`) || env.get(`${prefix}_NAME`);
|
|
12
|
+
const port = Number(env.get(`${prefix}_PORT`) || String(defaultPort));
|
|
13
|
+
if (!host || !database || !Number.isInteger(port) || port <= 0) {
|
|
14
|
+
throw new Error(`${label} 未暴露有效的 ${prefix}_HOST/PORT/DATABASE`);
|
|
15
|
+
}
|
|
16
|
+
const envUser = env.get(`${prefix}_USERNAME`) || env.get(`${prefix}_USER`);
|
|
17
|
+
const envPassword = env.get(`${prefix}_PASSWORD`);
|
|
18
|
+
if (envUser && envPassword) {
|
|
19
|
+
return {
|
|
20
|
+
host,
|
|
21
|
+
port,
|
|
22
|
+
database,
|
|
23
|
+
user: envUser,
|
|
24
|
+
password: envPassword,
|
|
25
|
+
credentialSource: "service-env"
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (!fallback?.user || !fallback.password) {
|
|
29
|
+
throw new Error(`${label} 未暴露完整的 ${prefix}_USERNAME/PASSWORD,且未配置 profile 数据库身份兜底`);
|
|
30
|
+
}
|
|
31
|
+
return { host, port, database, ...fallback, credentialSource: "profile" };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export { parseMysqlEnvTarget };
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// src/mysql/pod.ts
|
|
2
|
+
function preparePodSql(sql, values) {
|
|
3
|
+
let index = 0;
|
|
4
|
+
const rendered = sql.replace(/%/g, "%%").replace(/'(?:\\.|''|[^'\\])*'|"(?:\\.|""|[^"\\])*"|`(?:``|[^`])*`|--\s[^\n]*|#[^\n]*|\/\*[\s\S]*?\*\/|\?/g, (token) => {
|
|
5
|
+
if (token !== "?")
|
|
6
|
+
return token;
|
|
7
|
+
if (index >= values.length)
|
|
8
|
+
throw new Error("SQL placeholder count exceeds bound values");
|
|
9
|
+
index++;
|
|
10
|
+
return "%s";
|
|
11
|
+
});
|
|
12
|
+
if (index !== values.length)
|
|
13
|
+
throw new Error("SQL bound values exceed placeholder count");
|
|
14
|
+
return rendered;
|
|
15
|
+
}
|
|
16
|
+
var POD_QUERY = String.raw`
|
|
17
|
+
import json, math, signal, sys
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def timeout(signum, frame):
|
|
21
|
+
raise TimeoutError()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def encode(value):
|
|
25
|
+
if isinstance(value, bytes):
|
|
26
|
+
return {"type": "Buffer", "data": list(value)}
|
|
27
|
+
raise TypeError(type(value).__name__)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
try:
|
|
31
|
+
request = json.load(sys.stdin)
|
|
32
|
+
signal.signal(signal.SIGALRM, timeout)
|
|
33
|
+
signal.alarm(math.ceil(request["queryTimeoutMs"] / 1000))
|
|
34
|
+
import pymysql
|
|
35
|
+
from pymysql.constants import FIELD_TYPE
|
|
36
|
+
target = request["target"]
|
|
37
|
+
converters = pymysql.converters.conversions.copy()
|
|
38
|
+
# Match mysql2 dateStrings/supportBigNumbers/bigNumberStrings and native JSON decoding.
|
|
39
|
+
for field in (FIELD_TYPE.DATE, FIELD_TYPE.DATETIME, FIELD_TYPE.TIMESTAMP, FIELD_TYPE.TIME,
|
|
40
|
+
FIELD_TYPE.LONGLONG, FIELD_TYPE.DECIMAL, FIELD_TYPE.NEWDECIMAL):
|
|
41
|
+
converters[field] = str
|
|
42
|
+
converters[FIELD_TYPE.JSON] = json.loads
|
|
43
|
+
connection = pymysql.connect(
|
|
44
|
+
host=target["host"], port=target["port"], database=target["database"], user=target["user"],
|
|
45
|
+
password=target["password"], charset="utf8mb4",
|
|
46
|
+
connect_timeout=math.ceil(request["connectTimeoutMs"] / 1000),
|
|
47
|
+
read_timeout=math.ceil(request["queryTimeoutMs"] / 1000),
|
|
48
|
+
write_timeout=math.ceil(request["queryTimeoutMs"] / 1000),
|
|
49
|
+
conv=converters,
|
|
50
|
+
)
|
|
51
|
+
try:
|
|
52
|
+
with connection.cursor(pymysql.cursors.DictCursor) as cursor:
|
|
53
|
+
cursor.execute(request["sql"], request["values"])
|
|
54
|
+
rows = cursor.fetchall()
|
|
55
|
+
print(json.dumps({"rows": rows}, default=encode, ensure_ascii=False))
|
|
56
|
+
finally:
|
|
57
|
+
connection.close()
|
|
58
|
+
except Exception as error:
|
|
59
|
+
# DB errors may include SQL values; expose only the exception class and numeric server code.
|
|
60
|
+
code = error.args[0] if error.args and isinstance(error.args[0], int) else None
|
|
61
|
+
print(json.dumps({"error": type(error).__name__, "code": code}))
|
|
62
|
+
finally:
|
|
63
|
+
signal.alarm(0)
|
|
64
|
+
`;
|
|
65
|
+
async function queryMysqlViaPod(transport, target, sql, values, options) {
|
|
66
|
+
const raw = await transport.run(POD_QUERY, {
|
|
67
|
+
target,
|
|
68
|
+
sql: preparePodSql(sql, values),
|
|
69
|
+
values,
|
|
70
|
+
...options
|
|
71
|
+
}, options.queryTimeoutMs + 3000);
|
|
72
|
+
const response = JSON.parse(raw, (_key, value) => value?.type === "Buffer" && Array.isArray(value.data) ? Buffer.from(value.data) : value);
|
|
73
|
+
if (response.error || !Array.isArray(response.rows)) {
|
|
74
|
+
throw new Error(`MySQL via Pod failed: ${response.error ?? "invalid response"}${response.code ? ` (${response.code})` : ""}`);
|
|
75
|
+
}
|
|
76
|
+
return response.rows;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export { preparePodSql, queryMysqlViaPod };
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// src/opensearch/endpoint.ts
|
|
2
|
+
function parseOpenSearchEndpoint(value) {
|
|
3
|
+
const first = value.split(",").map((item) => item.trim()).find(Boolean);
|
|
4
|
+
if (!first)
|
|
5
|
+
throw new Error("OpenSearch endpoint 为空");
|
|
6
|
+
const schemeExplicit = first.includes("://");
|
|
7
|
+
let parsed;
|
|
8
|
+
try {
|
|
9
|
+
parsed = new URL(schemeExplicit ? first : `http://${first}`);
|
|
10
|
+
} catch {
|
|
11
|
+
throw new Error(`OpenSearch endpoint 格式无效:${first}`);
|
|
12
|
+
}
|
|
13
|
+
if (!parsed.hostname)
|
|
14
|
+
throw new Error(`OpenSearch endpoint 缺少 host:${first}`);
|
|
15
|
+
if (!["http:", "https:"].includes(parsed.protocol)) {
|
|
16
|
+
throw new Error(`OpenSearch endpoint scheme 仅支持 http/https:${parsed.protocol}`);
|
|
17
|
+
}
|
|
18
|
+
const scheme = parsed.protocol.slice(0, -1);
|
|
19
|
+
const username = parsed.username ? decodeURIComponent(parsed.username) : undefined;
|
|
20
|
+
const password = parsed.password ? decodeURIComponent(parsed.password) : undefined;
|
|
21
|
+
const port = parsed.port ? Number(parsed.port) : schemeExplicit ? scheme === "https" ? 443 : 80 : 9200;
|
|
22
|
+
parsed.username = "";
|
|
23
|
+
parsed.password = "";
|
|
24
|
+
parsed.port = String(port);
|
|
25
|
+
parsed.pathname = parsed.pathname === "/" ? "" : parsed.pathname.replace(/\/+$/, "");
|
|
26
|
+
const safeUrl = parsed.toString().replace(/\/$/, "");
|
|
27
|
+
const safeEndpoint = schemeExplicit ? safeUrl : `${parsed.hostname}:${port}`;
|
|
28
|
+
return {
|
|
29
|
+
url: safeUrl,
|
|
30
|
+
safeUrl,
|
|
31
|
+
safeEndpoint,
|
|
32
|
+
host: parsed.hostname,
|
|
33
|
+
port,
|
|
34
|
+
scheme,
|
|
35
|
+
schemeExplicit,
|
|
36
|
+
username,
|
|
37
|
+
password
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export { parseOpenSearchEndpoint };
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
// src/concurrency.ts
|
|
2
|
+
class ConcurrencyPool {
|
|
3
|
+
concurrency;
|
|
4
|
+
#active = 0;
|
|
5
|
+
#waiting = [];
|
|
6
|
+
constructor(concurrency) {
|
|
7
|
+
this.concurrency = concurrency;
|
|
8
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1)
|
|
9
|
+
throw new Error("concurrency must be a positive integer");
|
|
10
|
+
}
|
|
11
|
+
async run(work, signal) {
|
|
12
|
+
await this.#acquire(signal);
|
|
13
|
+
try {
|
|
14
|
+
signal?.throwIfAborted();
|
|
15
|
+
return await work();
|
|
16
|
+
} finally {
|
|
17
|
+
const next = this.#waiting.shift();
|
|
18
|
+
if (next)
|
|
19
|
+
next();
|
|
20
|
+
else
|
|
21
|
+
this.#active--;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
#acquire(signal) {
|
|
25
|
+
signal?.throwIfAborted();
|
|
26
|
+
if (this.#active < this.concurrency) {
|
|
27
|
+
this.#active++;
|
|
28
|
+
return Promise.resolve();
|
|
29
|
+
}
|
|
30
|
+
return new Promise((resolve, reject) => {
|
|
31
|
+
const start = () => {
|
|
32
|
+
signal?.removeEventListener("abort", abort);
|
|
33
|
+
resolve();
|
|
34
|
+
};
|
|
35
|
+
const abort = () => {
|
|
36
|
+
const index = this.#waiting.indexOf(start);
|
|
37
|
+
if (index >= 0)
|
|
38
|
+
this.#waiting.splice(index, 1);
|
|
39
|
+
reject(signal.reason);
|
|
40
|
+
};
|
|
41
|
+
this.#waiting.push(start);
|
|
42
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export { ConcurrencyPool };
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// src/opensearch/client.ts
|
|
2
|
+
import { Client } from "@opensearch-project/opensearch";
|
|
3
|
+
function isOpenSearchReadApi(search) {
|
|
4
|
+
return typeof search.request === "function";
|
|
5
|
+
}
|
|
6
|
+
function createClient(options) {
|
|
7
|
+
return new Client({
|
|
8
|
+
node: options.node,
|
|
9
|
+
auth: options.auth.username && options.auth.password ? { username: options.auth.username, password: options.auth.password } : undefined,
|
|
10
|
+
ssl: { rejectUnauthorized: false },
|
|
11
|
+
requestTimeout: options.requestTimeoutMs ?? 60000,
|
|
12
|
+
maxRetries: 0
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
class OpenSearchEngine {
|
|
17
|
+
client;
|
|
18
|
+
constructor(options, client) {
|
|
19
|
+
this.client = client ?? createClient(options);
|
|
20
|
+
}
|
|
21
|
+
async count(index, query) {
|
|
22
|
+
const response = await this.client.count({ index, body: { query } });
|
|
23
|
+
return Number(response.body.count ?? 0);
|
|
24
|
+
}
|
|
25
|
+
async search(index, body) {
|
|
26
|
+
const response = await this.client.search({ index, body });
|
|
27
|
+
return response.body;
|
|
28
|
+
}
|
|
29
|
+
async request(path, query) {
|
|
30
|
+
if (!this.client.transport)
|
|
31
|
+
throw new Error("OpenSearch client 不支持通用只读请求");
|
|
32
|
+
const response = await this.client.transport.request({
|
|
33
|
+
method: "GET",
|
|
34
|
+
path,
|
|
35
|
+
querystring: query
|
|
36
|
+
});
|
|
37
|
+
return response.body;
|
|
38
|
+
}
|
|
39
|
+
ping() {
|
|
40
|
+
return this.client.ping();
|
|
41
|
+
}
|
|
42
|
+
close() {
|
|
43
|
+
return this.client.close();
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
class OpenSearchClient {
|
|
48
|
+
source;
|
|
49
|
+
#controller = new AbortController;
|
|
50
|
+
signal;
|
|
51
|
+
#initialization;
|
|
52
|
+
#disposal;
|
|
53
|
+
#engine;
|
|
54
|
+
#operations = new Set;
|
|
55
|
+
constructor(source, lifecycle = {}) {
|
|
56
|
+
this.source = source;
|
|
57
|
+
this.signal = lifecycle.signal ? AbortSignal.any([lifecycle.signal, this.#controller.signal]) : this.#controller.signal;
|
|
58
|
+
}
|
|
59
|
+
initialize() {
|
|
60
|
+
this.signal.throwIfAborted();
|
|
61
|
+
return this.#initialization ??= (async () => {
|
|
62
|
+
const target = await this.source.resolve();
|
|
63
|
+
this.signal.throwIfAborted();
|
|
64
|
+
const transport = this.source.transports[0];
|
|
65
|
+
if (!transport || transport.kind !== "tcp")
|
|
66
|
+
throw new Error("OpenSearch requires a TCP transport");
|
|
67
|
+
const url = new URL(target.node);
|
|
68
|
+
const endpoint = await transport.connect({ host: url.hostname, port: Number(url.port || (url.protocol === "https:" ? 443 : 80)) });
|
|
69
|
+
this.signal.throwIfAborted();
|
|
70
|
+
url.hostname = endpoint.host;
|
|
71
|
+
url.port = String(endpoint.port);
|
|
72
|
+
this.#engine = new OpenSearchEngine({ ...target, node: url.toString() });
|
|
73
|
+
})();
|
|
74
|
+
}
|
|
75
|
+
#ready() {
|
|
76
|
+
this.signal.throwIfAborted();
|
|
77
|
+
if (!this.#engine)
|
|
78
|
+
throw new Error("OpenSearch client is not initialized");
|
|
79
|
+
return this.#engine;
|
|
80
|
+
}
|
|
81
|
+
#run(operation) {
|
|
82
|
+
const engine = this.#ready();
|
|
83
|
+
const pending = operation(engine);
|
|
84
|
+
this.#operations.add(pending);
|
|
85
|
+
pending.then(() => this.#operations.delete(pending), () => this.#operations.delete(pending));
|
|
86
|
+
return pending;
|
|
87
|
+
}
|
|
88
|
+
count(index, query) {
|
|
89
|
+
return this.#run((engine) => engine.count(index, query));
|
|
90
|
+
}
|
|
91
|
+
search(index, body) {
|
|
92
|
+
return this.#run((engine) => engine.search(index, body));
|
|
93
|
+
}
|
|
94
|
+
request(path, query) {
|
|
95
|
+
return this.#run((engine) => engine.request(path, query));
|
|
96
|
+
}
|
|
97
|
+
ping() {
|
|
98
|
+
return this.#run((engine) => engine.ping());
|
|
99
|
+
}
|
|
100
|
+
close() {
|
|
101
|
+
return this.dispose();
|
|
102
|
+
}
|
|
103
|
+
dispose() {
|
|
104
|
+
return this.#disposal ??= (async () => {
|
|
105
|
+
this.#controller.abort(new Error("OpenSearch client disposed"));
|
|
106
|
+
await this.#initialization?.catch(() => {});
|
|
107
|
+
await Promise.allSettled(this.#operations);
|
|
108
|
+
await this.#engine?.close();
|
|
109
|
+
})();
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
async function openOpenSearch(source, lifecycle = {}) {
|
|
113
|
+
const client = new OpenSearchClient(source, lifecycle);
|
|
114
|
+
try {
|
|
115
|
+
await client.initialize();
|
|
116
|
+
} catch (error) {
|
|
117
|
+
await client.dispose();
|
|
118
|
+
throw error;
|
|
119
|
+
}
|
|
120
|
+
lifecycle.onDispose?.(() => client.dispose());
|
|
121
|
+
return client;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export { isOpenSearchReadApi, OpenSearchEngine, OpenSearchClient, openOpenSearch };
|