@deepexi/datasense-connector 0.0.0-stage → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +191 -2
- package/package.json +33 -4
- package/scripts/build.ts +188 -0
- package/src/define.ts +267 -0
- package/src/descriptor.ts +116 -0
- package/src/errors.ts +105 -0
- package/src/index.ts +108 -0
- package/src/internal/location.ts +23 -0
- package/src/runtime/guest/bridge.ts +238 -0
- package/src/runtime/guest/cache.ts +43 -0
- package/src/runtime/guest/check.ts +17 -0
- package/src/runtime/guest/crypto.ts +166 -0
- package/src/runtime/guest/ctx.ts +157 -0
- package/src/runtime/guest/db.ts +59 -0
- package/src/runtime/guest/failure.ts +69 -0
- package/src/runtime/guest/http.ts +194 -0
- package/src/runtime/guest/index.ts +102 -0
- package/src/runtime/guest/limits.ts +20 -0
- package/src/runtime/guest/loader.ts +210 -0
- package/src/runtime/guest/log.ts +47 -0
- package/src/runtime/guest/map.ts +45 -0
- package/src/runtime/guest/operation.ts +193 -0
- package/src/runtime/guest/paginate.ts +169 -0
- package/src/runtime/guest/pull.ts +177 -0
- package/src/runtime/guest/push.ts +92 -0
- package/src/runtime/guest/signal.ts +14 -0
- package/src/runtime/guest/suggest.ts +51 -0
- package/src/runtime/guest/time.ts +137 -0
- package/src/runtime/guest-runtime.ts +75 -0
- package/src/runtime/index.ts +53 -0
- package/src/runtime/protocol.ts +557 -0
- package/src/runtime/session.ts +216 -0
- package/src/testing/child.ts +21 -0
- package/src/testing/compliance.ts +343 -0
- package/src/testing/engine.ts +445 -0
- package/src/testing/fixtures.ts +591 -0
- package/src/testing/gateway.ts +482 -0
- package/src/testing/import-guard.ts +221 -0
- package/src/testing/index.ts +43 -0
- package/src/testing/local-db.ts +226 -0
- package/src/testing/mock-external-system.ts +249 -0
- package/src/testing/pinned-http.ts +144 -0
- package/src/testing/recorder.ts +140 -0
- package/src/testing/redaction.ts +94 -0
- package/src/testing/replay.ts +437 -0
- package/src/testing/run-tests.ts +115 -0
- package/src/testing/schema.ts +70 -0
- package/src/testing/source-loader.ts +374 -0
- package/src/testing/types.ts +78 -0
- package/src/types.ts +329 -0
- package/src/validate.ts +267 -0
- package/test/build.test.ts +159 -0
- package/test/define.test-d.ts +182 -0
- package/test/fixtures/acceptance.ts +156 -0
- package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
- package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
- package/test/fixtures/connectors/http-pull/package.json +5 -0
- package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
- package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
- package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
- package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
- package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
- package/test/fixtures/connectors/pg-pull/package.json +5 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
- package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
- package/test/fixtures/descriptor.ts +54 -0
- package/test/fixtures/kingdee-example.ts +82 -0
- package/test/fixtures/record-acceptance.ts +32 -0
- package/test/runtime/ctx.test.ts +434 -0
- package/test/runtime/helpers.ts +157 -0
- package/test/runtime/jobs.test.ts +444 -0
- package/test/runtime/operation.test.ts +175 -0
- package/test/runtime/session.test.ts +229 -0
- package/test/testing/acceptance-fixtures.test.ts +69 -0
- package/test/testing/compliance.test.ts +247 -0
- package/test/testing/engine.test.ts +283 -0
- package/test/testing/fixtures.test.ts +221 -0
- package/test/testing/gateway.test.ts +337 -0
- package/test/testing/import-guard.test.ts +45 -0
- package/test/testing/mock-external-system.test.ts +144 -0
- package/test/testing/operation-fixtures.test.ts +160 -0
- package/test/testing/recorder.test.ts +163 -0
- package/test/testing/redaction.test.ts +54 -0
- package/test/testing/replay.test.ts +207 -0
- package/test/testing/run-tests.test.ts +137 -0
- package/test/testing/source-loader.test.ts +220 -0
- package/test/validate-config.test.ts +104 -0
- package/tsconfig.build.json +23 -0
- package/tsconfig.json +19 -0
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 模拟外部系统:连接器操作单元测试里替换 `ctx.http` 底层的 fetch(交给 `createLocalGateway({ fetch })`)。
|
|
3
|
+
* 它不是出站网关:主机求交、凭证句柄替换、超时都仍由网关做,这里只按路由应答并记下每次请求。
|
|
4
|
+
*
|
|
5
|
+
* - 路由键 `'<METHOD> <path>'`,path 段可写 `:名字` 占位(如 `GET /orders/:id`),处理函数拿到 `params`;
|
|
6
|
+
* - `script(route, steps)` 给该路由排一串结果,每次请求消费一步,排完后回到处理函数:
|
|
7
|
+
* `'ok'` 交给处理函数;`'reject'` 回 422 `{ code: 'REJECTED', message }`;`'timeout'` 一直不应答,
|
|
8
|
+
* 直到请求的 `signal` 中止(网关按请求超时中止,连接器得到 `TRANSIENT_ERROR`;执行器超时时为 `CANCELED`);
|
|
9
|
+
* `{ slowMs }` 等这么久再交给处理函数;
|
|
10
|
+
* - 没有匹配的路由回 404 `{ code: 'NOT_FOUND' }`,请求同样记下;
|
|
11
|
+
* - `calls(route?)` 返回记下的请求(头名小写,JSON 正文已解析),`reset()` 清空记录与排好的结果。
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export interface MockRequest {
|
|
15
|
+
method: string;
|
|
16
|
+
path: string;
|
|
17
|
+
/** 路由里 `:名字` 占位对应的路径段(已解码)。 */
|
|
18
|
+
params: Record<string, string>;
|
|
19
|
+
query: Record<string, string>;
|
|
20
|
+
/** 头名小写。 */
|
|
21
|
+
headers: Record<string, string>;
|
|
22
|
+
/** JSON 正文解析后的值;不是 JSON 时为文本;没有正文时为 null。 */
|
|
23
|
+
body: unknown;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface MockResponse {
|
|
27
|
+
/** 缺省 200。 */
|
|
28
|
+
status?: number;
|
|
29
|
+
headers?: Record<string, string>;
|
|
30
|
+
/** 字符串原样作为正文,其余按 JSON 序列化;省略则没有正文。 */
|
|
31
|
+
body?: unknown;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export type MockRouteHandler = (request: MockRequest) => MockResponse | Response | Promise<MockResponse | Response>;
|
|
35
|
+
|
|
36
|
+
export type MockStep = 'ok' | 'reject' | 'timeout' | { slowMs: number };
|
|
37
|
+
|
|
38
|
+
export interface MockCall {
|
|
39
|
+
method: string;
|
|
40
|
+
path: string;
|
|
41
|
+
headers: Record<string, string>;
|
|
42
|
+
body: unknown;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface MockExternalSystem {
|
|
46
|
+
/** 交给 `createLocalGateway({ fetch })` 或其他测试用的执行器 / 网关替身。 */
|
|
47
|
+
fetch: typeof fetch;
|
|
48
|
+
/** 给路由排一串结果,追加在尚未消费的结果之后。 */
|
|
49
|
+
script(route: string, steps: MockStep[]): void;
|
|
50
|
+
/** 记下的请求;给了路由键时只返回匹配该路由的请求。 */
|
|
51
|
+
calls(route?: string): MockCall[];
|
|
52
|
+
/** 清空请求记录与排好的结果(路由不变)。 */
|
|
53
|
+
reset(): void;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface CompiledRoute {
|
|
57
|
+
key: string;
|
|
58
|
+
method: string;
|
|
59
|
+
segments: string[];
|
|
60
|
+
handler: MockRouteHandler;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const ROUTE_KEY = /^([A-Za-z]+) (\/\S*)$/;
|
|
64
|
+
|
|
65
|
+
/** 拆出路由键的方法(大写)与路径;不合法时返回 null。 */
|
|
66
|
+
function parseKey(key: string): { method: string; path: string } | null {
|
|
67
|
+
const match = ROUTE_KEY.exec(key.trim());
|
|
68
|
+
const method = match?.[1];
|
|
69
|
+
const path = match?.[2];
|
|
70
|
+
return method !== undefined && path !== undefined ? { method: method.toUpperCase(), path } : null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function splitPath(path: string): string[] {
|
|
74
|
+
return path.split('/').filter((segment) => segment !== '');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function compileRoute(key: string, handler: MockRouteHandler): CompiledRoute {
|
|
78
|
+
const parsed = parseKey(key);
|
|
79
|
+
if (!parsed) throw new TypeError(`mock route "${key}" must look like "<METHOD> /path"`);
|
|
80
|
+
if (typeof handler !== 'function') throw new TypeError(`mock route "${key}" needs a handler function`);
|
|
81
|
+
return { key: `${parsed.method} ${parsed.path}`, method: parsed.method, segments: splitPath(parsed.path), handler };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function normalizeKey(key: string): string {
|
|
85
|
+
const parsed = parseKey(key);
|
|
86
|
+
return parsed ? `${parsed.method} ${parsed.path}` : key.trim();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function matchRoute(route: CompiledRoute, method: string, segments: string[]): Record<string, string> | null {
|
|
90
|
+
if (route.method !== method || route.segments.length !== segments.length) return null;
|
|
91
|
+
const params: Record<string, string> = {};
|
|
92
|
+
for (const [index, expected] of route.segments.entries()) {
|
|
93
|
+
const actual = segments[index] ?? '';
|
|
94
|
+
if (expected.startsWith(':')) {
|
|
95
|
+
try {
|
|
96
|
+
params[expected.slice(1)] = decodeURIComponent(actual);
|
|
97
|
+
} catch {
|
|
98
|
+
params[expected.slice(1)] = actual;
|
|
99
|
+
}
|
|
100
|
+
} else if (expected !== actual) {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return params;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function validateSteps(route: string, steps: unknown): MockStep[] {
|
|
108
|
+
if (!Array.isArray(steps)) throw new TypeError(`script("${route}") needs an array of steps`);
|
|
109
|
+
return steps.map((step, index) => {
|
|
110
|
+
if (step === 'ok' || step === 'reject' || step === 'timeout') return step;
|
|
111
|
+
if (typeof step === 'object' && step !== null && typeof (step as { slowMs?: unknown }).slowMs === 'number'
|
|
112
|
+
&& Number.isFinite((step as { slowMs: number }).slowMs) && (step as { slowMs: number }).slowMs >= 0) {
|
|
113
|
+
return { slowMs: (step as { slowMs: number }).slowMs };
|
|
114
|
+
}
|
|
115
|
+
throw new TypeError(`script("${route}") step ${index} must be 'ok', 'reject', 'timeout' or { slowMs: number }`);
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function bodyText(body: unknown): string | null {
|
|
120
|
+
if (body === undefined || body === null) return null;
|
|
121
|
+
if (typeof body === 'string') return body;
|
|
122
|
+
if (body instanceof Uint8Array) return new TextDecoder().decode(body);
|
|
123
|
+
if (body instanceof ArrayBuffer) return new TextDecoder().decode(new Uint8Array(body));
|
|
124
|
+
if (body instanceof URLSearchParams) return body.toString();
|
|
125
|
+
return String(body);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function parseBody(text: string | null): unknown {
|
|
129
|
+
if (text === null || text === '') return null;
|
|
130
|
+
try {
|
|
131
|
+
return JSON.parse(text) as unknown;
|
|
132
|
+
} catch {
|
|
133
|
+
return text;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function toResponse(result: MockResponse | Response): Response {
|
|
138
|
+
if (result instanceof Response) return result;
|
|
139
|
+
const headers = new Headers(result.headers ?? {});
|
|
140
|
+
let body: string | null = null;
|
|
141
|
+
if (result.body !== undefined) {
|
|
142
|
+
if (typeof result.body === 'string') {
|
|
143
|
+
body = result.body;
|
|
144
|
+
} else {
|
|
145
|
+
body = JSON.stringify(result.body);
|
|
146
|
+
if (!headers.has('content-type')) headers.set('content-type', 'application/json');
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return new Response(body, { status: result.status ?? 200, headers });
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function jsonResponse(status: number, body: unknown): Response {
|
|
153
|
+
return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function abortError(signal: AbortSignal): unknown {
|
|
157
|
+
return signal.reason ?? new DOMException('The operation was aborted', 'AbortError');
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** 等 `ms` 毫秒;`signal` 中止时立即拒绝。`ms` 为 Infinity 时只等中止。 */
|
|
161
|
+
function wait(ms: number, signal: AbortSignal | null | undefined): Promise<void> {
|
|
162
|
+
return new Promise((resolve, reject) => {
|
|
163
|
+
if (signal?.aborted) {
|
|
164
|
+
reject(abortError(signal));
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
const timer = Number.isFinite(ms) ? setTimeout(() => {
|
|
168
|
+
signal?.removeEventListener('abort', onAbort);
|
|
169
|
+
resolve();
|
|
170
|
+
}, ms) : null;
|
|
171
|
+
const onAbort = (): void => {
|
|
172
|
+
if (timer !== null) clearTimeout(timer);
|
|
173
|
+
reject(signal ? abortError(signal) : new DOMException('The operation was aborted', 'AbortError'));
|
|
174
|
+
};
|
|
175
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export function createMockExternalSystem(routes: Record<string, MockRouteHandler>): MockExternalSystem {
|
|
180
|
+
const compiled = Object.entries(routes).map(([key, handler]) => compileRoute(key, handler));
|
|
181
|
+
const byKey = new Map(compiled.map((route) => [route.key, route]));
|
|
182
|
+
const scripts = new Map<string, MockStep[]>();
|
|
183
|
+
const recorded: Array<{ route: string | null; call: MockCall }> = [];
|
|
184
|
+
|
|
185
|
+
const routeOf = (key: string): CompiledRoute => {
|
|
186
|
+
const route = byKey.get(normalizeKey(key));
|
|
187
|
+
if (!route) throw new TypeError(`mock route "${key}" is not defined; defined routes: ${[...byKey.keys()].join(', ') || '(none)'}`);
|
|
188
|
+
return route;
|
|
189
|
+
};
|
|
190
|
+
|
|
191
|
+
const mockFetch = async (input: string | URL | Request, init?: RequestInit): Promise<Response> => {
|
|
192
|
+
const request = input instanceof Request ? input : null;
|
|
193
|
+
const url = new URL(request ? request.url : String(input));
|
|
194
|
+
const method = (init?.method ?? request?.method ?? 'GET').toUpperCase();
|
|
195
|
+
const headers: Record<string, string> = {};
|
|
196
|
+
new Headers(init?.headers ?? request?.headers ?? {}).forEach((value, name) => {
|
|
197
|
+
headers[name.toLowerCase()] = value;
|
|
198
|
+
});
|
|
199
|
+
const text = init?.body !== undefined ? bodyText(init.body) : request ? await request.text() : null;
|
|
200
|
+
const body = parseBody(text);
|
|
201
|
+
const signal = init?.signal ?? request?.signal ?? null;
|
|
202
|
+
const segments = splitPath(url.pathname);
|
|
203
|
+
|
|
204
|
+
let matched: { route: CompiledRoute; params: Record<string, string> } | null = null;
|
|
205
|
+
for (const route of compiled) {
|
|
206
|
+
const params = matchRoute(route, method, segments);
|
|
207
|
+
if (params) {
|
|
208
|
+
matched = { route, params };
|
|
209
|
+
break;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
recorded.push({ route: matched?.route.key ?? null, call: { method, path: url.pathname, headers: { ...headers }, body } });
|
|
213
|
+
if (!matched) return jsonResponse(404, { code: 'NOT_FOUND', message: `no mock route for ${method} ${url.pathname}` });
|
|
214
|
+
|
|
215
|
+
const step = scripts.get(matched.route.key)?.shift() ?? 'ok';
|
|
216
|
+
if (step === 'reject') return jsonResponse(422, { code: 'REJECTED', message: 'rejected by the mock external system' });
|
|
217
|
+
if (step === 'timeout') {
|
|
218
|
+
await wait(Number.POSITIVE_INFINITY, signal);
|
|
219
|
+
} else if (typeof step === 'object') {
|
|
220
|
+
await wait(step.slowMs, signal);
|
|
221
|
+
}
|
|
222
|
+
const query: Record<string, string> = {};
|
|
223
|
+
url.searchParams.forEach((value, name) => {
|
|
224
|
+
query[name] = value;
|
|
225
|
+
});
|
|
226
|
+
const result = await matched.route.handler({ method, path: url.pathname, params: matched.params, query, headers: { ...headers }, body });
|
|
227
|
+
return toResponse(result);
|
|
228
|
+
};
|
|
229
|
+
|
|
230
|
+
return {
|
|
231
|
+
fetch: mockFetch as typeof fetch,
|
|
232
|
+
script(route, steps) {
|
|
233
|
+
const key = routeOf(route).key;
|
|
234
|
+
const queue = scripts.get(key) ?? [];
|
|
235
|
+
queue.push(...validateSteps(route, steps));
|
|
236
|
+
scripts.set(key, queue);
|
|
237
|
+
},
|
|
238
|
+
calls(route) {
|
|
239
|
+
const key = route === undefined ? null : routeOf(route).key;
|
|
240
|
+
return recorded
|
|
241
|
+
.filter((entry) => key === null || entry.route === key)
|
|
242
|
+
.map(({ call }) => ({ ...call, headers: { ...call.headers } }));
|
|
243
|
+
},
|
|
244
|
+
reset() {
|
|
245
|
+
recorded.length = 0;
|
|
246
|
+
scripts.clear();
|
|
247
|
+
},
|
|
248
|
+
};
|
|
249
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 本地网关走真实网络时的单跳请求:用 `node:http` / `node:https` 发出,连接地址固定为网关检查过的解析结果(`lookup` 只回这些地址),
|
|
3
|
+
* 关掉「检查时解析一次、fetch 再解析一次」之间的 DNS 重绑定窗口。主机名照常用于 `Host` 头、TLS 的 SNI 与证书校验。
|
|
4
|
+
*
|
|
5
|
+
* 行为与 `fetch(..., { redirect: 'manual' })` 对齐:不跟随重定向;按 `content-encoding`(gzip / deflate / br)解压正文;
|
|
6
|
+
* 返回缓冲好的 `Response`。正文(解压前后)超过 `maxBytes` 时抛 `PinnedBodyTooLargeError`;`signal` 中止时销毁连接。
|
|
7
|
+
*/
|
|
8
|
+
import { request as httpRequest, type IncomingMessage } from 'node:http';
|
|
9
|
+
import { request as httpsRequest } from 'node:https';
|
|
10
|
+
import type { LookupAddress } from 'node:dns';
|
|
11
|
+
import { brotliDecompressSync, gunzipSync, inflateRawSync, inflateSync } from 'node:zlib';
|
|
12
|
+
|
|
13
|
+
export class PinnedBodyTooLargeError extends Error {
|
|
14
|
+
constructor() {
|
|
15
|
+
super('response body too large');
|
|
16
|
+
this.name = 'PinnedBodyTooLargeError';
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** 这些状态码的 `Response` 不能带正文。 */
|
|
21
|
+
const NULL_BODY_STATUSES: ReadonlySet<number> = new Set([101, 103, 204, 205, 304]);
|
|
22
|
+
|
|
23
|
+
type LookupCallback = (error: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void;
|
|
24
|
+
|
|
25
|
+
function pinnedLookup(addresses: readonly LookupAddress[]) {
|
|
26
|
+
return (_hostname: string, options: unknown, callback: LookupCallback): void => {
|
|
27
|
+
const all = typeof options === 'object' && options !== null && (options as { all?: unknown }).all === true;
|
|
28
|
+
const first = addresses[0];
|
|
29
|
+
if (!first) {
|
|
30
|
+
callback(Object.assign(new Error('no address to connect to'), { code: 'ENOTFOUND' }), []);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (all) callback(null, [...addresses]);
|
|
34
|
+
else callback(null, first.address, first.family);
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function decompress(bytes: Buffer, encoding: string, maxBytes: number): Buffer {
|
|
39
|
+
const options = { maxOutputLength: maxBytes + 1 };
|
|
40
|
+
try {
|
|
41
|
+
switch (encoding) {
|
|
42
|
+
case 'gzip':
|
|
43
|
+
case 'x-gzip':
|
|
44
|
+
return gunzipSync(bytes, options);
|
|
45
|
+
case 'deflate':
|
|
46
|
+
try {
|
|
47
|
+
return inflateSync(bytes, options);
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (error instanceof RangeError) throw error;
|
|
50
|
+
return inflateRawSync(bytes, options);
|
|
51
|
+
}
|
|
52
|
+
case 'br':
|
|
53
|
+
return brotliDecompressSync(bytes, options);
|
|
54
|
+
default:
|
|
55
|
+
return bytes;
|
|
56
|
+
}
|
|
57
|
+
} catch (error) {
|
|
58
|
+
if (error instanceof RangeError || (error as { code?: unknown }).code === 'ERR_BUFFER_TOO_LARGE') throw new PinnedBodyTooLargeError();
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function toHeaders(response: IncomingMessage): Headers {
|
|
64
|
+
const headers = new Headers();
|
|
65
|
+
const raw = response.rawHeaders;
|
|
66
|
+
for (let index = 0; index + 1 < raw.length; index += 2) {
|
|
67
|
+
try {
|
|
68
|
+
headers.append(raw[index] ?? '', raw[index + 1] ?? '');
|
|
69
|
+
} catch {
|
|
70
|
+
// 非法的头名或头值:丢弃这一个
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return headers;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface PinnedRequestInput {
|
|
77
|
+
url: URL;
|
|
78
|
+
method: string;
|
|
79
|
+
headers: Record<string, string>;
|
|
80
|
+
body: Uint8Array | null;
|
|
81
|
+
addresses: readonly LookupAddress[];
|
|
82
|
+
signal: AbortSignal;
|
|
83
|
+
maxBytes: number;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function pinnedRequest(input: PinnedRequestInput): Promise<Response> {
|
|
87
|
+
const { url, method, signal, maxBytes } = input;
|
|
88
|
+
const send = url.protocol === 'https:' ? httpsRequest : httpRequest;
|
|
89
|
+
const body = input.body !== null && method !== 'GET' && method !== 'HEAD' ? Buffer.from(input.body) : null;
|
|
90
|
+
const headers: Record<string, string> = { ...input.headers };
|
|
91
|
+
if (body !== null) headers['content-length'] = String(body.byteLength);
|
|
92
|
+
|
|
93
|
+
return new Promise<Response>((resolve, reject) => {
|
|
94
|
+
if (signal.aborted) {
|
|
95
|
+
reject(new Error('aborted'));
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
let settled = false;
|
|
99
|
+
const fail = (error: Error): void => {
|
|
100
|
+
if (settled) return;
|
|
101
|
+
settled = true;
|
|
102
|
+
signal.removeEventListener('abort', onAbort);
|
|
103
|
+
reject(error);
|
|
104
|
+
};
|
|
105
|
+
const req = send(url, { method, headers, lookup: pinnedLookup(input.addresses) as never }, (response) => {
|
|
106
|
+
const chunks: Buffer[] = [];
|
|
107
|
+
let size = 0;
|
|
108
|
+
response.on('data', (chunk: Buffer) => {
|
|
109
|
+
size += chunk.byteLength;
|
|
110
|
+
if (size > maxBytes) {
|
|
111
|
+
req.destroy();
|
|
112
|
+
fail(new PinnedBodyTooLargeError());
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
chunks.push(chunk);
|
|
116
|
+
});
|
|
117
|
+
response.on('error', (error) => fail(error));
|
|
118
|
+
response.on('end', () => {
|
|
119
|
+
if (settled) return;
|
|
120
|
+
try {
|
|
121
|
+
const status = response.statusCode ?? 0;
|
|
122
|
+
const encoding = (response.headers['content-encoding'] ?? '').trim().toLowerCase();
|
|
123
|
+
const bytes = method === 'HEAD' ? Buffer.alloc(0) : decompress(Buffer.concat(chunks), encoding, maxBytes);
|
|
124
|
+
const responseHeaders = toHeaders(response);
|
|
125
|
+
// DOM lib 下 `Buffer<ArrayBufferLike>` 不算 BodyInit(CLI 带 DOM lib 编译本文件),运行时照收。
|
|
126
|
+
const body = NULL_BODY_STATUSES.has(status) || bytes.byteLength === 0 ? null : bytes as unknown as NonNullable<ConstructorParameters<typeof Response>[0]>;
|
|
127
|
+
const result = new Response(body, { status, headers: responseHeaders });
|
|
128
|
+
settled = true;
|
|
129
|
+
signal.removeEventListener('abort', onAbort);
|
|
130
|
+
resolve(result);
|
|
131
|
+
} catch (error) {
|
|
132
|
+
fail(error instanceof Error ? error : new Error(String(error)));
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
const onAbort = (): void => {
|
|
137
|
+
req.destroy(new Error('aborted'));
|
|
138
|
+
fail(new Error('aborted'));
|
|
139
|
+
};
|
|
140
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
141
|
+
req.on('error', (error) => fail(error));
|
|
142
|
+
req.end(body ?? undefined);
|
|
143
|
+
});
|
|
144
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 夹具录制:挂在本地模拟网关的 `onExchange` 上,把未遮蔽的原始往来(`LocalExchange`)按遮蔽规则写成 `FixtureExchange`。
|
|
3
|
+
*
|
|
4
|
+
* 遮蔽规则(与 shared `connector-redaction.ts` 同一套,实现见 `redaction.ts`):
|
|
5
|
+
* - 长度 ≥ 4 的 Secret 明文 → `***`(URL、头、body、db 的 SQL / 参数 / 行、crypto 的 key / data、错误文案);
|
|
6
|
+
* - `CREDENTIAL_HEADER_NAMES` 六个请求 / 响应头的值 → `***`;
|
|
7
|
+
* - 含句柄 `ontos:secret:` 的 query 值 → `***`;请求 body 与头里的句柄原样保留(网关替换前的形态)。
|
|
8
|
+
*
|
|
9
|
+
* db:一次查询的分页(`db.query` + 若干 `db.next`)合成一条往来,`rows` 为全部行;中途失败时记下已取到的行与错误。
|
|
10
|
+
* crypto 调用失败时不录(回放时 `hash` 本地计算,`hmac` / `sign` 报 `FIXTURE_EXCHANGE_MISMATCH`)。
|
|
11
|
+
*/
|
|
12
|
+
import type { CryptoRequest, DbNextResponse, DbQueryResponse, EgressHttpRequest, EgressHttpResponse } from '../runtime/protocol';
|
|
13
|
+
import { canonicalRequestUrl, type FixtureExchange } from './fixtures';
|
|
14
|
+
import type { LocalExchange } from './gateway';
|
|
15
|
+
import { redactHeaders, redactSecretValues, redactUrl, secretRedactionTexts } from './redaction';
|
|
16
|
+
|
|
17
|
+
type DbExchange = Extract<FixtureExchange, { kind: 'db' }>;
|
|
18
|
+
|
|
19
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
20
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** 深拷贝成 JSON 值,字符串逐个遮蔽明文。 */
|
|
24
|
+
function redactDeep(value: unknown, secrets: readonly string[]): unknown {
|
|
25
|
+
const plain: unknown = value === undefined ? null : JSON.parse(JSON.stringify(value));
|
|
26
|
+
const walk = (item: unknown): unknown => {
|
|
27
|
+
if (typeof item === 'string') return redactSecretValues(item, secrets);
|
|
28
|
+
if (Array.isArray(item)) return item.map(walk);
|
|
29
|
+
if (isRecord(item)) return Object.fromEntries(Object.entries(item).map(([key, child]) => [key, walk(child)]));
|
|
30
|
+
return item;
|
|
31
|
+
};
|
|
32
|
+
return walk(plain);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface FixtureRecorder {
|
|
36
|
+
onExchange(exchange: LocalExchange): void;
|
|
37
|
+
/** 至今录下的往来(按 `seq` 升序,深拷贝)。 */
|
|
38
|
+
exchanges(): FixtureExchange[];
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* `secrets`:配置里的 Secret 字段 → 明文;`extraSecrets`:其余要遮蔽的明文(如本地 `db` 的密码)。
|
|
43
|
+
* 遮蔽时每个明文连同它的 URL 编码与表单编码写法一起替换。
|
|
44
|
+
*/
|
|
45
|
+
export function createRecorder(options: { secrets: Record<string, string>; extraSecrets?: readonly string[] }): FixtureRecorder {
|
|
46
|
+
const secrets = secretRedactionTexts([...Object.values(options.secrets), ...(options.extraSecrets ?? [])]
|
|
47
|
+
.filter((value): value is string => typeof value === 'string'));
|
|
48
|
+
const redact = (text: string): string => redactSecretValues(text, secrets);
|
|
49
|
+
const recorded: FixtureExchange[] = [];
|
|
50
|
+
const openCursors = new Map<string, DbExchange>();
|
|
51
|
+
let seq = 0;
|
|
52
|
+
|
|
53
|
+
const errorOf = (exchange: LocalExchange): { code: string; message: string } | null =>
|
|
54
|
+
exchange.error ? { code: exchange.error.code, message: redact(exchange.error.message) } : null;
|
|
55
|
+
|
|
56
|
+
const recordHttp = (exchange: LocalExchange): void => {
|
|
57
|
+
const request = exchange.request as Partial<EgressHttpRequest>;
|
|
58
|
+
if (!isRecord(request) || typeof request.url !== 'string' || typeof request.method !== 'string') return;
|
|
59
|
+
const bodyEncoding = request.bodyEncoding === 'base64' ? 'base64' : 'utf8';
|
|
60
|
+
const body = typeof request.body === 'string' ? (bodyEncoding === 'utf8' ? redact(request.body) : request.body) : null;
|
|
61
|
+
const response = exchange.response as EgressHttpResponse | null;
|
|
62
|
+
seq += 1;
|
|
63
|
+
recorded.push({
|
|
64
|
+
seq,
|
|
65
|
+
kind: 'http',
|
|
66
|
+
request: {
|
|
67
|
+
method: request.method,
|
|
68
|
+
url: redactUrl(canonicalRequestUrl(request.url, request.query), secrets),
|
|
69
|
+
headers: redactHeaders(isRecord(request.headers) ? request.headers as Record<string, string> : {}, secrets),
|
|
70
|
+
body,
|
|
71
|
+
bodyEncoding,
|
|
72
|
+
},
|
|
73
|
+
response: exchange.error || !isRecord(response)
|
|
74
|
+
? null
|
|
75
|
+
: {
|
|
76
|
+
status: response.status,
|
|
77
|
+
headers: redactHeaders(response.headers ?? {}, secrets),
|
|
78
|
+
body: response.bodyEncoding === 'utf8' ? redact(response.body) : response.body,
|
|
79
|
+
bodyEncoding: response.bodyEncoding,
|
|
80
|
+
},
|
|
81
|
+
error: errorOf(exchange) ?? (isRecord(response) ? null : { code: 'HOST_CALL_FAILED', message: 'no response was recorded' }),
|
|
82
|
+
});
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const recordDb = (exchange: LocalExchange): void => {
|
|
86
|
+
const request = exchange.request;
|
|
87
|
+
if (!isRecord(request)) return;
|
|
88
|
+
if (typeof request.sql === 'string') {
|
|
89
|
+
const response = exchange.response as DbQueryResponse | null;
|
|
90
|
+
seq += 1;
|
|
91
|
+
const entry: DbExchange = {
|
|
92
|
+
seq,
|
|
93
|
+
kind: 'db',
|
|
94
|
+
sql: redact(request.sql),
|
|
95
|
+
params: redactDeep(Array.isArray(request.params) ? request.params : [], secrets) as unknown[],
|
|
96
|
+
fields: isRecord(response) && Array.isArray(response.fields) ? response.fields.map((field) => ({ name: field.name, dataTypeId: field.dataTypeId })) : [],
|
|
97
|
+
rows: isRecord(response) && Array.isArray(response.rows) ? redactDeep(response.rows, secrets) as Array<Record<string, unknown>> : [],
|
|
98
|
+
error: errorOf(exchange),
|
|
99
|
+
};
|
|
100
|
+
recorded.push(entry);
|
|
101
|
+
if (isRecord(response) && typeof response.cursorId === 'string' && response.done === false) openCursors.set(response.cursorId, entry);
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
if (typeof request.cursorId === 'string') {
|
|
105
|
+
const entry = openCursors.get(request.cursorId);
|
|
106
|
+
if (!entry) return;
|
|
107
|
+
const response = exchange.response as DbNextResponse | null;
|
|
108
|
+
if (isRecord(response) && Array.isArray(response.rows)) entry.rows.push(...(redactDeep(response.rows, secrets) as Array<Record<string, unknown>>));
|
|
109
|
+
const error = errorOf(exchange);
|
|
110
|
+
if (error) entry.error = error;
|
|
111
|
+
if (error || !isRecord(response) || response.done !== false) openCursors.delete(request.cursorId);
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
const recordCrypto = (exchange: LocalExchange): void => {
|
|
116
|
+
if (exchange.error || !isRecord(exchange.request) || !isRecord(exchange.response) || typeof exchange.response.value !== 'string') return;
|
|
117
|
+
const request = redactDeep(exchange.request, secrets) as CryptoRequest;
|
|
118
|
+
seq += 1;
|
|
119
|
+
recorded.push({ seq, kind: 'crypto', request, value: redact(exchange.response.value) });
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
return {
|
|
123
|
+
onExchange(exchange) {
|
|
124
|
+
switch (exchange.kind) {
|
|
125
|
+
case 'http':
|
|
126
|
+
recordHttp(exchange);
|
|
127
|
+
return;
|
|
128
|
+
case 'db':
|
|
129
|
+
recordDb(exchange);
|
|
130
|
+
return;
|
|
131
|
+
case 'crypto':
|
|
132
|
+
recordCrypto(exchange);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
exchanges() {
|
|
137
|
+
return JSON.parse(JSON.stringify(recorded)) as FixtureExchange[];
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 遮蔽规则(录制夹具、本地 `read` 的请求行、本地运行日志共用),与 shared `utils/connector-redaction.ts` 同一套规则;
|
|
3
|
+
* SDK 不依赖 shared,这里另写一份,`test/testing/redaction.test.ts` 对照两份实现的结果。
|
|
4
|
+
*
|
|
5
|
+
* - 长度 ≥ 4 的 Secret 明文 → `***`(长的先替换,避免短值截断长值);
|
|
6
|
+
* - `CREDENTIAL_HEADER_NAMES` 里的请求 / 响应头(小写比较)整值 → `***`;
|
|
7
|
+
* - 含句柄 `ontos:secret:` 的 query 值 → `***`;请求里的句柄本身原样保留(网关替换前的形态)。
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/** 凭证类请求 / 响应头(小写);与 shared 的 `CREDENTIAL_HEADER_NAMES` 同值。 */
|
|
11
|
+
export const CREDENTIAL_HEADER_NAMES: readonly string[] = Object.freeze([
|
|
12
|
+
'authorization',
|
|
13
|
+
'proxy-authorization',
|
|
14
|
+
'cookie',
|
|
15
|
+
'set-cookie',
|
|
16
|
+
'x-api-key',
|
|
17
|
+
'x-auth-token',
|
|
18
|
+
]);
|
|
19
|
+
|
|
20
|
+
/** 遮蔽的最短明文长度:更短的值误伤面太大,不替换。 */
|
|
21
|
+
export const REDACT_MIN_LENGTH = 4;
|
|
22
|
+
|
|
23
|
+
export const REDACTED = '***';
|
|
24
|
+
|
|
25
|
+
const CREDENTIAL_HEADER_SET: ReadonlySet<string> = new Set(CREDENTIAL_HEADER_NAMES);
|
|
26
|
+
|
|
27
|
+
/** 句柄的两种写法:原文 `ontos:secret:` 与 URL 编码后的 `ontos%3Asecret%3A`。 */
|
|
28
|
+
const HANDLE_PATTERN = /ontos(?::|%3[Aa])secret(?::|%3[Aa])/;
|
|
29
|
+
|
|
30
|
+
/** 文本里是否含凭证句柄(原文或 URL 编码写法)。 */
|
|
31
|
+
export function containsSecretHandle(text: string): boolean {
|
|
32
|
+
return HANDLE_PATTERN.test(text);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** 把文本里长度 ≥ 4 的 Secret 明文替换为 `***`(先替换较长的)。 */
|
|
36
|
+
export function redactSecretValues(text: string, secrets: readonly string[]): string {
|
|
37
|
+
const candidates = [...new Set(secrets.filter((secret) => typeof secret === 'string' && secret.length >= REDACT_MIN_LENGTH))]
|
|
38
|
+
.sort((a, b) => b.length - a.length);
|
|
39
|
+
let result = text;
|
|
40
|
+
for (const secret of candidates) {
|
|
41
|
+
if (result.includes(secret)) result = result.split(secret).join(REDACTED);
|
|
42
|
+
}
|
|
43
|
+
return result;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* 遮蔽用的明文清单:每个 Secret 加上它进 URL 后的两种写法——`encodeURIComponent` 与表单编码(`URLSearchParams`,空格写成 `+`)。
|
|
48
|
+
* 句柄替换进 URL 时用的是编码写法,响应的 `url`、日志与录制里出现的也是它,只遮明文会漏掉。
|
|
49
|
+
*/
|
|
50
|
+
export function secretRedactionTexts(secrets: readonly string[]): string[] {
|
|
51
|
+
const result = new Set<string>();
|
|
52
|
+
for (const secret of secrets) {
|
|
53
|
+
if (typeof secret !== 'string') continue;
|
|
54
|
+
result.add(secret);
|
|
55
|
+
result.add(encodeURIComponent(secret));
|
|
56
|
+
result.add(new URLSearchParams([['', secret]]).toString().slice(1));
|
|
57
|
+
}
|
|
58
|
+
return [...result];
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** 凭证类头(名字按小写比较)的值换成 `***`,其余头的值做 `redactSecretValues`;头名保持原样。 */
|
|
62
|
+
export function redactHeaders(headers: Record<string, string>, secrets: readonly string[]): Record<string, string> {
|
|
63
|
+
const result: Record<string, string> = {};
|
|
64
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
65
|
+
result[name] = CREDENTIAL_HEADER_SET.has(name.toLowerCase()) ? REDACTED : redactSecretValues(String(value), secrets);
|
|
66
|
+
}
|
|
67
|
+
return result;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** query 参数:值含句柄的换成 `***`,其余值做 `redactSecretValues`。 */
|
|
71
|
+
export function redactQuery(query: Record<string, string | string[]>, secrets: readonly string[]): Record<string, string | string[]> {
|
|
72
|
+
const one = (value: string): string => (containsSecretHandle(value) ? REDACTED : redactSecretValues(value, secrets));
|
|
73
|
+
const result: Record<string, string | string[]> = {};
|
|
74
|
+
for (const [name, value] of Object.entries(query)) result[name] = Array.isArray(value) ? value.map(one) : one(value);
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** URL:值含句柄的 query 参数换成 `***`,再遮蔽明文;解析不了时只做明文遮蔽。与 connector 的 `traceUrl` 相同。 */
|
|
79
|
+
export function redactUrl(url: string, secrets: readonly string[]): string {
|
|
80
|
+
let text = url;
|
|
81
|
+
try {
|
|
82
|
+
const parsed = new URL(url);
|
|
83
|
+
const entries = [...parsed.searchParams.entries()];
|
|
84
|
+
if (entries.some(([, value]) => containsSecretHandle(value))) {
|
|
85
|
+
const masked = new URLSearchParams();
|
|
86
|
+
for (const [name, value] of entries) masked.append(name, containsSecretHandle(value) ? REDACTED : value);
|
|
87
|
+
parsed.search = masked.toString();
|
|
88
|
+
text = parsed.toString();
|
|
89
|
+
}
|
|
90
|
+
} catch {
|
|
91
|
+
// 原样遮蔽
|
|
92
|
+
}
|
|
93
|
+
return redactSecretValues(text, secrets);
|
|
94
|
+
}
|