@deepexi/datasense-connector 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -139
- package/package.json +19 -28
- package/scripts/build.ts +188 -0
- package/src/define.ts +267 -0
- package/src/descriptor.ts +116 -0
- package/src/errors.ts +105 -0
- package/src/index.ts +108 -0
- package/src/internal/location.ts +23 -0
- package/src/runtime/guest/bridge.ts +238 -0
- package/src/runtime/guest/cache.ts +43 -0
- package/src/runtime/guest/check.ts +17 -0
- package/src/runtime/guest/crypto.ts +166 -0
- package/src/runtime/guest/ctx.ts +157 -0
- package/src/runtime/guest/db.ts +59 -0
- package/src/runtime/guest/failure.ts +69 -0
- package/src/runtime/guest/http.ts +194 -0
- package/src/runtime/guest/index.ts +102 -0
- package/{runtime/guest/limits.d.ts → src/runtime/guest/limits.ts} +12 -12
- package/src/runtime/guest/loader.ts +210 -0
- package/src/runtime/guest/log.ts +47 -0
- package/src/runtime/guest/map.ts +45 -0
- package/src/runtime/guest/operation.ts +193 -0
- package/src/runtime/guest/paginate.ts +169 -0
- package/src/runtime/guest/pull.ts +177 -0
- package/src/runtime/guest/push.ts +92 -0
- package/src/runtime/guest/signal.ts +14 -0
- package/src/runtime/guest/suggest.ts +51 -0
- package/src/runtime/guest/time.ts +137 -0
- package/src/runtime/guest-runtime.ts +75 -0
- package/src/runtime/index.ts +53 -0
- package/src/runtime/protocol.ts +557 -0
- package/src/runtime/session.ts +216 -0
- package/src/testing/child.ts +21 -0
- package/src/testing/compliance.ts +343 -0
- package/src/testing/engine.ts +445 -0
- package/src/testing/fixtures.ts +591 -0
- package/src/testing/gateway.ts +482 -0
- package/src/testing/import-guard.ts +221 -0
- package/src/testing/index.ts +43 -0
- package/src/testing/local-db.ts +226 -0
- package/src/testing/mock-external-system.ts +249 -0
- package/src/testing/pinned-http.ts +144 -0
- package/src/testing/recorder.ts +140 -0
- package/src/testing/redaction.ts +94 -0
- package/src/testing/replay.ts +437 -0
- package/src/testing/run-tests.ts +115 -0
- package/src/testing/schema.ts +70 -0
- package/src/testing/source-loader.ts +374 -0
- package/src/testing/types.ts +78 -0
- package/src/types.ts +329 -0
- package/src/validate.ts +267 -0
- package/test/build.test.ts +159 -0
- package/test/define.test-d.ts +182 -0
- package/test/fixtures/acceptance.ts +156 -0
- package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
- package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
- package/test/fixtures/connectors/http-pull/package.json +5 -0
- package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
- package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
- package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
- package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
- package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
- package/test/fixtures/connectors/pg-pull/package.json +5 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
- package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
- package/test/fixtures/descriptor.ts +54 -0
- package/test/fixtures/kingdee-example.ts +82 -0
- package/test/fixtures/record-acceptance.ts +32 -0
- package/test/runtime/ctx.test.ts +434 -0
- package/test/runtime/helpers.ts +157 -0
- package/test/runtime/jobs.test.ts +444 -0
- package/test/runtime/operation.test.ts +175 -0
- package/test/runtime/session.test.ts +229 -0
- package/test/testing/acceptance-fixtures.test.ts +69 -0
- package/test/testing/compliance.test.ts +247 -0
- package/test/testing/engine.test.ts +283 -0
- package/test/testing/fixtures.test.ts +221 -0
- package/test/testing/gateway.test.ts +337 -0
- package/test/testing/import-guard.test.ts +45 -0
- package/test/testing/mock-external-system.test.ts +144 -0
- package/test/testing/operation-fixtures.test.ts +160 -0
- package/test/testing/recorder.test.ts +163 -0
- package/test/testing/redaction.test.ts +54 -0
- package/test/testing/replay.test.ts +207 -0
- package/test/testing/run-tests.test.ts +137 -0
- package/test/testing/source-loader.test.ts +220 -0
- package/test/validate-config.test.ts +104 -0
- package/tsconfig.build.json +23 -0
- package/tsconfig.json +19 -0
- package/cjs/index.cjs +0 -1015
- package/cjs/runtime.cjs +0 -752
- package/cjs/testing.cjs +0 -3591
- package/errors.d.ts +0 -17
- package/errors.js +0 -53
- package/guest.cjs +0 -2659
- package/index.d.ts +0 -13
- package/index.js +0 -7
- package/internal/location.d.ts +0 -6
- package/internal/location.js +0 -8
- package/presentation.d.ts +0 -38
- package/presentation.js +0 -268
- package/runtime/guest/bridge.d.ts +0 -53
- package/runtime/guest/bridge.js +0 -187
- package/runtime/guest/cache.d.ts +0 -3
- package/runtime/guest/cache.js +0 -42
- package/runtime/guest/crypto.d.ts +0 -13
- package/runtime/guest/crypto.js +0 -177
- package/runtime/guest/ctx.d.ts +0 -30
- package/runtime/guest/ctx.js +0 -124
- package/runtime/guest/db.d.ts +0 -6
- package/runtime/guest/db.js +0 -59
- package/runtime/guest/discover.d.ts +0 -29
- package/runtime/guest/discover.js +0 -51
- package/runtime/guest/failure.d.ts +0 -24
- package/runtime/guest/failure.js +0 -63
- package/runtime/guest/http.d.ts +0 -4
- package/runtime/guest/http.js +0 -204
- package/runtime/guest/index.d.ts +0 -12
- package/runtime/guest/index.js +0 -89
- package/runtime/guest/limits.js +0 -12
- package/runtime/guest/loader.d.ts +0 -15
- package/runtime/guest/loader.js +0 -42
- package/runtime/guest/log.d.ts +0 -10
- package/runtime/guest/log.js +0 -36
- package/runtime/guest/map.d.ts +0 -4
- package/runtime/guest/map.js +0 -45
- package/runtime/guest/paginate.d.ts +0 -6
- package/runtime/guest/paginate.js +0 -180
- package/runtime/guest/pull.d.ts +0 -12
- package/runtime/guest/pull.js +0 -150
- package/runtime/guest/push.d.ts +0 -18
- package/runtime/guest/push.js +0 -80
- package/runtime/guest/signal.d.ts +0 -3
- package/runtime/guest/signal.js +0 -12
- package/runtime/guest/time.d.ts +0 -5
- package/runtime/guest/time.js +0 -127
- package/runtime/guest-runtime.d.ts +0 -7
- package/runtime/guest-runtime.js +0 -51
- package/runtime/index.d.ts +0 -18
- package/runtime/index.js +0 -3
- package/runtime/protocol.d.ts +0 -276
- package/runtime/protocol.js +0 -409
- package/runtime/session.d.ts +0 -12
- package/runtime/session.js +0 -191
- package/t.d.ts +0 -18
- package/t.js +0 -26
- package/testing/child.cjs +0 -350
- package/testing/child.d.ts +0 -1
- package/testing/child.js +0 -12
- package/testing/compliance.d.ts +0 -41
- package/testing/compliance.js +0 -291
- package/testing/engine.d.ts +0 -66
- package/testing/engine.js +0 -339
- package/testing/fixtures.d.ts +0 -145
- package/testing/fixtures.js +0 -407
- package/testing/gateway.d.ts +0 -65
- package/testing/gateway.js +0 -402
- package/testing/import-guard.d.ts +0 -57
- package/testing/import-guard.js +0 -187
- package/testing/index.d.ts +0 -33
- package/testing/index.js +0 -10
- package/testing/local-db.d.ts +0 -27
- package/testing/local-db.js +0 -181
- package/testing/pinned-http.d.ts +0 -14
- package/testing/pinned-http.js +0 -125
- package/testing/recorder.d.ts +0 -15
- package/testing/recorder.js +0 -116
- package/testing/redaction.d.ts +0 -28
- package/testing/redaction.js +0 -67
- package/testing/replay.d.ts +0 -50
- package/testing/replay.js +0 -313
- package/testing/schema.d.ts +0 -13
- package/testing/schema.js +0 -65
- package/testing/source-loader.d.ts +0 -19
- package/testing/source-loader.js +0 -238
- package/testing/types.d.ts +0 -79
- package/testing/types.js +0 -18
- package/types.d.ts +0 -302
- package/types.js +0 -13
- package/validate.d.ts +0 -78
- package/validate.js +0 -597
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 宿主侧的 guest 会话(在 runner 的执行子进程里运行,不在 vm 里)。
|
|
3
|
+
*
|
|
4
|
+
* 收到 `execute` 后建一个新的 vm 上下文,先执行 guest 运行时源码并调用它的 `bootGuest()`,再把连接器代码编译成
|
|
5
|
+
* `(function (module, exports, require) {…})` 交给 guest 加载。之后宿主与 guest 之间只有字符串:
|
|
6
|
+
* 父进程的 `reply` / `cancel` 经 `handle(json)` 交进去,guest 待发的消息经 `poll()` 取出、原样转给父进程。
|
|
7
|
+
*
|
|
8
|
+
* 宿主绝不往上下文里放宿主的函数或对象(`constructor` 链能拿到宿主的 `Function`),也不 `await` 上下文里的 Promise。
|
|
9
|
+
* 上下文里没有定时器:guest 的 `sleep` 登记到期时间,`poll()` 回报 `wake`,宿主按它定时再 poll。
|
|
10
|
+
* 每次交进消息或触发定时器之后,在下一个宏任务里 poll(此时上下文里的微任务已经跑完)。
|
|
11
|
+
*
|
|
12
|
+
* 会话不发 `ready`:由子进程入口在接好 IPC 之后自己发。
|
|
13
|
+
*/
|
|
14
|
+
import vm from 'node:vm';
|
|
15
|
+
import { parseParentMessage, type GuestFailureCode } from './protocol';
|
|
16
|
+
|
|
17
|
+
/** guest 交给宿主的接口;三个函数都属于上下文,参数与返回值只有字符串和上下文里的值。 */
|
|
18
|
+
interface GuestApi {
|
|
19
|
+
start(json: string, factory: unknown): void;
|
|
20
|
+
handle(json: string): void;
|
|
21
|
+
poll(): unknown;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const GUEST_FILENAME = 'ontos-guest.js';
|
|
25
|
+
const CONNECTOR_FILENAME = 'connector.js';
|
|
26
|
+
|
|
27
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
28
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** 从上下文里抛出的值只取 message 字符串;读属性本身也可能抛错。 */
|
|
32
|
+
function guestErrorMessage(error: unknown): string {
|
|
33
|
+
try {
|
|
34
|
+
if (error !== null && typeof error === 'object') {
|
|
35
|
+
const message = (error as { message?: unknown }).message;
|
|
36
|
+
if (typeof message === 'string') return message;
|
|
37
|
+
}
|
|
38
|
+
} catch {
|
|
39
|
+
// 落到缺省文案
|
|
40
|
+
}
|
|
41
|
+
return 'unknown error';
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function bootGuest(context: vm.Context, guestSource: string): GuestApi {
|
|
45
|
+
// guest 是 CJS 打包产物;module / exports 在上下文里创建,不用宿主对象。
|
|
46
|
+
const wrapped = `(() => { const module = { exports: {} }; const exports = module.exports; ${guestSource}\n; return module.exports; })()`;
|
|
47
|
+
const exported = new vm.Script(wrapped, { filename: GUEST_FILENAME }).runInContext(context) as { bootGuest?: unknown };
|
|
48
|
+
const boot = exported.bootGuest;
|
|
49
|
+
if (typeof boot !== 'function') throw new Error('guest runtime does not export bootGuest');
|
|
50
|
+
const api = (boot as () => unknown)() as Record<string, unknown>;
|
|
51
|
+
const { start, handle, poll } = api;
|
|
52
|
+
if (typeof start !== 'function' || typeof handle !== 'function' || typeof poll !== 'function') throw new Error('guest runtime returned an incomplete API');
|
|
53
|
+
// 立即取下函数引用:之后连接器代码改写这个对象也不影响宿主。
|
|
54
|
+
return { start: start as GuestApi['start'], handle: handle as GuestApi['handle'], poll: poll as GuestApi['poll'] };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
type BootOutcome = { ok: true; guest: GuestApi; factory: unknown } | { ok: false; code: GuestFailureCode; message: string };
|
|
58
|
+
|
|
59
|
+
/** vm 启动:新上下文里执行 guest 运行时并 `bootGuest()`,再在同一上下文里编译连接器代码。 */
|
|
60
|
+
function bootInVm(guestSource: string, code: string): BootOutcome {
|
|
61
|
+
const context = vm.createContext(Object.create(null), { codeGeneration: { strings: false, wasm: false } });
|
|
62
|
+
let guest: GuestApi;
|
|
63
|
+
try {
|
|
64
|
+
guest = bootGuest(context, guestSource);
|
|
65
|
+
} catch (error) {
|
|
66
|
+
return { ok: false, code: 'CONNECTOR_LOAD_FAILED', message: `connector runtime failed to start: ${guestErrorMessage(error)}` };
|
|
67
|
+
}
|
|
68
|
+
try {
|
|
69
|
+
// 包装与第一行同行(Bun 的 vm.Script 不认 lineOffset),调用栈的行号直接对应打包产物。
|
|
70
|
+
const factory = new vm.Script(`(function (module, exports, require) {${code}\n})`, { filename: CONNECTOR_FILENAME }).runInContext(context);
|
|
71
|
+
return { ok: true, guest, factory };
|
|
72
|
+
} catch (error) {
|
|
73
|
+
return { ok: false, code: 'CONNECTOR_LOAD_FAILED', message: `connector code failed to compile: ${guestErrorMessage(error)}` };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** 子进程入口里调用:收到 execute 时建 vm 上下文、加载其中的 guest 字符串;之后只吃字符串消息。 */
|
|
78
|
+
export function createGuestSession(send: (json: string) => void): { handle(json: string): void } {
|
|
79
|
+
return createSession(send, bootInVm);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* 进程内会话(`./testing` 的 `createInProcessExecutor`,给 vitest 用):不建 vm,在当前全局里重新求值一份 guest 运行时
|
|
84
|
+
* (模块状态每次执行独立),连接器是调用方已导入的模块(`{ descriptor, implementations }`),`execute.code` 被忽略。消息协议与 vm 会话完全相同。
|
|
85
|
+
* 这里不是隔离边界:连接器代码与宿主同进程同全局,只用于作者在自己机器上跑测试。
|
|
86
|
+
*/
|
|
87
|
+
export function createInProcessGuestSession(send: (json: string) => void, connectorModule: unknown): { handle(json: string): void } {
|
|
88
|
+
return createSession(send, (guestSource) => {
|
|
89
|
+
let exported: { bootLocalGuest?: unknown };
|
|
90
|
+
try {
|
|
91
|
+
const module = { exports: {} as unknown };
|
|
92
|
+
const noRequire = (specifier: string): never => { throw new Error(`guest runtime cannot require ${specifier}`); };
|
|
93
|
+
// 受信任的 guest 运行时(SDK 自己的打包产物)
|
|
94
|
+
const load = new Function('module', 'exports', 'require', guestSource) as (module: unknown, exports: unknown, require: unknown) => void;
|
|
95
|
+
load(module, module.exports, noRequire);
|
|
96
|
+
exported = module.exports as { bootLocalGuest?: unknown };
|
|
97
|
+
} catch (error) {
|
|
98
|
+
return { ok: false, code: 'CONNECTOR_LOAD_FAILED', message: `connector runtime failed to start: ${guestErrorMessage(error)}` };
|
|
99
|
+
}
|
|
100
|
+
if (typeof exported.bootLocalGuest !== 'function') {
|
|
101
|
+
return { ok: false, code: 'CONNECTOR_LOAD_FAILED', message: 'connector runtime failed to start: guest runtime does not export bootLocalGuest' };
|
|
102
|
+
}
|
|
103
|
+
const guest = (exported.bootLocalGuest as () => GuestApi)();
|
|
104
|
+
const factory = (module: { exports: unknown }): void => {
|
|
105
|
+
module.exports = connectorModule;
|
|
106
|
+
};
|
|
107
|
+
return { ok: true, guest, factory };
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function createSession(send: (json: string) => void, boot: (guestSource: string, code: string) => BootOutcome): { handle(json: string): void } {
|
|
112
|
+
let guest: GuestApi | null = null;
|
|
113
|
+
let started = false;
|
|
114
|
+
let done = false;
|
|
115
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
116
|
+
let timerAt = Number.POSITIVE_INFINITY;
|
|
117
|
+
|
|
118
|
+
const stop = () => {
|
|
119
|
+
done = true;
|
|
120
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
121
|
+
timer = undefined;
|
|
122
|
+
};
|
|
123
|
+
|
|
124
|
+
const fail = (code: GuestFailureCode, message: string) => {
|
|
125
|
+
if (done) return;
|
|
126
|
+
stop();
|
|
127
|
+
send(JSON.stringify({ type: 'failure', code, message, logs: [] }));
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
const schedule = (delay: number) => {
|
|
131
|
+
if (done) return;
|
|
132
|
+
const at = Date.now() + delay;
|
|
133
|
+
if (timer !== undefined && timerAt <= at) return;
|
|
134
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
135
|
+
timerAt = at;
|
|
136
|
+
timer = setTimeout(pump, delay);
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
function pump(): void {
|
|
140
|
+
timer = undefined;
|
|
141
|
+
timerAt = Number.POSITIVE_INFINITY;
|
|
142
|
+
if (done || !guest) return;
|
|
143
|
+
let text: unknown;
|
|
144
|
+
try {
|
|
145
|
+
text = guest.poll();
|
|
146
|
+
} catch (error) {
|
|
147
|
+
fail('FATAL_ERROR', `connector runtime failed: ${guestErrorMessage(error)}`);
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
let parsed: unknown;
|
|
151
|
+
try {
|
|
152
|
+
parsed = typeof text === 'string' ? JSON.parse(text) : null;
|
|
153
|
+
} catch {
|
|
154
|
+
parsed = null;
|
|
155
|
+
}
|
|
156
|
+
if (!isRecord(parsed) || !Array.isArray(parsed.out)) {
|
|
157
|
+
fail('FATAL_ERROR', 'connector runtime returned a malformed poll result');
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
for (const message of parsed.out) {
|
|
161
|
+
if (!isRecord(message)) continue;
|
|
162
|
+
send(JSON.stringify(message));
|
|
163
|
+
if (message.type === 'result' || message.type === 'failure') {
|
|
164
|
+
stop();
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
if (parsed.out.length > 0 || parsed.fired === true) schedule(0);
|
|
169
|
+
else if (typeof parsed.wake === 'number' && parsed.wake >= 0) schedule(parsed.wake);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const execute = (guestSource: string, code: string, payload: string) => {
|
|
173
|
+
const booted = boot(guestSource, code);
|
|
174
|
+
if (!booted.ok) {
|
|
175
|
+
fail(booted.code, booted.message);
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
guest = booted.guest;
|
|
179
|
+
try {
|
|
180
|
+
guest.start(payload, booted.factory);
|
|
181
|
+
} catch (error) {
|
|
182
|
+
fail('FATAL_ERROR', `connector runtime failed: ${guestErrorMessage(error)}`);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
schedule(0);
|
|
186
|
+
};
|
|
187
|
+
|
|
188
|
+
return {
|
|
189
|
+
handle(json: string): void {
|
|
190
|
+
if (done || typeof json !== 'string') return;
|
|
191
|
+
let raw: unknown;
|
|
192
|
+
try {
|
|
193
|
+
raw = JSON.parse(json);
|
|
194
|
+
} catch {
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
const message = parseParentMessage(raw);
|
|
198
|
+
if (!message) return;
|
|
199
|
+
if (message.type === 'execute') {
|
|
200
|
+
if (started) return;
|
|
201
|
+
started = true;
|
|
202
|
+
execute(message.guest, message.code, JSON.stringify({ expect: message.expect, job: message.job }));
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
if (!guest) return;
|
|
206
|
+
try {
|
|
207
|
+
// 重新序列化校验过的消息,guest 只看到契约里的字段。
|
|
208
|
+
guest.handle(JSON.stringify(message));
|
|
209
|
+
} catch (error) {
|
|
210
|
+
fail('FATAL_ERROR', `connector runtime failed: ${guestErrorMessage(error)}`);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
schedule(0);
|
|
214
|
+
},
|
|
215
|
+
};
|
|
216
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createVmExecutor` 的子进程入口(父进程用空环境变量 fork,不被 import)。与 runner 的子进程入口相同:
|
|
3
|
+
* IPC 收到的 JSON 字符串交给 guest 会话(`createGuestSession`,vm 上下文里跑 guest 与连接器),会话要发的消息原样发回。
|
|
4
|
+
* 子进程不读环境变量、不发网络请求;父进程收到结果、超时或取消时杀掉它,父进程断开时自行退出。
|
|
5
|
+
* 构建时打成 `dist/testing/child.cjs`(Node 可直接执行)。
|
|
6
|
+
*/
|
|
7
|
+
import { createGuestSession } from '../runtime/session';
|
|
8
|
+
|
|
9
|
+
const session = createGuestSession((json) => {
|
|
10
|
+
process.send?.(json);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
process.on('message', (raw: unknown) => {
|
|
14
|
+
if (typeof raw === 'string') session.handle(raw);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
process.on('disconnect', () => {
|
|
18
|
+
process.exit(0);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
process.send?.(JSON.stringify({ type: 'ready' }));
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 合规检查:拿一次本地运行(用例回放或预览)的结果对照连接器描述里的表,找出平台写入时会出问题、或违反连接器约定的地方。
|
|
3
|
+
* 纯函数,不依赖 Node 模块(meta 组装预览视图时调用,UI 也可能用)。
|
|
4
|
+
*
|
|
5
|
+
* - `UNDECLARED_TABLE`:记录 / 删除标记落在给定的表之外(拉取给该数据流的表、推送给该推送的表);
|
|
6
|
+
* - `UNDECLARED_FIELDS`:不可扩展表的记录带了未声明的字段(线上运行会丢弃这些字段),每张表一条,列出全部字段名;
|
|
7
|
+
* 可扩展表的额外字段可以成为扩展列,不算问题;
|
|
8
|
+
* - `RECORD_SCHEMA_MISMATCH`:值与声明的列类型不符(规则与 connector 写入时的值换算相同)、缺主键列;
|
|
9
|
+
* - `PRIMARY_KEY_DUPLICATE`:同一次运行里两条记录主键相同;
|
|
10
|
+
* - `CHECKPOINT_NOT_MONOTONIC`:断点 `state` 的顶层字段(有限数字或 ISO-8601 时间字符串)比上一个断点小;
|
|
11
|
+
* - `FULL_READ_NOT_REPEATABLE`:全量模式下两遍读到的记录集合不同(给了 `rerun` 时才比)。
|
|
12
|
+
*
|
|
13
|
+
* 同一类问题按「表 + 列 + 原因」合并成一条,`detail.count` 为条数、`detail.sample` 为第一条的值。
|
|
14
|
+
* `FIXTURE_EXCHANGE_*` 与 `EXPECTATION_FAILED` 由用例回放(`replay.ts`)给出。
|
|
15
|
+
*/
|
|
16
|
+
import { parseIsoInstant } from './schema';
|
|
17
|
+
import type { LocalRunResult } from './types';
|
|
18
|
+
|
|
19
|
+
/** 合规检查码(与 shared 的 `ComplianceCode` 同值)。 */
|
|
20
|
+
export type ComplianceCode =
|
|
21
|
+
| 'PRIMARY_KEY_DUPLICATE'
|
|
22
|
+
| 'RECORD_SCHEMA_MISMATCH'
|
|
23
|
+
| 'UNDECLARED_TABLE'
|
|
24
|
+
| 'UNDECLARED_FIELDS'
|
|
25
|
+
| 'CHECKPOINT_NOT_MONOTONIC'
|
|
26
|
+
| 'FULL_READ_NOT_REPEATABLE'
|
|
27
|
+
| 'FIXTURE_EXCHANGE_UNUSED'
|
|
28
|
+
| 'FIXTURE_EXCHANGE_MISMATCH'
|
|
29
|
+
| 'EXPECTATION_FAILED';
|
|
30
|
+
|
|
31
|
+
export const COMPLIANCE_CODES: readonly ComplianceCode[] = Object.freeze([
|
|
32
|
+
'PRIMARY_KEY_DUPLICATE',
|
|
33
|
+
'RECORD_SCHEMA_MISMATCH',
|
|
34
|
+
'UNDECLARED_TABLE',
|
|
35
|
+
'UNDECLARED_FIELDS',
|
|
36
|
+
'CHECKPOINT_NOT_MONOTONIC',
|
|
37
|
+
'FULL_READ_NOT_REPEATABLE',
|
|
38
|
+
'FIXTURE_EXCHANGE_UNUSED',
|
|
39
|
+
'FIXTURE_EXCHANGE_MISMATCH',
|
|
40
|
+
'EXPECTATION_FAILED',
|
|
41
|
+
]);
|
|
42
|
+
|
|
43
|
+
export interface ComplianceFinding { code: ComplianceCode; message: string; table?: string; detail?: unknown }
|
|
44
|
+
|
|
45
|
+
/** 检查用的表结构:`TableDescriptor` 的子集(可以把连接的扩展列并进 `columns` 再传进来)。 */
|
|
46
|
+
export interface ComplianceTable {
|
|
47
|
+
columns: Record<string, { type: string; precision?: number; scale?: number }>;
|
|
48
|
+
primaryKey: string[];
|
|
49
|
+
extensible: boolean;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface ComplianceInput {
|
|
53
|
+
/** 用例范围内的表(拉取取该数据流的表、推送取该推送的表);为 null 时跳过依赖表结构的检查。 */
|
|
54
|
+
tables: Record<string, ComplianceTable> | null;
|
|
55
|
+
/** 消息里的范围说明,如 `stream "orders"`。 */
|
|
56
|
+
scope?: string;
|
|
57
|
+
mode?: 'full' | 'incremental';
|
|
58
|
+
result: LocalRunResult;
|
|
59
|
+
rerun?: LocalRunResult;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
type DeclaredColumn = ComplianceTable['columns'][string];
|
|
63
|
+
type DeclaredTable = ComplianceTable;
|
|
64
|
+
|
|
65
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
66
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** 未声明字段在消息里最多列出的个数(与线上丢弃告警一致)。 */
|
|
70
|
+
const UNDECLARED_FIELDS_LISTED = 20;
|
|
71
|
+
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
// 值(与 connector `storage/writer.ts` 的 `coerceValue` 同一套规则,只判断不换算)
|
|
74
|
+
|
|
75
|
+
const INSTANT_PATTERN = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?(Z|[+-]\d{2}(?::?\d{2})?)$/i;
|
|
76
|
+
const DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/;
|
|
77
|
+
const INTEGER_PATTERN = /^-?\d+$/;
|
|
78
|
+
const DECIMAL_PATTERN = /^-?(\d+)(?:\.(\d+))?$/;
|
|
79
|
+
|
|
80
|
+
function validCalendarDate(year: number, month: number, day: number): boolean {
|
|
81
|
+
return month >= 1 && month <= 12 && day >= 1 && day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** 值不符合列类型时返回原因,符合(含空值)时返回 null。 */
|
|
85
|
+
export function columnValueProblem(type: DeclaredColumn, value: unknown): string | null {
|
|
86
|
+
if (value === null || value === undefined) return null;
|
|
87
|
+
switch (type.type) {
|
|
88
|
+
case 'string':
|
|
89
|
+
return typeof value === 'string' ? null : 'is not a valid string';
|
|
90
|
+
case 'integer':
|
|
91
|
+
if (typeof value === 'number' && Number.isSafeInteger(value)) return null;
|
|
92
|
+
if (typeof value === 'bigint') return null;
|
|
93
|
+
if (typeof value === 'string' && INTEGER_PATTERN.test(value)) return null;
|
|
94
|
+
return 'is not a valid integer';
|
|
95
|
+
case 'decimal': {
|
|
96
|
+
let text: string;
|
|
97
|
+
if (typeof value === 'number' && Number.isFinite(value)) text = String(value);
|
|
98
|
+
else if (typeof value === 'string') text = value;
|
|
99
|
+
else return 'is not a valid decimal';
|
|
100
|
+
const match = DECIMAL_PATTERN.exec(text);
|
|
101
|
+
if (!match) return 'is not a valid decimal';
|
|
102
|
+
if (type.precision !== undefined && type.scale !== undefined) {
|
|
103
|
+
const integerDigits = (match[1] ?? '').replace(/^0+(?=\d)/, '');
|
|
104
|
+
if (integerDigits !== '0' && integerDigits.length > type.precision - type.scale) return `exceeds decimal precision ${type.precision},${type.scale}`;
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
case 'boolean':
|
|
109
|
+
return typeof value === 'boolean' ? null : 'is not a valid boolean';
|
|
110
|
+
case 'date': {
|
|
111
|
+
if (typeof value !== 'string') return 'is not a valid date';
|
|
112
|
+
const match = DATE_PATTERN.exec(value);
|
|
113
|
+
return match && validCalendarDate(Number(match[1]), Number(match[2]), Number(match[3])) ? null : 'is not a valid date';
|
|
114
|
+
}
|
|
115
|
+
case 'instant': {
|
|
116
|
+
if (typeof value !== 'string') return 'is not a valid instant';
|
|
117
|
+
const match = INSTANT_PATTERN.exec(value);
|
|
118
|
+
if (!match) return 'is not an ISO 8601 timestamp with a time zone offset';
|
|
119
|
+
const [, year, month, day, hour, minute, second] = match;
|
|
120
|
+
const valid = validCalendarDate(Number(year), Number(month), Number(day)) && Number(hour) <= 23 && Number(minute) <= 59 && Number(second ?? '0') <= 59;
|
|
121
|
+
return valid && parseIsoInstant(value) !== null ? null : 'is not a valid point in time';
|
|
122
|
+
}
|
|
123
|
+
case 'json':
|
|
124
|
+
try {
|
|
125
|
+
return JSON.stringify(value) === undefined ? 'is not valid JSON' : null;
|
|
126
|
+
} catch {
|
|
127
|
+
return 'is not valid JSON';
|
|
128
|
+
}
|
|
129
|
+
default:
|
|
130
|
+
return `has an unsupported column type ${type.type}`;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
// 合并同类问题
|
|
136
|
+
|
|
137
|
+
class FindingSet {
|
|
138
|
+
private readonly items = new Map<string, { finding: ComplianceFinding; count: number }>();
|
|
139
|
+
private readonly ordered: ComplianceFinding[] = [];
|
|
140
|
+
|
|
141
|
+
/** `key` 相同的问题合并:只留第一条的 message 与 sample,`detail.count` 累加。 */
|
|
142
|
+
add(key: string, build: () => { code: ComplianceCode; message: string; table?: string; sample?: unknown; extra?: Record<string, unknown> }): void {
|
|
143
|
+
const existing = this.items.get(key);
|
|
144
|
+
if (existing) {
|
|
145
|
+
existing.count += 1;
|
|
146
|
+
(existing.finding.detail as Record<string, unknown>).count = existing.count;
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
const { code, message, table, sample, extra } = build();
|
|
150
|
+
const finding: ComplianceFinding = { code, message, ...(table !== undefined ? { table } : {}), detail: { count: 1, ...(sample !== undefined ? { sample } : {}), ...extra } };
|
|
151
|
+
this.items.set(key, { finding, count: 1 });
|
|
152
|
+
this.ordered.push(finding);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
push(finding: ComplianceFinding): void {
|
|
156
|
+
this.ordered.push(finding);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
prepend(items: readonly ComplianceFinding[]): void {
|
|
160
|
+
this.ordered.unshift(...items);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
list(): ComplianceFinding[] {
|
|
164
|
+
return this.ordered;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
// 记录与删除标记
|
|
170
|
+
|
|
171
|
+
function checkRows(input: ComplianceInput, tables: Record<string, DeclaredTable>, findings: FindingSet): void {
|
|
172
|
+
const keysSeen = new Map<string, Set<string>>();
|
|
173
|
+
const undeclaredFields = new Map<string, Set<string>>();
|
|
174
|
+
const undeclared = (table: string, sample: unknown): void => {
|
|
175
|
+
findings.add(`undeclared\0${table}`, () => ({
|
|
176
|
+
code: 'UNDECLARED_TABLE', table, sample,
|
|
177
|
+
message: `table "${table}" is not declared${input.scope !== undefined ? ` by ${input.scope}` : ''}`,
|
|
178
|
+
}));
|
|
179
|
+
};
|
|
180
|
+
const mismatch = (table: string, column: string, reason: string, sample: unknown): void => {
|
|
181
|
+
findings.add(`schema\0${table}\0${column}\0${reason}`, () => ({
|
|
182
|
+
code: 'RECORD_SCHEMA_MISMATCH', table, sample, extra: { column },
|
|
183
|
+
message: `column "${column}" of table "${table}" ${reason}`,
|
|
184
|
+
}));
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
for (const record of input.result.records) {
|
|
188
|
+
const declared = Object.hasOwn(tables, record.table) ? tables[record.table] : undefined;
|
|
189
|
+
if (!declared) {
|
|
190
|
+
undeclared(record.table, record.row);
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
if (!declared.extensible) {
|
|
194
|
+
for (const column of Object.keys(record.row)) {
|
|
195
|
+
if (Object.hasOwn(declared.columns, column)) continue;
|
|
196
|
+
const fields = undeclaredFields.get(record.table) ?? new Set<string>();
|
|
197
|
+
fields.add(column);
|
|
198
|
+
undeclaredFields.set(record.table, fields);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
for (const [column, type] of Object.entries(declared.columns)) {
|
|
202
|
+
const problem = columnValueProblem(type, record.row[column]);
|
|
203
|
+
if (problem) mismatch(record.table, column, problem, record.row[column]);
|
|
204
|
+
}
|
|
205
|
+
const missing = declared.primaryKey.filter((column) => record.row[column] === null || record.row[column] === undefined);
|
|
206
|
+
for (const column of missing) mismatch(record.table, column, 'is a primary key column and is missing', record.row);
|
|
207
|
+
if (declared.primaryKey.length === 0 || missing.length > 0) continue;
|
|
208
|
+
const key = JSON.stringify(declared.primaryKey.map((column) => record.row[column]));
|
|
209
|
+
const seen = keysSeen.get(record.table) ?? new Set<string>();
|
|
210
|
+
if (seen.has(key)) {
|
|
211
|
+
const sample = Object.fromEntries(declared.primaryKey.map((column) => [column, record.row[column]]));
|
|
212
|
+
findings.add(`pk\0${record.table}`, () => ({
|
|
213
|
+
code: 'PRIMARY_KEY_DUPLICATE', table: record.table, sample,
|
|
214
|
+
message: `table "${record.table}" received more than one record with the same primary key (${declared.primaryKey.join(', ')})`,
|
|
215
|
+
}));
|
|
216
|
+
}
|
|
217
|
+
seen.add(key);
|
|
218
|
+
keysSeen.set(record.table, seen);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// 未声明字段放在最前:线上运行会静默丢弃它们,是最需要作者先看到的问题。
|
|
222
|
+
const fieldFindings: ComplianceFinding[] = [];
|
|
223
|
+
for (const [table, set] of undeclaredFields) {
|
|
224
|
+
const fields = [...set].sort();
|
|
225
|
+
const listed = fields.slice(0, UNDECLARED_FIELDS_LISTED).join(', ');
|
|
226
|
+
fieldFindings.push({
|
|
227
|
+
code: 'UNDECLARED_FIELDS', table,
|
|
228
|
+
message: `table "${table}" received ${fields.length} undeclared field(s): ${listed}${fields.length > UNDECLARED_FIELDS_LISTED ? ', …' : ''}; table "${table}" is not extensible, so these fields would be discarded`,
|
|
229
|
+
detail: { fields },
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
findings.prepend(fieldFindings);
|
|
233
|
+
|
|
234
|
+
for (const removal of input.result.removals) {
|
|
235
|
+
const declared = Object.hasOwn(tables, removal.table) ? tables[removal.table] : undefined;
|
|
236
|
+
if (!declared) {
|
|
237
|
+
undeclared(removal.table, removal.key);
|
|
238
|
+
continue;
|
|
239
|
+
}
|
|
240
|
+
for (const column of Object.keys(removal.key)) {
|
|
241
|
+
if (!declared.primaryKey.includes(column)) mismatch(removal.table, column, 'is not part of the primary key (removal)', removal.key);
|
|
242
|
+
}
|
|
243
|
+
for (const column of declared.primaryKey) {
|
|
244
|
+
const value = removal.key[column];
|
|
245
|
+
if (value === null || value === undefined) {
|
|
246
|
+
mismatch(removal.table, column, 'is a primary key column and is missing (removal)', removal.key);
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
const type = declared.columns[column];
|
|
250
|
+
const problem = type ? columnValueProblem(type, value) : 'is not declared';
|
|
251
|
+
if (problem) mismatch(removal.table, column, `${problem} (removal)`, value);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// ---------------------------------------------------------------------------
|
|
257
|
+
// 断点
|
|
258
|
+
|
|
259
|
+
/** 可比较的断点字段值:有限数字按数值,ISO-8601 时间字符串按时刻;其他类型不判。 */
|
|
260
|
+
function comparable(value: unknown): { kind: 'number' | 'time'; at: number } | null {
|
|
261
|
+
if (typeof value === 'number' && Number.isFinite(value)) return { kind: 'number', at: value };
|
|
262
|
+
if (typeof value === 'string') {
|
|
263
|
+
const at = parseIsoInstant(value);
|
|
264
|
+
if (at !== null) return { kind: 'time', at };
|
|
265
|
+
}
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function checkCheckpoints(checkpoints: LocalRunResult['checkpoints'], findings: FindingSet): void {
|
|
270
|
+
for (let index = 1; index < checkpoints.length; index += 1) {
|
|
271
|
+
const previous = checkpoints[index - 1];
|
|
272
|
+
const next = checkpoints[index];
|
|
273
|
+
if (!previous || !next || !isRecord(previous.state) || !isRecord(next.state)) continue;
|
|
274
|
+
for (const [field, before] of Object.entries(previous.state)) {
|
|
275
|
+
if (!Object.hasOwn(next.state, field)) continue;
|
|
276
|
+
const after = next.state[field];
|
|
277
|
+
const a = comparable(before);
|
|
278
|
+
const b = comparable(after);
|
|
279
|
+
if (!a || !b || a.kind !== b.kind || b.at >= a.at) continue;
|
|
280
|
+
findings.push({
|
|
281
|
+
code: 'CHECKPOINT_NOT_MONOTONIC',
|
|
282
|
+
message: `checkpoint field "${field}" went backwards from ${JSON.stringify(before)} to ${JSON.stringify(after)} (checkpoint ${next.seq})`,
|
|
283
|
+
detail: { field, previous: before, next: after, seq: next.seq },
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
// ---------------------------------------------------------------------------
|
|
290
|
+
// 全量可重复
|
|
291
|
+
|
|
292
|
+
function canonical(value: unknown): string {
|
|
293
|
+
return JSON.stringify(value, (_key, item: unknown) => (isRecord(item) ? Object.fromEntries(Object.keys(item).sort().map((key) => [key, item[key]])) : item));
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** 记录按「表 + 主键 + 整行」排序后的规范文本(主键缺失时只按整行)。 */
|
|
297
|
+
function recordSet(records: LocalRunResult['records'], tables: Record<string, DeclaredTable> | null): string[] {
|
|
298
|
+
return records
|
|
299
|
+
.map((record) => {
|
|
300
|
+
const primaryKey = tables && Object.hasOwn(tables, record.table) ? tables[record.table]?.primaryKey ?? [] : [];
|
|
301
|
+
return `${record.table}\0${canonical(primaryKey.map((column) => record.row[column]))}\0${canonical(record.row)}`;
|
|
302
|
+
})
|
|
303
|
+
.sort();
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function checkRepeatable(input: ComplianceInput, tables: Record<string, DeclaredTable> | null, findings: FindingSet): void {
|
|
307
|
+
const rerun = input.rerun;
|
|
308
|
+
if (input.mode !== 'full' || !rerun) return;
|
|
309
|
+
if (rerun.error) {
|
|
310
|
+
findings.push({
|
|
311
|
+
code: 'FULL_READ_NOT_REPEATABLE',
|
|
312
|
+
message: `the second full read failed (${rerun.error.code}: ${rerun.error.message})`,
|
|
313
|
+
detail: { error: rerun.error },
|
|
314
|
+
});
|
|
315
|
+
return;
|
|
316
|
+
}
|
|
317
|
+
const first = recordSet(input.result.records, tables);
|
|
318
|
+
const second = recordSet(rerun.records, tables);
|
|
319
|
+
if (first.length === second.length && first.every((item, index) => item === second[index])) return;
|
|
320
|
+
const firstSet = new Set(first);
|
|
321
|
+
const secondSet = new Set(second);
|
|
322
|
+
findings.push({
|
|
323
|
+
code: 'FULL_READ_NOT_REPEATABLE',
|
|
324
|
+
message: `two full reads returned different records (${first.length} vs ${second.length} record(s))`,
|
|
325
|
+
detail: {
|
|
326
|
+
firstCount: first.length,
|
|
327
|
+
secondCount: second.length,
|
|
328
|
+
onlyInFirst: first.filter((item) => !secondSet.has(item)).length,
|
|
329
|
+
onlyInSecond: second.filter((item) => !firstSet.has(item)).length,
|
|
330
|
+
},
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// ---------------------------------------------------------------------------
|
|
335
|
+
|
|
336
|
+
export function checkCompliance(input: ComplianceInput): ComplianceFinding[] {
|
|
337
|
+
const findings = new FindingSet();
|
|
338
|
+
const tables = input.tables;
|
|
339
|
+
if (tables) checkRows(input, tables, findings);
|
|
340
|
+
checkCheckpoints(input.result.checkpoints, findings);
|
|
341
|
+
checkRepeatable(input, tables, findings);
|
|
342
|
+
return findings.list();
|
|
343
|
+
}
|