@deepexi/datasense-connector 3.0.1 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/README.md +158 -139
  2. package/package.json +19 -28
  3. package/scripts/build.ts +188 -0
  4. package/src/define.ts +267 -0
  5. package/src/descriptor.ts +116 -0
  6. package/src/errors.ts +105 -0
  7. package/src/index.ts +108 -0
  8. package/src/internal/location.ts +23 -0
  9. package/src/runtime/guest/bridge.ts +238 -0
  10. package/src/runtime/guest/cache.ts +43 -0
  11. package/src/runtime/guest/check.ts +17 -0
  12. package/src/runtime/guest/crypto.ts +166 -0
  13. package/src/runtime/guest/ctx.ts +157 -0
  14. package/src/runtime/guest/db.ts +59 -0
  15. package/src/runtime/guest/failure.ts +69 -0
  16. package/src/runtime/guest/http.ts +194 -0
  17. package/src/runtime/guest/index.ts +102 -0
  18. package/{runtime/guest/limits.d.ts → src/runtime/guest/limits.ts} +12 -12
  19. package/src/runtime/guest/loader.ts +210 -0
  20. package/src/runtime/guest/log.ts +47 -0
  21. package/src/runtime/guest/map.ts +45 -0
  22. package/src/runtime/guest/operation.ts +193 -0
  23. package/src/runtime/guest/paginate.ts +169 -0
  24. package/src/runtime/guest/pull.ts +177 -0
  25. package/src/runtime/guest/push.ts +92 -0
  26. package/src/runtime/guest/signal.ts +14 -0
  27. package/src/runtime/guest/suggest.ts +51 -0
  28. package/src/runtime/guest/time.ts +137 -0
  29. package/src/runtime/guest-runtime.ts +75 -0
  30. package/src/runtime/index.ts +53 -0
  31. package/src/runtime/protocol.ts +557 -0
  32. package/src/runtime/session.ts +216 -0
  33. package/src/testing/child.ts +21 -0
  34. package/src/testing/compliance.ts +343 -0
  35. package/src/testing/engine.ts +445 -0
  36. package/src/testing/fixtures.ts +591 -0
  37. package/src/testing/gateway.ts +482 -0
  38. package/src/testing/import-guard.ts +221 -0
  39. package/src/testing/index.ts +43 -0
  40. package/src/testing/local-db.ts +226 -0
  41. package/src/testing/mock-external-system.ts +249 -0
  42. package/src/testing/pinned-http.ts +144 -0
  43. package/src/testing/recorder.ts +140 -0
  44. package/src/testing/redaction.ts +94 -0
  45. package/src/testing/replay.ts +437 -0
  46. package/src/testing/run-tests.ts +115 -0
  47. package/src/testing/schema.ts +70 -0
  48. package/src/testing/source-loader.ts +374 -0
  49. package/src/testing/types.ts +78 -0
  50. package/src/types.ts +329 -0
  51. package/src/validate.ts +267 -0
  52. package/test/build.test.ts +159 -0
  53. package/test/define.test-d.ts +182 -0
  54. package/test/fixtures/acceptance.ts +156 -0
  55. package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
  56. package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
  57. package/test/fixtures/connectors/http-pull/package.json +5 -0
  58. package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
  59. package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
  60. package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
  61. package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
  62. package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
  63. package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
  64. package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
  65. package/test/fixtures/connectors/pg-pull/package.json +5 -0
  66. package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
  67. package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
  68. package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
  69. package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
  70. package/test/fixtures/descriptor.ts +54 -0
  71. package/test/fixtures/kingdee-example.ts +82 -0
  72. package/test/fixtures/record-acceptance.ts +32 -0
  73. package/test/runtime/ctx.test.ts +434 -0
  74. package/test/runtime/helpers.ts +157 -0
  75. package/test/runtime/jobs.test.ts +444 -0
  76. package/test/runtime/operation.test.ts +175 -0
  77. package/test/runtime/session.test.ts +229 -0
  78. package/test/testing/acceptance-fixtures.test.ts +69 -0
  79. package/test/testing/compliance.test.ts +247 -0
  80. package/test/testing/engine.test.ts +283 -0
  81. package/test/testing/fixtures.test.ts +221 -0
  82. package/test/testing/gateway.test.ts +337 -0
  83. package/test/testing/import-guard.test.ts +45 -0
  84. package/test/testing/mock-external-system.test.ts +144 -0
  85. package/test/testing/operation-fixtures.test.ts +160 -0
  86. package/test/testing/recorder.test.ts +163 -0
  87. package/test/testing/redaction.test.ts +54 -0
  88. package/test/testing/replay.test.ts +207 -0
  89. package/test/testing/run-tests.test.ts +137 -0
  90. package/test/testing/source-loader.test.ts +220 -0
  91. package/test/validate-config.test.ts +104 -0
  92. package/tsconfig.build.json +23 -0
  93. package/tsconfig.json +19 -0
  94. package/cjs/index.cjs +0 -1015
  95. package/cjs/runtime.cjs +0 -752
  96. package/cjs/testing.cjs +0 -3591
  97. package/errors.d.ts +0 -17
  98. package/errors.js +0 -53
  99. package/guest.cjs +0 -2659
  100. package/index.d.ts +0 -13
  101. package/index.js +0 -7
  102. package/internal/location.d.ts +0 -6
  103. package/internal/location.js +0 -8
  104. package/presentation.d.ts +0 -38
  105. package/presentation.js +0 -268
  106. package/runtime/guest/bridge.d.ts +0 -53
  107. package/runtime/guest/bridge.js +0 -187
  108. package/runtime/guest/cache.d.ts +0 -3
  109. package/runtime/guest/cache.js +0 -42
  110. package/runtime/guest/crypto.d.ts +0 -13
  111. package/runtime/guest/crypto.js +0 -177
  112. package/runtime/guest/ctx.d.ts +0 -30
  113. package/runtime/guest/ctx.js +0 -124
  114. package/runtime/guest/db.d.ts +0 -6
  115. package/runtime/guest/db.js +0 -59
  116. package/runtime/guest/discover.d.ts +0 -29
  117. package/runtime/guest/discover.js +0 -51
  118. package/runtime/guest/failure.d.ts +0 -24
  119. package/runtime/guest/failure.js +0 -63
  120. package/runtime/guest/http.d.ts +0 -4
  121. package/runtime/guest/http.js +0 -204
  122. package/runtime/guest/index.d.ts +0 -12
  123. package/runtime/guest/index.js +0 -89
  124. package/runtime/guest/limits.js +0 -12
  125. package/runtime/guest/loader.d.ts +0 -15
  126. package/runtime/guest/loader.js +0 -42
  127. package/runtime/guest/log.d.ts +0 -10
  128. package/runtime/guest/log.js +0 -36
  129. package/runtime/guest/map.d.ts +0 -4
  130. package/runtime/guest/map.js +0 -45
  131. package/runtime/guest/paginate.d.ts +0 -6
  132. package/runtime/guest/paginate.js +0 -180
  133. package/runtime/guest/pull.d.ts +0 -12
  134. package/runtime/guest/pull.js +0 -150
  135. package/runtime/guest/push.d.ts +0 -18
  136. package/runtime/guest/push.js +0 -80
  137. package/runtime/guest/signal.d.ts +0 -3
  138. package/runtime/guest/signal.js +0 -12
  139. package/runtime/guest/time.d.ts +0 -5
  140. package/runtime/guest/time.js +0 -127
  141. package/runtime/guest-runtime.d.ts +0 -7
  142. package/runtime/guest-runtime.js +0 -51
  143. package/runtime/index.d.ts +0 -18
  144. package/runtime/index.js +0 -3
  145. package/runtime/protocol.d.ts +0 -276
  146. package/runtime/protocol.js +0 -409
  147. package/runtime/session.d.ts +0 -12
  148. package/runtime/session.js +0 -191
  149. package/t.d.ts +0 -18
  150. package/t.js +0 -26
  151. package/testing/child.cjs +0 -350
  152. package/testing/child.d.ts +0 -1
  153. package/testing/child.js +0 -12
  154. package/testing/compliance.d.ts +0 -41
  155. package/testing/compliance.js +0 -291
  156. package/testing/engine.d.ts +0 -66
  157. package/testing/engine.js +0 -339
  158. package/testing/fixtures.d.ts +0 -145
  159. package/testing/fixtures.js +0 -407
  160. package/testing/gateway.d.ts +0 -65
  161. package/testing/gateway.js +0 -402
  162. package/testing/import-guard.d.ts +0 -57
  163. package/testing/import-guard.js +0 -187
  164. package/testing/index.d.ts +0 -33
  165. package/testing/index.js +0 -10
  166. package/testing/local-db.d.ts +0 -27
  167. package/testing/local-db.js +0 -181
  168. package/testing/pinned-http.d.ts +0 -14
  169. package/testing/pinned-http.js +0 -125
  170. package/testing/recorder.d.ts +0 -15
  171. package/testing/recorder.js +0 -116
  172. package/testing/redaction.d.ts +0 -28
  173. package/testing/redaction.js +0 -67
  174. package/testing/replay.d.ts +0 -50
  175. package/testing/replay.js +0 -313
  176. package/testing/schema.d.ts +0 -13
  177. package/testing/schema.js +0 -65
  178. package/testing/source-loader.d.ts +0 -19
  179. package/testing/source-loader.js +0 -238
  180. package/testing/types.d.ts +0 -79
  181. package/testing/types.js +0 -18
  182. package/types.d.ts +0 -302
  183. package/types.js +0 -13
  184. package/validate.d.ts +0 -78
  185. package/validate.js +0 -597
@@ -1,33 +0,0 @@
1
- /**
2
- * `@deepexi/datasense-connector/testing`:在作者自己的机器上运行连接器(vitest 与 CLI 共用)。
3
- *
4
- * - `runStream` / `runDiscover` / `runCheck` / `runPush`:在当前进程里执行已导入的定义对象(vitest 用);
5
- * - `loadConnectorSource` + `createVmExecutor`:把源码目录转译成 CommonJS 打包串,在空环境变量的子进程里用与平台 runner
6
- * 相同的 guest 会话执行(CLI 用);
7
- * - `inspectDefinition`:只加载连接器并取定义摘要(核对 `package.json` 的名字与版本),不执行作业;
8
- * - `createLocalGateway`:本地模拟网关,应答 guest 的出站 HTTP、数据库、加密、缓存与凭证调用(主机求交、限流、句柄替换与平台一致);
9
- * - 夹具:`createRecorder` 挂在网关的 `onExchange` 上录制、`serializeFixture` / `parseFixtures` / `readFixtureDir` 读写
10
- * `fixtures/<用例名>/`,`createReplayHost` / `runFixture` / `runFixtures` 不联网回放并做合规检查;
11
- * - 纯函数 `inferRecordSchema`、`checkCompliance`(不依赖 Node 模块)。
12
- *
13
- * 连接器源码不要导入本入口:平台打包时只认 `@deepexi/datasense-connector` 本身,子路径一律拒绝。
14
- */
15
- export type { GuestJob, HostMethod, LocalHost, LocalRunErrorCode, LocalRunResult, GuestExecutor, SourceLoadErrorCode } from './types.js';
16
- export { LocalRunError, SourceLoadError } from './types.js';
17
- export type { LocalExchange, LocalGatewayOptions } from './gateway.js';
18
- export { createLocalGateway } from './gateway.js';
19
- export type { DefinitionInspection, StreamRunInput, VmExecutorOptions } from './engine.js';
20
- export { createInProcessExecutor, createVmExecutor, inspectDefinition, runCheck, runDiscover, runPush, runStream } from './engine.js';
21
- export type { LoadedSource } from './source-loader.js';
22
- export { loadConnectorSource } from './source-loader.js';
23
- export { CREDENTIAL_HEADER_NAMES, redactHeaders, redactQuery, redactSecretValues, redactUrl, secretRedactionTexts } from './redaction.js';
24
- export type { ConnectorFixtureFile, FixtureCase, FixtureExchange, FixtureExpectation, FixtureRequest, LoadedFixture, } from './fixtures.js';
25
- export { FixtureFormatError, parseFixtures, readFixtureDir, serializeFixture } from './fixtures.js';
26
- export type { FixtureRecorder } from './recorder.js';
27
- export { createRecorder } from './recorder.js';
28
- export type { FixtureCaseResult, FixtureDeclaration, FixtureMismatch, ReplayHost, RunFixtureOptions } from './replay.js';
29
- export { createReplayHost, FixtureReplayError, runFixture, runFixtures } from './replay.js';
30
- export type { InferredColumnType } from './schema.js';
31
- export { inferRecordSchema } from './schema.js';
32
- export type { ComplianceCode, ComplianceFinding, ComplianceInput } from './compliance.js';
33
- export { checkCompliance, COMPLIANCE_CODES } from './compliance.js';
package/testing/index.js DELETED
@@ -1,10 +0,0 @@
1
- export { LocalRunError, SourceLoadError } from './types.js';
2
- export { createLocalGateway } from './gateway.js';
3
- export { createInProcessExecutor, createVmExecutor, inspectDefinition, runCheck, runDiscover, runPush, runStream } from './engine.js';
4
- export { loadConnectorSource } from './source-loader.js';
5
- export { CREDENTIAL_HEADER_NAMES, redactHeaders, redactQuery, redactSecretValues, redactUrl, secretRedactionTexts } from './redaction.js';
6
- export { FixtureFormatError, parseFixtures, readFixtureDir, serializeFixture } from './fixtures.js';
7
- export { createRecorder } from './recorder.js';
8
- export { createReplayHost, FixtureReplayError, runFixture, runFixtures } from './replay.js';
9
- export { inferRecordSchema } from './schema.js';
10
- export { checkCompliance, COMPLIANCE_CODES } from './compliance.js';
@@ -1,27 +0,0 @@
1
- import type { DbNextResponse, DbQueryResponse } from '../runtime/protocol.js';
2
- export interface LocalDbTarget {
3
- host: string;
4
- port: number;
5
- database: string;
6
- user: string;
7
- password: string;
8
- ssl?: boolean;
9
- }
10
- /** 带 `code` 的宿主调用失败(与平台数据库代理同码)。 */
11
- export declare class LocalDbError extends Error {
12
- readonly code: string;
13
- constructor(code: string, message: string);
14
- }
15
- export interface LocalDb {
16
- query(request: {
17
- sql: string;
18
- params: unknown[];
19
- }): Promise<DbQueryResponse>;
20
- next(cursorId: string): Promise<DbNextResponse>;
21
- close(cursorId: string): Promise<void>;
22
- end(): Promise<void>;
23
- }
24
- export declare function createLocalDb(target: LocalDbTarget, options: {
25
- driver?: unknown;
26
- secrets: readonly string[];
27
- }): LocalDb;
@@ -1,181 +0,0 @@
1
- import { randomUUID } from 'node:crypto';
2
- import { redactSecretValues } from './redaction.js';
3
- import { LocalRunError } from './types.js';
4
- const BATCH_ROWS = 1000;
5
- const STATEMENT_TIMEOUT_MS = 300_000;
6
- const APPLICATION_NAME = 'datasense-connector-local';
7
- const RAW_TEXT_OIDS = [1082, 1083, 1114, 1266];
8
- function isRecord(value) {
9
- return typeof value === 'object' && value !== null && !Array.isArray(value);
10
- }
11
- function factoryOf(driver) {
12
- if (typeof driver === 'function')
13
- return driver;
14
- if (isRecord(driver) && typeof driver.default === 'function')
15
- return driver.default;
16
- return null;
17
- }
18
- async function loadDriver(injected) {
19
- if (injected !== undefined && injected !== null) {
20
- const factory = factoryOf(injected);
21
- if (!factory)
22
- throw new LocalRunError('DB_DRIVER_MISSING', 'the injected postgres driver is not the postgres.js module');
23
- return factory;
24
- }
25
- const specifier = 'postgres';
26
- let loaded;
27
- try {
28
- loaded = await import(specifier);
29
- }
30
- catch {
31
- throw new LocalRunError('DB_DRIVER_MISSING', 'ctx.db needs the postgres driver: install it next to the connector (npm install --save-dev postgres)');
32
- }
33
- const factory = factoryOf(loaded);
34
- if (!factory)
35
- throw new LocalRunError('DB_DRIVER_MISSING', 'the installed postgres package is not the postgres.js driver');
36
- return factory;
37
- }
38
- function toParam(value) {
39
- if (value === undefined)
40
- return null;
41
- if (isRecord(value))
42
- return JSON.stringify(value);
43
- return value;
44
- }
45
- function toJsonValue(value) {
46
- if (value instanceof Date)
47
- return Number.isNaN(value.getTime()) ? null : value.toISOString();
48
- if (typeof value === 'bigint')
49
- return value.toString();
50
- if (value instanceof Uint8Array)
51
- return Buffer.from(value).toString('base64');
52
- if (Array.isArray(value))
53
- return value.map(toJsonValue);
54
- return value;
55
- }
56
- function toRows(rows) {
57
- return rows.map((row) => {
58
- const out = {};
59
- for (const [name, value] of Object.entries(row))
60
- out[name] = toJsonValue(value);
61
- return out;
62
- });
63
- }
64
- export class LocalDbError extends Error {
65
- code;
66
- constructor(code, message) {
67
- super(message);
68
- this.name = 'LocalDbError';
69
- this.code = code;
70
- }
71
- }
72
- export function createLocalDb(target, options) {
73
- let pool = null;
74
- const cursors = new Map();
75
- const secrets = [...options.secrets, target.password];
76
- const poolOf = async () => {
77
- if (pool)
78
- return pool;
79
- const factory = await loadDriver(options.driver);
80
- pool = factory({
81
- host: target.host,
82
- port: target.port,
83
- database: target.database,
84
- username: target.user,
85
- password: target.password,
86
- ssl: target.ssl ? 'require' : false,
87
- max: 4,
88
- idle_timeout: 60,
89
- connect_timeout: 10,
90
- prepare: false,
91
- onnotice: () => { },
92
- connection: { application_name: APPLICATION_NAME },
93
- types: {
94
- rawText: { to: 25, from: RAW_TEXT_OIDS, serialize: (value) => String(value), parse: (value) => value },
95
- },
96
- });
97
- return pool;
98
- };
99
- const failed = (error) => {
100
- if (error instanceof LocalDbError || error instanceof LocalRunError)
101
- return error;
102
- const message = redactSecretValues(error instanceof Error ? error.message : String(error), secrets);
103
- return new LocalDbError('DB_PROXY_FAILED', `The query to the source database failed: ${message}`);
104
- };
105
- const finish = async (id, cursor, ok) => {
106
- cursors.delete(id);
107
- try {
108
- if (ok)
109
- await cursor.iterator.return?.();
110
- await cursor.reserved.unsafe(ok ? 'commit' : 'rollback');
111
- }
112
- catch {
113
- await Promise.resolve(cursor.reserved.unsafe('rollback')).then(undefined, () => undefined);
114
- }
115
- finally {
116
- cursor.reserved.release();
117
- }
118
- };
119
- return {
120
- async query(request) {
121
- const sql = await poolOf();
122
- let reserved = null;
123
- try {
124
- reserved = await sql.reserve();
125
- await reserved.unsafe('begin read only');
126
- await reserved.unsafe(`set local statement_timeout = ${STATEMENT_TIMEOUT_MS}`);
127
- const iterator = reserved.unsafe(request.sql, request.params.map(toParam)).cursor(BATCH_ROWS)[Symbol.asyncIterator]();
128
- const first = await iterator.next();
129
- const rows = first.done ? [] : first.value;
130
- const fields = (rows.columns ?? []).map((column) => ({ name: column.name, dataTypeId: column.type }));
131
- const id = randomUUID();
132
- const cursor = { reserved, iterator };
133
- reserved = null;
134
- if (first.done === true || rows.length < BATCH_ROWS) {
135
- await finish(id, cursor, true);
136
- return { cursorId: null, fields, rows: toRows(rows), done: true };
137
- }
138
- cursors.set(id, cursor);
139
- return { cursorId: id, fields, rows: toRows(rows), done: false };
140
- }
141
- catch (error) {
142
- if (reserved) {
143
- const held = reserved;
144
- await Promise.resolve(held.unsafe('rollback')).then(undefined, () => undefined);
145
- held.release();
146
- }
147
- throw failed(error);
148
- }
149
- },
150
- async next(cursorId) {
151
- const cursor = cursors.get(cursorId);
152
- if (!cursor)
153
- throw new LocalDbError('DB_CURSOR_NOT_FOUND', "The cursor doesn't exist or was closed");
154
- try {
155
- const step = await cursor.iterator.next();
156
- const rows = step.done ? [] : step.value;
157
- const done = step.done === true || rows.length < BATCH_ROWS;
158
- if (done)
159
- await finish(cursorId, cursor, true);
160
- return { rows: toRows(rows), done };
161
- }
162
- catch (error) {
163
- await finish(cursorId, cursor, false);
164
- throw failed(error);
165
- }
166
- },
167
- async close(cursorId) {
168
- const cursor = cursors.get(cursorId);
169
- if (cursor)
170
- await finish(cursorId, cursor, true);
171
- },
172
- async end() {
173
- for (const [id, cursor] of [...cursors])
174
- await finish(id, cursor, false);
175
- const current = pool;
176
- pool = null;
177
- if (current)
178
- await current.end({ timeout: 5 }).catch(() => undefined);
179
- },
180
- };
181
- }
@@ -1,14 +0,0 @@
1
- import type { LookupAddress } from 'node:dns';
2
- export declare class PinnedBodyTooLargeError extends Error {
3
- constructor();
4
- }
5
- export interface PinnedRequestInput {
6
- url: URL;
7
- method: string;
8
- headers: Record<string, string>;
9
- body: Uint8Array | null;
10
- addresses: readonly LookupAddress[];
11
- signal: AbortSignal;
12
- maxBytes: number;
13
- }
14
- export declare function pinnedRequest(input: PinnedRequestInput): Promise<Response>;
@@ -1,125 +0,0 @@
1
- import { request as httpRequest } from 'node:http';
2
- import { request as httpsRequest } from 'node:https';
3
- import { brotliDecompressSync, gunzipSync, inflateRawSync, inflateSync } from 'node:zlib';
4
- export class PinnedBodyTooLargeError extends Error {
5
- constructor() {
6
- super('response body too large');
7
- this.name = 'PinnedBodyTooLargeError';
8
- }
9
- }
10
- const NULL_BODY_STATUSES = new Set([101, 103, 204, 205, 304]);
11
- function pinnedLookup(addresses) {
12
- return (_hostname, options, callback) => {
13
- const all = typeof options === 'object' && options !== null && options.all === true;
14
- const first = addresses[0];
15
- if (!first) {
16
- callback(Object.assign(new Error('no address to connect to'), { code: 'ENOTFOUND' }), []);
17
- return;
18
- }
19
- if (all)
20
- callback(null, [...addresses]);
21
- else
22
- callback(null, first.address, first.family);
23
- };
24
- }
25
- function decompress(bytes, encoding, maxBytes) {
26
- const options = { maxOutputLength: maxBytes + 1 };
27
- try {
28
- switch (encoding) {
29
- case 'gzip':
30
- case 'x-gzip':
31
- return gunzipSync(bytes, options);
32
- case 'deflate':
33
- try {
34
- return inflateSync(bytes, options);
35
- }
36
- catch (error) {
37
- if (error instanceof RangeError)
38
- throw error;
39
- return inflateRawSync(bytes, options);
40
- }
41
- case 'br':
42
- return brotliDecompressSync(bytes, options);
43
- default:
44
- return bytes;
45
- }
46
- }
47
- catch (error) {
48
- if (error instanceof RangeError || error.code === 'ERR_BUFFER_TOO_LARGE')
49
- throw new PinnedBodyTooLargeError();
50
- throw error;
51
- }
52
- }
53
- function toHeaders(response) {
54
- const headers = new Headers();
55
- const raw = response.rawHeaders;
56
- for (let index = 0; index + 1 < raw.length; index += 2) {
57
- try {
58
- headers.append(raw[index] ?? '', raw[index + 1] ?? '');
59
- }
60
- catch {
61
- }
62
- }
63
- return headers;
64
- }
65
- export function pinnedRequest(input) {
66
- const { url, method, signal, maxBytes } = input;
67
- const send = url.protocol === 'https:' ? httpsRequest : httpRequest;
68
- const body = input.body !== null && method !== 'GET' && method !== 'HEAD' ? Buffer.from(input.body) : null;
69
- const headers = { ...input.headers };
70
- if (body !== null)
71
- headers['content-length'] = String(body.byteLength);
72
- return new Promise((resolve, reject) => {
73
- if (signal.aborted) {
74
- reject(new Error('aborted'));
75
- return;
76
- }
77
- let settled = false;
78
- const fail = (error) => {
79
- if (settled)
80
- return;
81
- settled = true;
82
- signal.removeEventListener('abort', onAbort);
83
- reject(error);
84
- };
85
- const req = send(url, { method, headers, lookup: pinnedLookup(input.addresses) }, (response) => {
86
- const chunks = [];
87
- let size = 0;
88
- response.on('data', (chunk) => {
89
- size += chunk.byteLength;
90
- if (size > maxBytes) {
91
- req.destroy();
92
- fail(new PinnedBodyTooLargeError());
93
- return;
94
- }
95
- chunks.push(chunk);
96
- });
97
- response.on('error', (error) => fail(error));
98
- response.on('end', () => {
99
- if (settled)
100
- return;
101
- try {
102
- const status = response.statusCode ?? 0;
103
- const encoding = (response.headers['content-encoding'] ?? '').trim().toLowerCase();
104
- const bytes = method === 'HEAD' ? Buffer.alloc(0) : decompress(Buffer.concat(chunks), encoding, maxBytes);
105
- const responseHeaders = toHeaders(response);
106
- const body = NULL_BODY_STATUSES.has(status) || bytes.byteLength === 0 ? null : bytes;
107
- const result = new Response(body, { status, headers: responseHeaders });
108
- settled = true;
109
- signal.removeEventListener('abort', onAbort);
110
- resolve(result);
111
- }
112
- catch (error) {
113
- fail(error instanceof Error ? error : new Error(String(error)));
114
- }
115
- });
116
- });
117
- const onAbort = () => {
118
- req.destroy(new Error('aborted'));
119
- fail(new Error('aborted'));
120
- };
121
- signal.addEventListener('abort', onAbort, { once: true });
122
- req.on('error', (error) => fail(error));
123
- req.end(body ?? undefined);
124
- });
125
- }
@@ -1,15 +0,0 @@
1
- import { type FixtureExchange } from './fixtures.js';
2
- import type { LocalExchange } from './gateway.js';
3
- export interface FixtureRecorder {
4
- onExchange(exchange: LocalExchange): void;
5
- /** 至今录下的往来(按 `seq` 升序,深拷贝)。 */
6
- exchanges(): FixtureExchange[];
7
- }
8
- /**
9
- * `secrets`:配置里的 Secret 字段 → 明文;`extraSecrets`:其余要遮蔽的明文(如本地 `db` 的密码)。
10
- * 遮蔽时每个明文连同它的 URL 编码与表单编码写法一起替换。
11
- */
12
- export declare function createRecorder(options: {
13
- secrets: Record<string, string>;
14
- extraSecrets?: readonly string[];
15
- }): FixtureRecorder;
@@ -1,116 +0,0 @@
1
- import { canonicalRequestUrl } from './fixtures.js';
2
- import { redactHeaders, redactSecretValues, redactUrl, secretRedactionTexts } from './redaction.js';
3
- function isRecord(value) {
4
- return typeof value === 'object' && value !== null && !Array.isArray(value);
5
- }
6
- function redactDeep(value, secrets) {
7
- const plain = value === undefined ? null : JSON.parse(JSON.stringify(value));
8
- const walk = (item) => {
9
- if (typeof item === 'string')
10
- return redactSecretValues(item, secrets);
11
- if (Array.isArray(item))
12
- return item.map(walk);
13
- if (isRecord(item))
14
- return Object.fromEntries(Object.entries(item).map(([key, child]) => [key, walk(child)]));
15
- return item;
16
- };
17
- return walk(plain);
18
- }
19
- export function createRecorder(options) {
20
- const secrets = secretRedactionTexts([...Object.values(options.secrets), ...(options.extraSecrets ?? [])]
21
- .filter((value) => typeof value === 'string'));
22
- const redact = (text) => redactSecretValues(text, secrets);
23
- const recorded = [];
24
- const openCursors = new Map();
25
- let seq = 0;
26
- const errorOf = (exchange) => exchange.error ? { code: exchange.error.code, message: redact(exchange.error.message) } : null;
27
- const recordHttp = (exchange) => {
28
- const request = exchange.request;
29
- if (!isRecord(request) || typeof request.url !== 'string' || typeof request.method !== 'string')
30
- return;
31
- const bodyEncoding = request.bodyEncoding === 'base64' ? 'base64' : 'utf8';
32
- const body = typeof request.body === 'string' ? (bodyEncoding === 'utf8' ? redact(request.body) : request.body) : null;
33
- const response = exchange.response;
34
- seq += 1;
35
- recorded.push({
36
- seq,
37
- kind: 'http',
38
- request: {
39
- method: request.method,
40
- url: redactUrl(canonicalRequestUrl(request.url, request.query), secrets),
41
- headers: redactHeaders(isRecord(request.headers) ? request.headers : {}, secrets),
42
- body,
43
- bodyEncoding,
44
- },
45
- response: exchange.error || !isRecord(response)
46
- ? null
47
- : {
48
- status: response.status,
49
- headers: redactHeaders(response.headers ?? {}, secrets),
50
- body: response.bodyEncoding === 'utf8' ? redact(response.body) : response.body,
51
- bodyEncoding: response.bodyEncoding,
52
- },
53
- error: errorOf(exchange) ?? (isRecord(response) ? null : { code: 'HOST_CALL_FAILED', message: 'no response was recorded' }),
54
- });
55
- };
56
- const recordDb = (exchange) => {
57
- const request = exchange.request;
58
- if (!isRecord(request))
59
- return;
60
- if (typeof request.sql === 'string') {
61
- const response = exchange.response;
62
- seq += 1;
63
- const entry = {
64
- seq,
65
- kind: 'db',
66
- sql: redact(request.sql),
67
- params: redactDeep(Array.isArray(request.params) ? request.params : [], secrets),
68
- fields: isRecord(response) && Array.isArray(response.fields) ? response.fields.map((field) => ({ name: field.name, dataTypeId: field.dataTypeId })) : [],
69
- rows: isRecord(response) && Array.isArray(response.rows) ? redactDeep(response.rows, secrets) : [],
70
- error: errorOf(exchange),
71
- };
72
- recorded.push(entry);
73
- if (isRecord(response) && typeof response.cursorId === 'string' && response.done === false)
74
- openCursors.set(response.cursorId, entry);
75
- return;
76
- }
77
- if (typeof request.cursorId === 'string') {
78
- const entry = openCursors.get(request.cursorId);
79
- if (!entry)
80
- return;
81
- const response = exchange.response;
82
- if (isRecord(response) && Array.isArray(response.rows))
83
- entry.rows.push(...redactDeep(response.rows, secrets));
84
- const error = errorOf(exchange);
85
- if (error)
86
- entry.error = error;
87
- if (error || !isRecord(response) || response.done !== false)
88
- openCursors.delete(request.cursorId);
89
- }
90
- };
91
- const recordCrypto = (exchange) => {
92
- if (exchange.error || !isRecord(exchange.request) || !isRecord(exchange.response) || typeof exchange.response.value !== 'string')
93
- return;
94
- const request = redactDeep(exchange.request, secrets);
95
- seq += 1;
96
- recorded.push({ seq, kind: 'crypto', request, value: redact(exchange.response.value) });
97
- };
98
- return {
99
- onExchange(exchange) {
100
- switch (exchange.kind) {
101
- case 'http':
102
- recordHttp(exchange);
103
- return;
104
- case 'db':
105
- recordDb(exchange);
106
- return;
107
- case 'crypto':
108
- recordCrypto(exchange);
109
- return;
110
- }
111
- },
112
- exchanges() {
113
- return JSON.parse(JSON.stringify(recorded));
114
- },
115
- };
116
- }
@@ -1,28 +0,0 @@
1
- /**
2
- * 遮蔽规则(录制夹具、本地 `read` 的请求行、本地运行日志共用),与 shared `utils/connector-redaction.ts` 同一套规则;
3
- * SDK 不依赖 shared,这里另写一份,`test/testing/redaction.test.ts` 对照两份实现的结果。
4
- *
5
- * - 长度 ≥ 4 的 Secret 明文 → `***`(长的先替换,避免短值截断长值);
6
- * - `CREDENTIAL_HEADER_NAMES` 里的请求 / 响应头(小写比较)整值 → `***`;
7
- * - 含句柄 `ontos:secret:` 的 query 值 → `***`;请求里的句柄本身原样保留(网关替换前的形态)。
8
- */
9
- /** 凭证类请求 / 响应头(小写);与 shared 的 `CREDENTIAL_HEADER_NAMES` 同值。 */
10
- export declare const CREDENTIAL_HEADER_NAMES: readonly string[];
11
- /** 遮蔽的最短明文长度:更短的值误伤面太大,不替换。 */
12
- export declare const REDACT_MIN_LENGTH = 4;
13
- export declare const REDACTED = "***";
14
- /** 文本里是否含凭证句柄(原文或 URL 编码写法)。 */
15
- export declare function containsSecretHandle(text: string): boolean;
16
- /** 把文本里长度 ≥ 4 的 Secret 明文替换为 `***`(先替换较长的)。 */
17
- export declare function redactSecretValues(text: string, secrets: readonly string[]): string;
18
- /**
19
- * 遮蔽用的明文清单:每个 Secret 加上它进 URL 后的两种写法——`encodeURIComponent` 与表单编码(`URLSearchParams`,空格写成 `+`)。
20
- * 句柄替换进 URL 时用的是编码写法,响应的 `url`、日志与录制里出现的也是它,只遮明文会漏掉。
21
- */
22
- export declare function secretRedactionTexts(secrets: readonly string[]): string[];
23
- /** 凭证类头(名字按小写比较)的值换成 `***`,其余头的值做 `redactSecretValues`;头名保持原样。 */
24
- export declare function redactHeaders(headers: Record<string, string>, secrets: readonly string[]): Record<string, string>;
25
- /** query 参数:值含句柄的换成 `***`,其余值做 `redactSecretValues`。 */
26
- export declare function redactQuery(query: Record<string, string | string[]>, secrets: readonly string[]): Record<string, string | string[]>;
27
- /** URL:值含句柄的 query 参数换成 `***`,再遮蔽明文;解析不了时只做明文遮蔽。与 connector 的 `traceUrl` 相同。 */
28
- export declare function redactUrl(url: string, secrets: readonly string[]): string;
@@ -1,67 +0,0 @@
1
- export const CREDENTIAL_HEADER_NAMES = Object.freeze([
2
- 'authorization',
3
- 'proxy-authorization',
4
- 'cookie',
5
- 'set-cookie',
6
- 'x-api-key',
7
- 'x-auth-token',
8
- ]);
9
- export const REDACT_MIN_LENGTH = 4;
10
- export const REDACTED = '***';
11
- const CREDENTIAL_HEADER_SET = new Set(CREDENTIAL_HEADER_NAMES);
12
- const HANDLE_PATTERN = /ontos(?::|%3[Aa])secret(?::|%3[Aa])/;
13
- export function containsSecretHandle(text) {
14
- return HANDLE_PATTERN.test(text);
15
- }
16
- export function redactSecretValues(text, secrets) {
17
- const candidates = [...new Set(secrets.filter((secret) => typeof secret === 'string' && secret.length >= REDACT_MIN_LENGTH))]
18
- .sort((a, b) => b.length - a.length);
19
- let result = text;
20
- for (const secret of candidates) {
21
- if (result.includes(secret))
22
- result = result.split(secret).join(REDACTED);
23
- }
24
- return result;
25
- }
26
- export function secretRedactionTexts(secrets) {
27
- const result = new Set();
28
- for (const secret of secrets) {
29
- if (typeof secret !== 'string')
30
- continue;
31
- result.add(secret);
32
- result.add(encodeURIComponent(secret));
33
- result.add(new URLSearchParams([['', secret]]).toString().slice(1));
34
- }
35
- return [...result];
36
- }
37
- export function redactHeaders(headers, secrets) {
38
- const result = {};
39
- for (const [name, value] of Object.entries(headers)) {
40
- result[name] = CREDENTIAL_HEADER_SET.has(name.toLowerCase()) ? REDACTED : redactSecretValues(String(value), secrets);
41
- }
42
- return result;
43
- }
44
- export function redactQuery(query, secrets) {
45
- const one = (value) => (containsSecretHandle(value) ? REDACTED : redactSecretValues(value, secrets));
46
- const result = {};
47
- for (const [name, value] of Object.entries(query))
48
- result[name] = Array.isArray(value) ? value.map(one) : one(value);
49
- return result;
50
- }
51
- export function redactUrl(url, secrets) {
52
- let text = url;
53
- try {
54
- const parsed = new URL(url);
55
- const entries = [...parsed.searchParams.entries()];
56
- if (entries.some(([, value]) => containsSecretHandle(value))) {
57
- const masked = new URLSearchParams();
58
- for (const [name, value] of entries)
59
- masked.append(name, containsSecretHandle(value) ? REDACTED : value);
60
- parsed.search = masked.toString();
61
- text = parsed.toString();
62
- }
63
- }
64
- catch {
65
- }
66
- return redactSecretValues(text, secrets);
67
- }
@@ -1,50 +0,0 @@
1
- import type { DefinitionSummary, NormalizedDiscoverResult } from '../validate.js';
2
- import { type ComplianceFinding } from './compliance.js';
3
- import { type ConnectorFixtureFile, type FixtureExchange, type LoadedFixture } from './fixtures.js';
4
- import type { GuestExecutor, LocalHost } from './types.js';
5
- /** 回放宿主自己的宿主调用失败(`code` 回给 guest)。 */
6
- export declare class FixtureReplayError extends Error {
7
- readonly code: string;
8
- constructor(code: string, message: string);
9
- }
10
- export interface FixtureMismatch {
11
- method: string;
12
- detail: string;
13
- }
14
- export type ReplayHost = LocalHost & {
15
- unused(): FixtureExchange[];
16
- mismatches(): FixtureMismatch[];
17
- };
18
- /** 与 shared 的 `ConnectorDeclaration` 同形。 */
19
- export interface FixtureDeclaration {
20
- definition: DefinitionSummary;
21
- streams: NormalizedDiscoverResult['streams'];
22
- pushes: NormalizedDiscoverResult['pushes'];
23
- }
24
- /** 一个夹具用例的回放结果(与 shared 的 `FixtureCaseResult` 同形)。 */
25
- export interface FixtureCaseResult {
26
- name: string;
27
- ok: boolean;
28
- findings: ComplianceFinding[];
29
- stats: {
30
- records: Record<string, number>;
31
- removals: Record<string, number>;
32
- checkpoints: number;
33
- rejected: number;
34
- };
35
- declaration: FixtureDeclaration | null;
36
- error: {
37
- code: string;
38
- message: string;
39
- stack: string | null;
40
- } | null;
41
- }
42
- export interface RunFixtureOptions {
43
- /** 每次执行(不是整个用例)的超时,交给 `executor.run`。 */
44
- timeoutMs?: number;
45
- }
46
- export declare function createReplayHost(fixture: LoadedFixture): ReplayHost;
47
- /** 回放一个用例(见文件头)。 */
48
- export declare function runFixture(executor: GuestExecutor, fixture: LoadedFixture, options?: RunFixtureOptions): Promise<FixtureCaseResult>;
49
- /** 解析夹具文件(格式错误抛 `FixtureFormatError`)后逐个回放,按用例名排序。 */
50
- export declare function runFixtures(executor: GuestExecutor, files: ConnectorFixtureFile[], options?: RunFixtureOptions): Promise<FixtureCaseResult[]>;