@deepexi/datasense-connector 3.0.1 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/README.md +158 -139
  2. package/package.json +19 -28
  3. package/scripts/build.ts +188 -0
  4. package/src/define.ts +267 -0
  5. package/src/descriptor.ts +116 -0
  6. package/src/errors.ts +105 -0
  7. package/src/index.ts +108 -0
  8. package/src/internal/location.ts +23 -0
  9. package/src/runtime/guest/bridge.ts +238 -0
  10. package/src/runtime/guest/cache.ts +43 -0
  11. package/src/runtime/guest/check.ts +17 -0
  12. package/src/runtime/guest/crypto.ts +166 -0
  13. package/src/runtime/guest/ctx.ts +157 -0
  14. package/src/runtime/guest/db.ts +59 -0
  15. package/src/runtime/guest/failure.ts +69 -0
  16. package/src/runtime/guest/http.ts +194 -0
  17. package/src/runtime/guest/index.ts +102 -0
  18. package/{runtime/guest/limits.d.ts → src/runtime/guest/limits.ts} +12 -12
  19. package/src/runtime/guest/loader.ts +210 -0
  20. package/src/runtime/guest/log.ts +47 -0
  21. package/src/runtime/guest/map.ts +45 -0
  22. package/src/runtime/guest/operation.ts +193 -0
  23. package/src/runtime/guest/paginate.ts +169 -0
  24. package/src/runtime/guest/pull.ts +177 -0
  25. package/src/runtime/guest/push.ts +92 -0
  26. package/src/runtime/guest/signal.ts +14 -0
  27. package/src/runtime/guest/suggest.ts +51 -0
  28. package/src/runtime/guest/time.ts +137 -0
  29. package/src/runtime/guest-runtime.ts +75 -0
  30. package/src/runtime/index.ts +53 -0
  31. package/src/runtime/protocol.ts +557 -0
  32. package/src/runtime/session.ts +216 -0
  33. package/src/testing/child.ts +21 -0
  34. package/src/testing/compliance.ts +343 -0
  35. package/src/testing/engine.ts +445 -0
  36. package/src/testing/fixtures.ts +591 -0
  37. package/src/testing/gateway.ts +482 -0
  38. package/src/testing/import-guard.ts +221 -0
  39. package/src/testing/index.ts +43 -0
  40. package/src/testing/local-db.ts +226 -0
  41. package/src/testing/mock-external-system.ts +249 -0
  42. package/src/testing/pinned-http.ts +144 -0
  43. package/src/testing/recorder.ts +140 -0
  44. package/src/testing/redaction.ts +94 -0
  45. package/src/testing/replay.ts +437 -0
  46. package/src/testing/run-tests.ts +115 -0
  47. package/src/testing/schema.ts +70 -0
  48. package/src/testing/source-loader.ts +374 -0
  49. package/src/testing/types.ts +78 -0
  50. package/src/types.ts +329 -0
  51. package/src/validate.ts +267 -0
  52. package/test/build.test.ts +159 -0
  53. package/test/define.test-d.ts +182 -0
  54. package/test/fixtures/acceptance.ts +156 -0
  55. package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
  56. package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
  57. package/test/fixtures/connectors/http-pull/package.json +5 -0
  58. package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
  59. package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
  60. package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
  61. package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
  62. package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
  63. package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
  64. package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
  65. package/test/fixtures/connectors/pg-pull/package.json +5 -0
  66. package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
  67. package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
  68. package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
  69. package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
  70. package/test/fixtures/descriptor.ts +54 -0
  71. package/test/fixtures/kingdee-example.ts +82 -0
  72. package/test/fixtures/record-acceptance.ts +32 -0
  73. package/test/runtime/ctx.test.ts +434 -0
  74. package/test/runtime/helpers.ts +157 -0
  75. package/test/runtime/jobs.test.ts +444 -0
  76. package/test/runtime/operation.test.ts +175 -0
  77. package/test/runtime/session.test.ts +229 -0
  78. package/test/testing/acceptance-fixtures.test.ts +69 -0
  79. package/test/testing/compliance.test.ts +247 -0
  80. package/test/testing/engine.test.ts +283 -0
  81. package/test/testing/fixtures.test.ts +221 -0
  82. package/test/testing/gateway.test.ts +337 -0
  83. package/test/testing/import-guard.test.ts +45 -0
  84. package/test/testing/mock-external-system.test.ts +144 -0
  85. package/test/testing/operation-fixtures.test.ts +160 -0
  86. package/test/testing/recorder.test.ts +163 -0
  87. package/test/testing/redaction.test.ts +54 -0
  88. package/test/testing/replay.test.ts +207 -0
  89. package/test/testing/run-tests.test.ts +137 -0
  90. package/test/testing/source-loader.test.ts +220 -0
  91. package/test/validate-config.test.ts +104 -0
  92. package/tsconfig.build.json +23 -0
  93. package/tsconfig.json +19 -0
  94. package/cjs/index.cjs +0 -1015
  95. package/cjs/runtime.cjs +0 -752
  96. package/cjs/testing.cjs +0 -3591
  97. package/errors.d.ts +0 -17
  98. package/errors.js +0 -53
  99. package/guest.cjs +0 -2659
  100. package/index.d.ts +0 -13
  101. package/index.js +0 -7
  102. package/internal/location.d.ts +0 -6
  103. package/internal/location.js +0 -8
  104. package/presentation.d.ts +0 -38
  105. package/presentation.js +0 -268
  106. package/runtime/guest/bridge.d.ts +0 -53
  107. package/runtime/guest/bridge.js +0 -187
  108. package/runtime/guest/cache.d.ts +0 -3
  109. package/runtime/guest/cache.js +0 -42
  110. package/runtime/guest/crypto.d.ts +0 -13
  111. package/runtime/guest/crypto.js +0 -177
  112. package/runtime/guest/ctx.d.ts +0 -30
  113. package/runtime/guest/ctx.js +0 -124
  114. package/runtime/guest/db.d.ts +0 -6
  115. package/runtime/guest/db.js +0 -59
  116. package/runtime/guest/discover.d.ts +0 -29
  117. package/runtime/guest/discover.js +0 -51
  118. package/runtime/guest/failure.d.ts +0 -24
  119. package/runtime/guest/failure.js +0 -63
  120. package/runtime/guest/http.d.ts +0 -4
  121. package/runtime/guest/http.js +0 -204
  122. package/runtime/guest/index.d.ts +0 -12
  123. package/runtime/guest/index.js +0 -89
  124. package/runtime/guest/limits.js +0 -12
  125. package/runtime/guest/loader.d.ts +0 -15
  126. package/runtime/guest/loader.js +0 -42
  127. package/runtime/guest/log.d.ts +0 -10
  128. package/runtime/guest/log.js +0 -36
  129. package/runtime/guest/map.d.ts +0 -4
  130. package/runtime/guest/map.js +0 -45
  131. package/runtime/guest/paginate.d.ts +0 -6
  132. package/runtime/guest/paginate.js +0 -180
  133. package/runtime/guest/pull.d.ts +0 -12
  134. package/runtime/guest/pull.js +0 -150
  135. package/runtime/guest/push.d.ts +0 -18
  136. package/runtime/guest/push.js +0 -80
  137. package/runtime/guest/signal.d.ts +0 -3
  138. package/runtime/guest/signal.js +0 -12
  139. package/runtime/guest/time.d.ts +0 -5
  140. package/runtime/guest/time.js +0 -127
  141. package/runtime/guest-runtime.d.ts +0 -7
  142. package/runtime/guest-runtime.js +0 -51
  143. package/runtime/index.d.ts +0 -18
  144. package/runtime/index.js +0 -3
  145. package/runtime/protocol.d.ts +0 -276
  146. package/runtime/protocol.js +0 -409
  147. package/runtime/session.d.ts +0 -12
  148. package/runtime/session.js +0 -191
  149. package/t.d.ts +0 -18
  150. package/t.js +0 -26
  151. package/testing/child.cjs +0 -350
  152. package/testing/child.d.ts +0 -1
  153. package/testing/child.js +0 -12
  154. package/testing/compliance.d.ts +0 -41
  155. package/testing/compliance.js +0 -291
  156. package/testing/engine.d.ts +0 -66
  157. package/testing/engine.js +0 -339
  158. package/testing/fixtures.d.ts +0 -145
  159. package/testing/fixtures.js +0 -407
  160. package/testing/gateway.d.ts +0 -65
  161. package/testing/gateway.js +0 -402
  162. package/testing/import-guard.d.ts +0 -57
  163. package/testing/import-guard.js +0 -187
  164. package/testing/index.d.ts +0 -33
  165. package/testing/index.js +0 -10
  166. package/testing/local-db.d.ts +0 -27
  167. package/testing/local-db.js +0 -181
  168. package/testing/pinned-http.d.ts +0 -14
  169. package/testing/pinned-http.js +0 -125
  170. package/testing/recorder.d.ts +0 -15
  171. package/testing/recorder.js +0 -116
  172. package/testing/redaction.d.ts +0 -28
  173. package/testing/redaction.js +0 -67
  174. package/testing/replay.d.ts +0 -50
  175. package/testing/replay.js +0 -313
  176. package/testing/schema.d.ts +0 -13
  177. package/testing/schema.js +0 -65
  178. package/testing/source-loader.d.ts +0 -19
  179. package/testing/source-loader.js +0 -238
  180. package/testing/types.d.ts +0 -79
  181. package/testing/types.js +0 -18
  182. package/types.d.ts +0 -302
  183. package/types.js +0 -13
  184. package/validate.d.ts +0 -78
  185. package/validate.js +0 -597
package/testing/replay.js DELETED
@@ -1,313 +0,0 @@
1
- import { createHash } from 'node:crypto';
2
- import { checkCompliance, scopedTables } from './compliance.js';
3
- import { canonicalJson, canonicalRequestUrl, parseFixtures, } from './fixtures.js';
4
- import { redactSecretValues, redactUrl } from './redaction.js';
5
- export class FixtureReplayError extends Error {
6
- code;
7
- constructor(code, message) {
8
- super(message);
9
- this.name = 'FixtureReplayError';
10
- this.code = code;
11
- }
12
- }
13
- const MISMATCH = 'FIXTURE_EXCHANGE_MISMATCH';
14
- const PLAIN_HANDLE = /ontos:secret:([A-Za-z_][A-Za-z0-9_]*)/g;
15
- function isRecord(value) {
16
- return typeof value === 'object' && value !== null && !Array.isArray(value);
17
- }
18
- function preview(text, max = 200) {
19
- return text.length > max ? `${text.slice(0, max)}…` : text;
20
- }
21
- export function createReplayHost(fixture) {
22
- const caseSecrets = fixture.case.secrets ?? {};
23
- const secretValues = Object.values(caseSecrets);
24
- const redact = (text) => redactSecretValues(text, secretValues);
25
- const used = new Set();
26
- const mismatched = [];
27
- const failingCursors = new Map();
28
- const cache = new Map();
29
- const mismatch = (method, detail) => {
30
- mismatched.push({ method, detail });
31
- throw new FixtureReplayError(MISMATCH, `no recorded exchange in fixture "${fixture.name}" matches ${method} ${detail}`);
32
- };
33
- const take = (kind, matches) => {
34
- for (const exchange of fixture.exchanges) {
35
- if (exchange.kind !== kind || used.has(exchange.seq))
36
- continue;
37
- const typed = exchange;
38
- if (!matches(typed))
39
- continue;
40
- used.add(exchange.seq);
41
- return typed;
42
- }
43
- return null;
44
- };
45
- const lookupSecret = (field) => {
46
- const value = Object.hasOwn(caseSecrets, field) ? caseSecrets[field] : undefined;
47
- if (typeof value !== 'string')
48
- throw new FixtureReplayError('EGRESS_SECRET_UNAVAILABLE', `Secret '${field}' is not in the fixture's case.json secrets`);
49
- return value;
50
- };
51
- const http = (payload) => {
52
- if (!isRecord(payload) || typeof payload.method !== 'string' || typeof payload.url !== 'string') {
53
- throw new FixtureReplayError('BAD_REQUEST', 'http request must have a method and a url');
54
- }
55
- const query = isRecord(payload.query) ? payload.query : undefined;
56
- const scriptUrl = canonicalRequestUrl(payload.url, query);
57
- const url = redactUrl(scriptUrl, secretValues);
58
- const bodyEncoding = payload.bodyEncoding === 'base64' ? 'base64' : 'utf8';
59
- const body = typeof payload.body === 'string' ? (bodyEncoding === 'utf8' ? redact(payload.body) : payload.body) : null;
60
- const method = payload.method;
61
- const exchange = take('http', (item) => item.request.method === method && item.request.url === url && item.request.body === body && (body === null || item.request.bodyEncoding === bodyEncoding));
62
- if (!exchange)
63
- return mismatch('http', `${method} ${url}${body !== null ? ` with body ${preview(body)}` : ''}`);
64
- if (exchange.error)
65
- throw new FixtureReplayError(exchange.error.code, exchange.error.message);
66
- if (!exchange.response)
67
- throw new FixtureReplayError('HOST_CALL_FAILED', 'the recorded exchange has no response');
68
- return { status: exchange.response.status, url: scriptUrl, headers: { ...exchange.response.headers }, body: exchange.response.body, bodyEncoding: exchange.response.bodyEncoding };
69
- };
70
- const dbQuery = (payload) => {
71
- if (!isRecord(payload) || typeof payload.sql !== 'string')
72
- throw new FixtureReplayError('BAD_REQUEST', 'db.query needs sql');
73
- const sql = redact(payload.sql);
74
- const params = canonicalJson(JSON.parse(redact(JSON.stringify(Array.isArray(payload.params) ? payload.params : []))));
75
- const exchange = take('db', (item) => item.sql === sql && canonicalJson(item.params) === params);
76
- if (!exchange)
77
- return mismatch('db.query', `${preview(sql.replace(/\s+/g, ' ').trim())} with params ${preview(params)}`);
78
- const rows = exchange.rows.map((row) => ({ ...row }));
79
- const fields = exchange.fields.map((field) => ({ ...field }));
80
- if (!exchange.error)
81
- return { cursorId: null, fields, rows, done: true };
82
- if (fields.length === 0 && rows.length === 0)
83
- throw new FixtureReplayError(exchange.error.code, exchange.error.message);
84
- const cursorId = `replay-${exchange.seq}`;
85
- failingCursors.set(cursorId, exchange.error);
86
- return { cursorId, fields, rows, done: false };
87
- };
88
- const crypto = (payload) => {
89
- if (!isRecord(payload) || typeof payload.op !== 'string')
90
- throw new FixtureReplayError('BAD_REQUEST', 'crypto request must have an op');
91
- const request = JSON.parse(redact(JSON.stringify(payload)));
92
- const key = canonicalJson(request);
93
- const exchange = take('crypto', (item) => canonicalJson(item.request) === key);
94
- if (exchange)
95
- return { value: exchange.value };
96
- if (request.op === 'hash') {
97
- const data = request.data.replace(PLAIN_HANDLE, (_match, field) => lookupSecret(field));
98
- return { value: createHash(request.algorithm).update(data, 'utf8').digest(request.encoding) };
99
- }
100
- return mismatch('crypto', `${request.op} ${request.algorithm}`);
101
- };
102
- const call = async (method, payload) => {
103
- switch (method) {
104
- case 'http':
105
- return http(payload);
106
- case 'db.query':
107
- return dbQuery(payload);
108
- case 'db.next': {
109
- const cursorId = isRecord(payload) && typeof payload.cursorId === 'string' ? payload.cursorId : '';
110
- const error = failingCursors.get(cursorId);
111
- if (!error)
112
- throw new FixtureReplayError('DB_CURSOR_NOT_FOUND', "The cursor doesn't exist or was closed");
113
- failingCursors.delete(cursorId);
114
- throw new FixtureReplayError(error.code, error.message);
115
- }
116
- case 'db.close': {
117
- const cursorId = isRecord(payload) && typeof payload.cursorId === 'string' ? payload.cursorId : '';
118
- failingCursors.delete(cursorId);
119
- return null;
120
- }
121
- case 'crypto':
122
- return crypto(payload);
123
- case 'cache.get': {
124
- const key = isRecord(payload) && typeof payload.key === 'string' ? payload.key : '';
125
- const entry = cache.get(key);
126
- if (!entry || Date.parse(entry.expiresAt) <= Date.now())
127
- return null;
128
- return entry;
129
- }
130
- case 'cache.set': {
131
- if (!isRecord(payload) || typeof payload.key !== 'string' || typeof payload.value !== 'string' || typeof payload.ttlSeconds !== 'number') {
132
- throw new FixtureReplayError('BAD_REQUEST', 'cache.set needs key, value and ttlSeconds');
133
- }
134
- cache.set(payload.key, { value: payload.value, expiresAt: new Date(Date.now() + payload.ttlSeconds * 1000).toISOString() });
135
- return null;
136
- }
137
- case 'secret.reveal': {
138
- const field = isRecord(payload) && typeof payload.field === 'string' ? payload.field : '';
139
- return { value: lookupSecret(field) };
140
- }
141
- default:
142
- throw new FixtureReplayError('HOST_METHOD_UNSUPPORTED', `fixture replay does not answer ${method}`);
143
- }
144
- };
145
- return {
146
- call,
147
- redact,
148
- async close() {
149
- failingCursors.clear();
150
- },
151
- unused() {
152
- return fixture.exchanges.filter((exchange) => !used.has(exchange.seq)).map((exchange) => JSON.parse(JSON.stringify(exchange)));
153
- },
154
- mismatches() {
155
- return mismatched.map((item) => ({ ...item }));
156
- },
157
- };
158
- }
159
- function countBy(items) {
160
- const counts = {};
161
- for (const item of items)
162
- counts[item.table] = (counts[item.table] ?? 0) + 1;
163
- return counts;
164
- }
165
- function toDeclaration(declaration) {
166
- if (!declaration || !isRecord(declaration.discover))
167
- return null;
168
- const discover = declaration.discover;
169
- return { definition: declaration.summary, streams: discover.streams, pushes: discover.pushes };
170
- }
171
- function exchangeSummary(exchange) {
172
- switch (exchange.kind) {
173
- case 'http':
174
- return `${exchange.request.method} ${exchange.request.url}`;
175
- case 'db':
176
- return preview(exchange.sql.replace(/\s+/g, ' ').trim());
177
- case 'crypto':
178
- return `${exchange.request.op} ${exchange.request.algorithm}`;
179
- }
180
- }
181
- function scopeOf(fixtureCase) {
182
- if (fixtureCase.kind === 'pull')
183
- return { stream: fixtureCase.stream };
184
- if (fixtureCase.kind === 'push')
185
- return { push: fixtureCase.push };
186
- return {};
187
- }
188
- function sameCounts(expected, actual) {
189
- return Object.entries(expected)
190
- .filter(([table, count]) => (actual[table] ?? 0) !== count)
191
- .map(([table, count]) => `${table}: expected ${count}, got ${actual[table] ?? 0}`);
192
- }
193
- function checkExpectation(fixture, result, declaration) {
194
- const expect = fixture.case.expect;
195
- if (!expect)
196
- return [];
197
- const findings = [];
198
- const failed = (what, problems, detail) => {
199
- findings.push({ code: 'EXPECTATION_FAILED', message: `expect.${what} does not match (${problems.join('; ')})`, detail });
200
- };
201
- if (expect.records) {
202
- const actual = countBy(result.records);
203
- const problems = sameCounts(expect.records, actual);
204
- if (problems.length > 0)
205
- failed('records', problems, { expected: expect.records, actual });
206
- }
207
- if (expect.removals) {
208
- const actual = countBy(result.removals);
209
- const problems = sameCounts(expect.removals, actual);
210
- if (problems.length > 0)
211
- failed('removals', problems, { expected: expect.removals, actual });
212
- }
213
- if (expect.rejected !== undefined && expect.rejected !== result.rejected.length) {
214
- failed('rejected', [`expected ${expect.rejected}, got ${result.rejected.length}`], { expected: expect.rejected, actual: result.rejected.length });
215
- }
216
- if (expect.checkpoints !== undefined && expect.checkpoints !== result.checkpoints.length) {
217
- failed('checkpoints', [`expected ${expect.checkpoints}, got ${result.checkpoints.length}`], { expected: expect.checkpoints, actual: result.checkpoints.length });
218
- }
219
- if (expect.tables) {
220
- const actual = Object.keys(scopedTables(declaration, scopeOf(fixture.case)) ?? {}).sort();
221
- const expected = [...new Set(expect.tables)].sort();
222
- if (actual.length !== expected.length || actual.some((table, index) => table !== expected[index])) {
223
- failed('tables', [`expected [${expected.join(', ')}], got [${actual.join(', ')}]`], { expected, actual });
224
- }
225
- }
226
- return findings;
227
- }
228
- export async function runFixture(executor, fixture, options = {}) {
229
- const fixtureCase = fixture.case;
230
- const config = fixtureCase.config;
231
- const secretFields = Object.keys(fixtureCase.secrets ?? {});
232
- const execute = async (job, limit) => {
233
- const host = createReplayHost(fixture);
234
- try {
235
- const result = await executor.run(job, host, {
236
- ...(limit !== undefined ? { limit } : {}),
237
- ...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
238
- });
239
- return { host, result };
240
- }
241
- finally {
242
- await host.close();
243
- }
244
- };
245
- let main;
246
- let declaration = null;
247
- let rerun;
248
- let error = null;
249
- switch (fixtureCase.kind) {
250
- case 'pull': {
251
- const job = {
252
- kind: 'pull', config, secretFields, stream: fixtureCase.stream, mode: fixtureCase.mode, state: fixtureCase.state ?? null,
253
- streamConfig: fixtureCase.streamConfig ?? {}, tables: [], discoverFirst: true,
254
- };
255
- main = await execute(job, fixtureCase.limit);
256
- declaration = main.result.declaration;
257
- if (fixtureCase.mode === 'full' && main.result.error === null)
258
- rerun = (await execute(job, fixtureCase.limit)).result;
259
- break;
260
- }
261
- case 'push': {
262
- const discovered = await execute({ kind: 'discover', config, secretFields });
263
- if (discovered.result.error) {
264
- main = discovered;
265
- error = { ...discovered.result.error, message: `discover failed before replaying the push requests: ${discovered.result.error.message}` };
266
- break;
267
- }
268
- declaration = discovered.result.declaration;
269
- main = await execute({
270
- kind: 'push', config, secretFields,
271
- requests: fixture.requests.map((request) => ({ ...request, headers: { ...request.headers }, push: fixtureCase.push })),
272
- });
273
- break;
274
- }
275
- case 'discover':
276
- main = await execute({ kind: 'discover', config, secretFields });
277
- declaration = main.result.declaration;
278
- break;
279
- }
280
- const result = main.result;
281
- error ??= result.error;
282
- const findings = [];
283
- for (const item of main.host.mismatches()) {
284
- findings.push({ code: 'FIXTURE_EXCHANGE_MISMATCH', message: `no recorded exchange matches ${item.method} ${item.detail}`, detail: item });
285
- }
286
- if (error === null) {
287
- for (const exchange of main.host.unused()) {
288
- findings.push({
289
- code: 'FIXTURE_EXCHANGE_UNUSED',
290
- message: `recorded exchange ${exchange.seq} (${exchange.kind} ${exchangeSummary(exchange)}) was not used`,
291
- detail: { seq: exchange.seq, kind: exchange.kind },
292
- });
293
- }
294
- findings.push(...checkCompliance({
295
- declaration, ...scopeOf(fixtureCase), ...(fixtureCase.kind === 'pull' ? { mode: fixtureCase.mode } : {}), result, ...(rerun ? { rerun } : {}),
296
- }));
297
- findings.push(...checkExpectation(fixture, result, declaration));
298
- }
299
- return {
300
- name: fixture.name,
301
- ok: error === null && findings.every((finding) => finding.code === 'FIXTURE_EXCHANGE_UNUSED'),
302
- findings,
303
- stats: { records: countBy(result.records), removals: countBy(result.removals), checkpoints: result.checkpoints.length, rejected: result.rejected.length },
304
- declaration: toDeclaration(declaration),
305
- error,
306
- };
307
- }
308
- export async function runFixtures(executor, files, options = {}) {
309
- const results = [];
310
- for (const fixture of parseFixtures(files))
311
- results.push(await runFixture(executor, fixture, options));
312
- return results;
313
- }
@@ -1,13 +0,0 @@
1
- /**
2
- * 由记录推断列类型(预览面板的「推断结构」、合规检查共用)。纯函数,不依赖 Node 模块(meta 与 UI 都可能用)。
3
- */
4
- export type InferredColumnType = 'string' | 'number' | 'boolean' | 'timestamp' | 'json' | 'null';
5
- /** ISO-8601 日期时间字符串 → 毫秒时刻;不是时返回 null。只有日期的写法不算(推断为 `string`)。 */
6
- export declare function parseIsoDateTime(value: string): number | null;
7
- /** ISO-8601 时间字符串(日期时间,或只有日期)→ 毫秒时刻;断点单调判定用。 */
8
- export declare function parseIsoInstant(value: string): number | null;
9
- /**
10
- * 列 → 推断类型:空值不参与;全是空值为 `'null'`;字符串全是 ISO 日期时间为 `'timestamp'`,混有普通字符串为 `'string'`;
11
- * 对象 / 数组或多种类型混用为 `'json'`。列按首次出现的顺序。
12
- */
13
- export declare function inferRecordSchema(rows: Array<Record<string, unknown>>): Record<string, InferredColumnType>;
package/testing/schema.js DELETED
@@ -1,65 +0,0 @@
1
- const ISO_DATE_TIME = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2})(?:\.\d{1,9})?)?(Z|[+-]\d{2}(?::?\d{2})?)?$/i;
2
- const ISO_DATE = /^(\d{4})-(\d{2})-(\d{2})$/;
3
- function validDate(year, month, day) {
4
- return month >= 1 && month <= 12 && day >= 1 && day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
5
- }
6
- export function parseIsoDateTime(value) {
7
- const match = ISO_DATE_TIME.exec(value);
8
- if (!match)
9
- return null;
10
- const [, year, month, day, hour, minute, second, offset] = match;
11
- if (!validDate(Number(year), Number(month), Number(day)) || Number(hour) > 23 || Number(minute) > 59 || Number(second ?? '0') > 59)
12
- return null;
13
- const local = value.slice(0, value.length - (offset?.length ?? 0)).replace(' ', 'T').replace(/(\.\d{3})\d+$/, '$1');
14
- let zone = 'Z';
15
- if (offset !== undefined && offset.toUpperCase() !== 'Z') {
16
- const digits = offset.slice(1).replace(':', '');
17
- zone = `${offset[0]}${digits.slice(0, 2)}:${digits.length > 2 ? digits.slice(2) : '00'}`;
18
- }
19
- const parsed = Date.parse(`${local}${zone}`);
20
- return Number.isNaN(parsed) ? null : parsed;
21
- }
22
- export function parseIsoInstant(value) {
23
- const dateTime = parseIsoDateTime(value);
24
- if (dateTime !== null)
25
- return dateTime;
26
- const match = ISO_DATE.exec(value);
27
- if (!match || !validDate(Number(match[1]), Number(match[2]), Number(match[3])))
28
- return null;
29
- return Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]));
30
- }
31
- function kindOf(value) {
32
- if (value === null || value === undefined)
33
- return null;
34
- if (typeof value === 'number' || typeof value === 'bigint')
35
- return 'number';
36
- if (typeof value === 'boolean')
37
- return 'boolean';
38
- if (typeof value === 'string')
39
- return parseIsoDateTime(value) !== null ? 'timestamp' : 'string';
40
- return 'json';
41
- }
42
- export function inferRecordSchema(rows) {
43
- const seen = new Map();
44
- for (const row of rows) {
45
- for (const [column, value] of Object.entries(row)) {
46
- const kinds = seen.get(column) ?? new Set();
47
- const kind = kindOf(value);
48
- if (kind)
49
- kinds.add(kind);
50
- seen.set(column, kinds);
51
- }
52
- }
53
- const schema = {};
54
- for (const [column, kinds] of seen) {
55
- if (kinds.size === 0)
56
- schema[column] = 'null';
57
- else if (kinds.size === 1)
58
- schema[column] = [...kinds][0] ?? 'json';
59
- else if (kinds.size === 2 && kinds.has('string') && kinds.has('timestamp'))
60
- schema[column] = 'string';
61
- else
62
- schema[column] = 'json';
63
- }
64
- return schema;
65
- }
@@ -1,19 +0,0 @@
1
- export interface LoadedSource {
2
- name: string;
3
- version: string;
4
- /** CommonJS 打包串,交给 guest 会话的 `execute.code`。 */
5
- code: string;
6
- /** 索引映射形式的 sourcemap(JSON 文本),`sources` 为相对源码根目录的路径。 */
7
- sourcemap: string;
8
- /** 源码文件(相对路径 → 文本,不含 `fixtures/`)。 */
9
- files: Record<string, string>;
10
- /** `dependencies` 里 SDK 的写法(版本范围或 `file:` 路径),由调用方按共享规则校验。 */
11
- sdkRange: string;
12
- }
13
- /** 与 shared `connector-limits.ts` 同值。 */
14
- export declare const LOCAL_SOURCE_MAX_BYTES: number;
15
- export declare const LOCAL_SOURCE_MAX_FILES = 200;
16
- /** 读取并转译一个连接器源码目录。`options.typescript` 是项目自己的 TypeScript 编译器 API(`import ts from 'typescript'`)。 */
17
- export declare function loadConnectorSource(dir: string, options: {
18
- typescript: unknown;
19
- }): Promise<LoadedSource>;
@@ -1,238 +0,0 @@
1
- import { lstat, readdir, readFile } from 'node:fs/promises';
2
- import path from 'node:path';
3
- import { asTypeScript, classifySpecifier, collectRequires, collectSourceSpecifiers, CONNECTOR_SDK_SPECIFIER, findForbiddenSourceSyntax, } from './import-guard.js';
4
- import { SourceLoadError } from './types.js';
5
- export const LOCAL_SOURCE_MAX_BYTES = 2 * 1024 * 1024;
6
- export const LOCAL_SOURCE_MAX_FILES = 200;
7
- const EXCLUDED_DIRECTORIES = new Set(['node_modules', '.git', 'dist', '.ontos', '.datasense']);
8
- const FIXTURES_DIRECTORY = 'fixtures';
9
- const DEFAULT_MAIN = 'src/index.ts';
10
- const NAME_PATTERN = /^[a-z][a-z0-9_]{0,27}$/;
11
- const VERSION_PATTERN = /^\d+\.\d+\.\d+$/;
12
- const CODE_FILE = /\.[cm]?[jt]sx?$/;
13
- const RESOLVE_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.json'];
14
- function isRecord(value) {
15
- return typeof value === 'object' && value !== null && !Array.isArray(value);
16
- }
17
- async function collectFiles(root) {
18
- const files = {};
19
- let count = 0;
20
- let bytes = 0;
21
- const visit = async (absolute, relative) => {
22
- const entries = await readdir(absolute, { withFileTypes: true });
23
- entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
24
- for (const entry of entries) {
25
- const childRelative = relative ? `${relative}/${entry.name}` : entry.name;
26
- const excluded = EXCLUDED_DIRECTORIES.has(entry.name) || (relative === '' && entry.name === FIXTURES_DIRECTORY);
27
- if (excluded && (entry.isDirectory() || entry.isSymbolicLink()))
28
- continue;
29
- if (entry.isSymbolicLink()) {
30
- throw new SourceLoadError('SYMLINK', `symbolic links are not supported in connector source: ${childRelative}`, childRelative);
31
- }
32
- if (entry.isDirectory()) {
33
- await visit(path.join(absolute, entry.name), childRelative);
34
- }
35
- else if (entry.isFile()) {
36
- count += 1;
37
- if (count > LOCAL_SOURCE_MAX_FILES) {
38
- throw new SourceLoadError('TOO_LARGE', `connector source has more than ${LOCAL_SOURCE_MAX_FILES} files`);
39
- }
40
- const text = await readFile(path.join(absolute, entry.name), 'utf8');
41
- bytes += Buffer.byteLength(text, 'utf8');
42
- if (bytes > LOCAL_SOURCE_MAX_BYTES) {
43
- throw new SourceLoadError('TOO_LARGE', `connector source exceeds ${LOCAL_SOURCE_MAX_BYTES} bytes`);
44
- }
45
- files[childRelative] = text;
46
- }
47
- }
48
- };
49
- await visit(root, '');
50
- return files;
51
- }
52
- function readManifest(files) {
53
- const text = files['package.json'];
54
- if (text === undefined)
55
- throw new SourceLoadError('MANIFEST_INVALID', 'package.json is missing', 'package.json');
56
- let pkg;
57
- try {
58
- pkg = JSON.parse(text);
59
- }
60
- catch {
61
- throw new SourceLoadError('MANIFEST_INVALID', 'package.json is not valid JSON', 'package.json');
62
- }
63
- if (!isRecord(pkg))
64
- throw new SourceLoadError('MANIFEST_INVALID', 'package.json must be a JSON object', 'package.json');
65
- const invalid = (message) => new SourceLoadError('MANIFEST_INVALID', message, 'package.json');
66
- if (typeof pkg.name !== 'string' || !NAME_PATTERN.test(pkg.name))
67
- throw invalid(`name must match ${NAME_PATTERN.source}`);
68
- if (typeof pkg.version !== 'string' || !VERSION_PATTERN.test(pkg.version))
69
- throw invalid('version must be a semver string like 1.0.0');
70
- let main = DEFAULT_MAIN;
71
- if (pkg.main !== undefined) {
72
- if (typeof pkg.main !== 'string')
73
- throw invalid('main must be a string');
74
- main = path.posix.normalize(pkg.main.startsWith('./') ? pkg.main.slice(2) : pkg.main);
75
- if (main.startsWith('/') || main === '..' || main.startsWith('../')) {
76
- throw new SourceLoadError('PATH_ESCAPE', `main escapes the connector source root: ${pkg.main}`, 'package.json');
77
- }
78
- }
79
- if (!CODE_FILE.test(main))
80
- throw invalid(`main must be a .ts or .js file: ${main}`);
81
- if (files[main] === undefined)
82
- throw invalid(`entry file not found: ${main}`);
83
- const dependencies = pkg.dependencies;
84
- if (!isRecord(dependencies))
85
- throw invalid(`dependencies must list ${CONNECTOR_SDK_SPECIFIER}`);
86
- for (const dependency of Object.keys(dependencies)) {
87
- if (dependency !== CONNECTOR_SDK_SPECIFIER)
88
- throw invalid(`connectors can only depend on ${CONNECTOR_SDK_SPECIFIER}, found ${dependency}`);
89
- }
90
- const sdkRange = dependencies[CONNECTOR_SDK_SPECIFIER];
91
- if (typeof sdkRange !== 'string' || sdkRange.length === 0)
92
- throw invalid(`dependencies must list ${CONNECTOR_SDK_SPECIFIER}`);
93
- return { name: pkg.name, version: pkg.version, main, sdkRange };
94
- }
95
- function resolveModule(files, target) {
96
- const base = target === '.' ? 'index' : target;
97
- const candidates = [base, ...RESOLVE_EXTENSIONS.map((ext) => `${base}${ext}`), ...RESOLVE_EXTENSIONS.map((ext) => `${base}/index${ext}`)];
98
- for (const candidate of candidates) {
99
- if (Object.hasOwn(files, candidate) && (CODE_FILE.test(candidate) || candidate.endsWith('.json')))
100
- return candidate;
101
- }
102
- return null;
103
- }
104
- const SOURCE_MAP_COMMENT = /\n?\/\/# sourceMappingURL=[^\n]*\s*$/;
105
- function transpile(ts, id, content) {
106
- const output = ts.transpileModule(content, {
107
- fileName: id,
108
- reportDiagnostics: false,
109
- compilerOptions: {
110
- module: ts.ModuleKind.CommonJS,
111
- target: ts.ScriptTarget.ES2022,
112
- esModuleInterop: true,
113
- sourceMap: true,
114
- inlineSourceMap: false,
115
- inlineSources: true,
116
- jsx: ts.JsxEmit.React,
117
- },
118
- });
119
- const code = output.outputText.replace(SOURCE_MAP_COMMENT, '');
120
- let map = { version: 3, sources: [id], names: [], mappings: '' };
121
- if (output.sourceMapText) {
122
- const parsed = JSON.parse(output.sourceMapText);
123
- map = { ...parsed, sources: [id], sourceRoot: '' };
124
- delete map.file;
125
- }
126
- return { code, map };
127
- }
128
- function loadModule(ts, files, id) {
129
- const content = files[id] ?? '';
130
- if (id.endsWith('.json')) {
131
- try {
132
- JSON.parse(content);
133
- }
134
- catch (error) {
135
- throw new SourceLoadError('SYNTAX_ERROR', `${id} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, id);
136
- }
137
- return { id, body: `module.exports = JSON.parse(${JSON.stringify(content)});`, map: null, requires: {} };
138
- }
139
- const problems = findForbiddenSourceSyntax(ts, id, content);
140
- const syntax = problems.find((problem) => problem.kind === 'syntax');
141
- if (syntax)
142
- throw new SourceLoadError('SYNTAX_ERROR', `${id}:${syntax.line}:${syntax.column + 1} ${syntax.message}`, id);
143
- const forbidden = problems[0];
144
- if (forbidden)
145
- throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}:${forbidden.line}:${forbidden.column + 1} ${forbidden.message}`, id);
146
- for (const entry of collectSourceSpecifiers(ts, id, content)) {
147
- const kind = classifySpecifier(entry.specifier, id);
148
- if (kind.kind === 'escape')
149
- throw new SourceLoadError('PATH_ESCAPE', `${id}:${entry.line} import '${entry.specifier}' escapes the connector source root`, id);
150
- if (kind.kind === 'forbidden')
151
- throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}:${entry.line} ${kind.message}`, id);
152
- }
153
- const { code, map } = transpile(ts, id, content);
154
- const requires = {};
155
- for (const entry of collectRequires(ts, id, code)) {
156
- if (entry.specifier === null) {
157
- throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}: imports must use a literal path (computed require / import() is not allowed)`, id);
158
- }
159
- const specifier = entry.specifier;
160
- if (Object.hasOwn(requires, specifier))
161
- continue;
162
- const kind = classifySpecifier(specifier, id);
163
- switch (kind.kind) {
164
- case 'sdk':
165
- requires[specifier] = null;
166
- break;
167
- case 'escape':
168
- throw new SourceLoadError('PATH_ESCAPE', `${id}: import '${specifier}' escapes the connector source root`, id);
169
- case 'forbidden':
170
- throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}: ${kind.message}`, id);
171
- case 'relative': {
172
- const resolved = resolveModule(files, kind.target);
173
- if (resolved === null)
174
- throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}: cannot resolve import '${specifier}'`, id);
175
- requires[specifier] = resolved;
176
- break;
177
- }
178
- }
179
- }
180
- return { id, body: code, map, requires };
181
- }
182
- function countLines(text) {
183
- let lines = 0;
184
- for (let i = 0; i < text.length; i += 1)
185
- if (text.charCodeAt(i) === 10)
186
- lines += 1;
187
- return lines;
188
- }
189
- function assemble(modules, entry) {
190
- const lines = ['var __ontosModules = {'];
191
- const sections = [];
192
- let line = 1;
193
- for (const module of modules) {
194
- lines.push(`${JSON.stringify(module.id)}: [${JSON.stringify(module.requires)}, function (module, exports, require) {`);
195
- line += 1;
196
- if (module.map)
197
- sections.push({ offset: { line, column: 0 }, map: module.map });
198
- const body = module.body.endsWith('\n') ? module.body.slice(0, -1) : module.body;
199
- lines.push(body);
200
- line += countLines(body) + 1;
201
- lines.push('}],');
202
- line += 1;
203
- }
204
- lines.push('};', 'var __ontosCache = {};', 'function __ontosRequire(id) {', ' var cached = __ontosCache[id];', ' if (cached) return cached.exports;', ' var entry = __ontosModules[id];', ' var mod = { exports: {} };', ' __ontosCache[id] = mod;', ' var table = entry[0];', ' entry[1].call(mod.exports, mod, mod.exports, function (specifier) {', ' if (!Object.prototype.hasOwnProperty.call(table, specifier)) throw new Error("cannot require " + String(specifier) + " from " + id);', ' var target = table[specifier];', ' return target === null ? require(specifier) : __ontosRequire(target);', ' });', ' return mod.exports;', '}', `module.exports = __ontosRequire(${JSON.stringify(entry)});`);
205
- return {
206
- code: lines.join('\n'),
207
- sourcemap: JSON.stringify({ version: 3, file: 'connector.js', sections }),
208
- };
209
- }
210
- export async function loadConnectorSource(dir, options) {
211
- const ts = asTypeScript(options.typescript);
212
- const root = path.resolve(dir);
213
- const stat = await lstat(root).catch(() => null);
214
- if (!stat)
215
- throw new SourceLoadError('MANIFEST_INVALID', `connector source directory not found: ${dir}`);
216
- if (stat.isSymbolicLink())
217
- throw new SourceLoadError('SYMLINK', `the connector source directory is a symbolic link: ${dir}`);
218
- if (!stat.isDirectory())
219
- throw new SourceLoadError('MANIFEST_INVALID', `not a directory: ${dir}`);
220
- const files = await collectFiles(root);
221
- const manifest = readManifest(files);
222
- const modules = [];
223
- const seen = new Set();
224
- const queue = [manifest.main];
225
- while (queue.length > 0) {
226
- const id = queue.shift();
227
- if (seen.has(id))
228
- continue;
229
- seen.add(id);
230
- const module = loadModule(ts, files, id);
231
- modules.push(module);
232
- for (const target of Object.values(module.requires))
233
- if (target !== null && !seen.has(target))
234
- queue.push(target);
235
- }
236
- const { code, sourcemap } = assemble(modules, manifest.main);
237
- return { name: manifest.name, version: manifest.version, code, sourcemap, files, sdkRange: manifest.sdkRange };
238
- }