@deepexi/datasense-connector 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -139
- package/package.json +19 -28
- package/scripts/build.ts +188 -0
- package/src/define.ts +267 -0
- package/src/descriptor.ts +116 -0
- package/src/errors.ts +105 -0
- package/src/index.ts +108 -0
- package/src/internal/location.ts +23 -0
- package/src/runtime/guest/bridge.ts +238 -0
- package/src/runtime/guest/cache.ts +43 -0
- package/src/runtime/guest/check.ts +17 -0
- package/src/runtime/guest/crypto.ts +166 -0
- package/src/runtime/guest/ctx.ts +157 -0
- package/src/runtime/guest/db.ts +59 -0
- package/src/runtime/guest/failure.ts +69 -0
- package/src/runtime/guest/http.ts +194 -0
- package/src/runtime/guest/index.ts +102 -0
- package/{runtime/guest/limits.d.ts → src/runtime/guest/limits.ts} +12 -12
- package/src/runtime/guest/loader.ts +210 -0
- package/src/runtime/guest/log.ts +47 -0
- package/src/runtime/guest/map.ts +45 -0
- package/src/runtime/guest/operation.ts +193 -0
- package/src/runtime/guest/paginate.ts +169 -0
- package/src/runtime/guest/pull.ts +177 -0
- package/src/runtime/guest/push.ts +92 -0
- package/src/runtime/guest/signal.ts +14 -0
- package/src/runtime/guest/suggest.ts +51 -0
- package/src/runtime/guest/time.ts +137 -0
- package/src/runtime/guest-runtime.ts +75 -0
- package/src/runtime/index.ts +53 -0
- package/src/runtime/protocol.ts +557 -0
- package/src/runtime/session.ts +216 -0
- package/src/testing/child.ts +21 -0
- package/src/testing/compliance.ts +343 -0
- package/src/testing/engine.ts +445 -0
- package/src/testing/fixtures.ts +591 -0
- package/src/testing/gateway.ts +482 -0
- package/src/testing/import-guard.ts +221 -0
- package/src/testing/index.ts +43 -0
- package/src/testing/local-db.ts +226 -0
- package/src/testing/mock-external-system.ts +249 -0
- package/src/testing/pinned-http.ts +144 -0
- package/src/testing/recorder.ts +140 -0
- package/src/testing/redaction.ts +94 -0
- package/src/testing/replay.ts +437 -0
- package/src/testing/run-tests.ts +115 -0
- package/src/testing/schema.ts +70 -0
- package/src/testing/source-loader.ts +374 -0
- package/src/testing/types.ts +78 -0
- package/src/types.ts +329 -0
- package/src/validate.ts +267 -0
- package/test/build.test.ts +159 -0
- package/test/define.test-d.ts +182 -0
- package/test/fixtures/acceptance.ts +156 -0
- package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
- package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
- package/test/fixtures/connectors/http-pull/package.json +5 -0
- package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
- package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
- package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
- package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
- package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
- package/test/fixtures/connectors/pg-pull/package.json +5 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
- package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
- package/test/fixtures/descriptor.ts +54 -0
- package/test/fixtures/kingdee-example.ts +82 -0
- package/test/fixtures/record-acceptance.ts +32 -0
- package/test/runtime/ctx.test.ts +434 -0
- package/test/runtime/helpers.ts +157 -0
- package/test/runtime/jobs.test.ts +444 -0
- package/test/runtime/operation.test.ts +175 -0
- package/test/runtime/session.test.ts +229 -0
- package/test/testing/acceptance-fixtures.test.ts +69 -0
- package/test/testing/compliance.test.ts +247 -0
- package/test/testing/engine.test.ts +283 -0
- package/test/testing/fixtures.test.ts +221 -0
- package/test/testing/gateway.test.ts +337 -0
- package/test/testing/import-guard.test.ts +45 -0
- package/test/testing/mock-external-system.test.ts +144 -0
- package/test/testing/operation-fixtures.test.ts +160 -0
- package/test/testing/recorder.test.ts +163 -0
- package/test/testing/redaction.test.ts +54 -0
- package/test/testing/replay.test.ts +207 -0
- package/test/testing/run-tests.test.ts +137 -0
- package/test/testing/source-loader.test.ts +220 -0
- package/test/validate-config.test.ts +104 -0
- package/tsconfig.build.json +23 -0
- package/tsconfig.json +19 -0
- package/cjs/index.cjs +0 -1015
- package/cjs/runtime.cjs +0 -752
- package/cjs/testing.cjs +0 -3591
- package/errors.d.ts +0 -17
- package/errors.js +0 -53
- package/guest.cjs +0 -2659
- package/index.d.ts +0 -13
- package/index.js +0 -7
- package/internal/location.d.ts +0 -6
- package/internal/location.js +0 -8
- package/presentation.d.ts +0 -38
- package/presentation.js +0 -268
- package/runtime/guest/bridge.d.ts +0 -53
- package/runtime/guest/bridge.js +0 -187
- package/runtime/guest/cache.d.ts +0 -3
- package/runtime/guest/cache.js +0 -42
- package/runtime/guest/crypto.d.ts +0 -13
- package/runtime/guest/crypto.js +0 -177
- package/runtime/guest/ctx.d.ts +0 -30
- package/runtime/guest/ctx.js +0 -124
- package/runtime/guest/db.d.ts +0 -6
- package/runtime/guest/db.js +0 -59
- package/runtime/guest/discover.d.ts +0 -29
- package/runtime/guest/discover.js +0 -51
- package/runtime/guest/failure.d.ts +0 -24
- package/runtime/guest/failure.js +0 -63
- package/runtime/guest/http.d.ts +0 -4
- package/runtime/guest/http.js +0 -204
- package/runtime/guest/index.d.ts +0 -12
- package/runtime/guest/index.js +0 -89
- package/runtime/guest/limits.js +0 -12
- package/runtime/guest/loader.d.ts +0 -15
- package/runtime/guest/loader.js +0 -42
- package/runtime/guest/log.d.ts +0 -10
- package/runtime/guest/log.js +0 -36
- package/runtime/guest/map.d.ts +0 -4
- package/runtime/guest/map.js +0 -45
- package/runtime/guest/paginate.d.ts +0 -6
- package/runtime/guest/paginate.js +0 -180
- package/runtime/guest/pull.d.ts +0 -12
- package/runtime/guest/pull.js +0 -150
- package/runtime/guest/push.d.ts +0 -18
- package/runtime/guest/push.js +0 -80
- package/runtime/guest/signal.d.ts +0 -3
- package/runtime/guest/signal.js +0 -12
- package/runtime/guest/time.d.ts +0 -5
- package/runtime/guest/time.js +0 -127
- package/runtime/guest-runtime.d.ts +0 -7
- package/runtime/guest-runtime.js +0 -51
- package/runtime/index.d.ts +0 -18
- package/runtime/index.js +0 -3
- package/runtime/protocol.d.ts +0 -276
- package/runtime/protocol.js +0 -409
- package/runtime/session.d.ts +0 -12
- package/runtime/session.js +0 -191
- package/t.d.ts +0 -18
- package/t.js +0 -26
- package/testing/child.cjs +0 -350
- package/testing/child.d.ts +0 -1
- package/testing/child.js +0 -12
- package/testing/compliance.d.ts +0 -41
- package/testing/compliance.js +0 -291
- package/testing/engine.d.ts +0 -66
- package/testing/engine.js +0 -339
- package/testing/fixtures.d.ts +0 -145
- package/testing/fixtures.js +0 -407
- package/testing/gateway.d.ts +0 -65
- package/testing/gateway.js +0 -402
- package/testing/import-guard.d.ts +0 -57
- package/testing/import-guard.js +0 -187
- package/testing/index.d.ts +0 -33
- package/testing/index.js +0 -10
- package/testing/local-db.d.ts +0 -27
- package/testing/local-db.js +0 -181
- package/testing/pinned-http.d.ts +0 -14
- package/testing/pinned-http.js +0 -125
- package/testing/recorder.d.ts +0 -15
- package/testing/recorder.js +0 -116
- package/testing/redaction.d.ts +0 -28
- package/testing/redaction.js +0 -67
- package/testing/replay.d.ts +0 -50
- package/testing/replay.js +0 -313
- package/testing/schema.d.ts +0 -13
- package/testing/schema.js +0 -65
- package/testing/source-loader.d.ts +0 -19
- package/testing/source-loader.js +0 -238
- package/testing/types.d.ts +0 -79
- package/testing/types.js +0 -18
- package/types.d.ts +0 -302
- package/types.js +0 -13
- package/validate.d.ts +0 -78
- package/validate.js +0 -597
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 本地加载连接器:读 `.ontos/descriptor.json`(codegen 生成),按虚拟入口契约把源码目录转译成一个 CommonJS 打包串
|
|
3
|
+
* (给 `createVmExecutor`,与 runner 同一个 guest 会话执行),并收集能力目录下的测试用例文件。
|
|
4
|
+
*
|
|
5
|
+
* - 只用调用方传入的 TypeScript 编译器 API 做语法转译(`transpileModule`:CommonJS、ES2022、单独产 sourcemap),
|
|
6
|
+
* 不执行任何构建期插件,不依赖打包器,也不依赖 shared;
|
|
7
|
+
* - 入口由描述生成(与平台的虚拟入口同一结构):导出 `descriptor` 与 `implementations`,后者按目录约定引用
|
|
8
|
+
* `check.ts`、`streams/<名>/index.ts`、`streams/<名>/suggest-columns.ts`、`pushes/<名>/index.ts`、`operations/<名>/index.ts`
|
|
9
|
+
* 的默认导出;文件不存在时不放进 `implementations`,由 guest 加载时报出缺失;
|
|
10
|
+
* - 从这些文件起按 `require` 关系收集模块,每个模块先过 `import-guard.ts` 的检查(与服务端打包同一套规则):
|
|
11
|
+
* 只允许相对路径与 `@deepexi/datasense-connector`;
|
|
12
|
+
* - 打包串是一张模块表加一个按预先解析好的相对路径查表的 `require`;SDK 说明符留给 guest 的 `require`(vm 里由 guest 提供);
|
|
13
|
+
* - sourcemap 是按模块分段的索引映射(`sections`),行号与打包串一一对应(guest 把打包串包进函数时与第一行同行)。
|
|
14
|
+
*
|
|
15
|
+
* 收集源码时跳过 `node_modules`、`.git`、`dist`、`.ontos`、`.datasense` 与能力目录下的 `tests/`(与 CLI 上传一致);
|
|
16
|
+
* 遇到符号链接直接拒绝(`SYMLINK`),源码总量超过 2 MiB 或文件数超过 200 拒绝(`TOO_LARGE`)。
|
|
17
|
+
*/
|
|
18
|
+
import { lstat, readdir, readFile } from 'node:fs/promises';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import type * as TS from 'typescript';
|
|
21
|
+
import type { ConnectorDescriptor } from '../descriptor';
|
|
22
|
+
import { OPERATION_NAME_PATTERN, SDK_NAME_PATTERN } from '../validate';
|
|
23
|
+
import { readFixtureDir, type ConnectorFixtureFile } from './fixtures';
|
|
24
|
+
import {
|
|
25
|
+
asTypeScript, classifySpecifier, collectRequires, collectSourceSpecifiers, CONNECTOR_SDK_SPECIFIER, findForbiddenSourceSyntax, type TypeScriptApi,
|
|
26
|
+
} from './import-guard';
|
|
27
|
+
import { SourceLoadError } from './types';
|
|
28
|
+
|
|
29
|
+
/** 本地加载好的连接器。 */
|
|
30
|
+
export interface LocalConnector {
|
|
31
|
+
/** 源码根目录(绝对路径)。 */
|
|
32
|
+
dir: string;
|
|
33
|
+
name: string;
|
|
34
|
+
version: string;
|
|
35
|
+
/** `.ontos/descriptor.json` 的内容。 */
|
|
36
|
+
descriptor: ConnectorDescriptor;
|
|
37
|
+
/** CommonJS 打包串,交给 guest 会话的 `execute.code`;导出 `{ descriptor, implementations }`。 */
|
|
38
|
+
code: string;
|
|
39
|
+
/** 索引映射形式的 sourcemap(JSON 文本),`sources` 为相对源码根目录的路径。 */
|
|
40
|
+
sourcemap: string;
|
|
41
|
+
/** 源码文件(相对路径 → 文本,不含 `tests/` 与 `.ontos/`)。 */
|
|
42
|
+
files: Record<string, string>;
|
|
43
|
+
/** 能力目录下 `tests/` 里的全部文件(路径相对源码根目录,按路径排序)。 */
|
|
44
|
+
testFiles: ConnectorFixtureFile[];
|
|
45
|
+
/** `dependencies` 里 SDK 的写法(版本范围或 `file:` 路径),由调用方按共享规则校验。 */
|
|
46
|
+
sdkRange: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** 与 shared `connector-limits.ts` 同值。 */
|
|
50
|
+
export const LOCAL_SOURCE_MAX_BYTES = 2 * 1024 * 1024;
|
|
51
|
+
export const LOCAL_SOURCE_MAX_FILES = 200;
|
|
52
|
+
|
|
53
|
+
export const DESCRIPTOR_FILE = '.ontos/descriptor.json';
|
|
54
|
+
/** codegen 生成的类型与已绑定类型的编写函数;源码可以 `import` 它,加载时并入模块表(不算源码文件,不上传)。 */
|
|
55
|
+
export const GENERATED_TYPES_FILE = '.ontos/types.ts';
|
|
56
|
+
|
|
57
|
+
const EXCLUDED_DIRECTORIES: ReadonlySet<string> = new Set(['node_modules', '.git', 'dist', '.ontos', '.datasense']);
|
|
58
|
+
const CAPABILITY_GROUPS: ReadonlySet<string> = new Set(['streams', 'pushes', 'operations']);
|
|
59
|
+
const TESTS_DIRECTORY = 'tests';
|
|
60
|
+
const CODE_FILE = /\.[cm]?[jt]sx?$/;
|
|
61
|
+
const RESOLVE_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.json'];
|
|
62
|
+
const ENTRY_ID = '.ontos/entry.js';
|
|
63
|
+
|
|
64
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
65
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** `<streams|pushes|operations>/<名>/tests` 目录(测试用例,不算源码)。 */
|
|
69
|
+
function isTestsDirectory(relative: string): boolean {
|
|
70
|
+
const parts = relative.split('/');
|
|
71
|
+
return parts.length === 3 && CAPABILITY_GROUPS.has(parts[0] ?? '') && parts[2] === TESTS_DIRECTORY;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function collectFiles(root: string): Promise<Record<string, string>> {
|
|
75
|
+
const files: Record<string, string> = {};
|
|
76
|
+
let count = 0;
|
|
77
|
+
let bytes = 0;
|
|
78
|
+
const visit = async (absolute: string, relative: string): Promise<void> => {
|
|
79
|
+
const entries = await readdir(absolute, { withFileTypes: true });
|
|
80
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
81
|
+
for (const entry of entries) {
|
|
82
|
+
const childRelative = relative ? `${relative}/${entry.name}` : entry.name;
|
|
83
|
+
// 排除目录先判:`node_modules` 等本身是符号链接(pnpm、monorepo 链接)时也直接跳过,不报 SYMLINK。
|
|
84
|
+
const excluded = EXCLUDED_DIRECTORIES.has(entry.name) || isTestsDirectory(childRelative);
|
|
85
|
+
if (excluded && (entry.isDirectory() || entry.isSymbolicLink())) continue;
|
|
86
|
+
if (entry.isSymbolicLink()) {
|
|
87
|
+
throw new SourceLoadError('SYMLINK', `symbolic links are not supported in connector source: ${childRelative}`, childRelative);
|
|
88
|
+
}
|
|
89
|
+
if (entry.isDirectory()) {
|
|
90
|
+
await visit(path.join(absolute, entry.name), childRelative);
|
|
91
|
+
} else if (entry.isFile()) {
|
|
92
|
+
count += 1;
|
|
93
|
+
if (count > LOCAL_SOURCE_MAX_FILES) {
|
|
94
|
+
throw new SourceLoadError('TOO_LARGE', `connector source has more than ${LOCAL_SOURCE_MAX_FILES} files`);
|
|
95
|
+
}
|
|
96
|
+
const text = await readFile(path.join(absolute, entry.name), 'utf8');
|
|
97
|
+
bytes += Buffer.byteLength(text, 'utf8');
|
|
98
|
+
if (bytes > LOCAL_SOURCE_MAX_BYTES) {
|
|
99
|
+
throw new SourceLoadError('TOO_LARGE', `connector source exceeds ${LOCAL_SOURCE_MAX_BYTES} bytes`);
|
|
100
|
+
}
|
|
101
|
+
files[childRelative] = text;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
await visit(root, '');
|
|
106
|
+
return files;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** `package.json` 只读 SDK 依赖(其余字段的规则由平台校验)。 */
|
|
110
|
+
function readSdkRange(files: Record<string, string>): string {
|
|
111
|
+
const text = files['package.json'];
|
|
112
|
+
if (text === undefined) throw new SourceLoadError('MANIFEST_INVALID', 'package.json is missing', 'package.json');
|
|
113
|
+
let pkg: unknown;
|
|
114
|
+
try {
|
|
115
|
+
pkg = JSON.parse(text);
|
|
116
|
+
} catch {
|
|
117
|
+
throw new SourceLoadError('MANIFEST_INVALID', 'package.json is not valid JSON', 'package.json');
|
|
118
|
+
}
|
|
119
|
+
if (!isRecord(pkg)) throw new SourceLoadError('MANIFEST_INVALID', 'package.json must be a JSON object', 'package.json');
|
|
120
|
+
const invalid = (message: string) => new SourceLoadError('MANIFEST_INVALID', message, 'package.json');
|
|
121
|
+
const dependencies = pkg.dependencies;
|
|
122
|
+
if (!isRecord(dependencies)) throw invalid(`dependencies must list ${CONNECTOR_SDK_SPECIFIER}`);
|
|
123
|
+
for (const dependency of Object.keys(dependencies)) {
|
|
124
|
+
if (dependency !== CONNECTOR_SDK_SPECIFIER) throw invalid(`connectors can only depend on ${CONNECTOR_SDK_SPECIFIER}, found ${dependency}`);
|
|
125
|
+
}
|
|
126
|
+
const sdkRange = dependencies[CONNECTOR_SDK_SPECIFIER];
|
|
127
|
+
if (typeof sdkRange !== 'string' || sdkRange.length === 0) throw invalid(`dependencies must list ${CONNECTOR_SDK_SPECIFIER}`);
|
|
128
|
+
return sdkRange;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** 读 `.ontos/types.ts`(没有时为 null;符号链接或非普通文件拒绝)。 */
|
|
132
|
+
async function readGeneratedTypes(root: string): Promise<string | null> {
|
|
133
|
+
const file = path.join(root, GENERATED_TYPES_FILE);
|
|
134
|
+
const stat = await lstat(file).catch(() => null);
|
|
135
|
+
if (!stat) return null;
|
|
136
|
+
if (stat.isSymbolicLink()) throw new SourceLoadError('SYMLINK', `symbolic links are not supported in connector source: ${GENERATED_TYPES_FILE}`, GENERATED_TYPES_FILE);
|
|
137
|
+
if (!stat.isFile()) throw new SourceLoadError('DESCRIPTOR_INVALID', `${GENERATED_TYPES_FILE} must be a regular file; ${REGENERATE_HINT}`, GENERATED_TYPES_FILE);
|
|
138
|
+
return readFile(file, 'utf8');
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const REGENERATE_HINT = 'run `ontos admin connector codegen` to regenerate it';
|
|
142
|
+
|
|
143
|
+
/** 读 `.ontos/descriptor.json`:名字会拼进模块路径,所以先按名字规则检查。 */
|
|
144
|
+
async function readDescriptor(root: string): Promise<ConnectorDescriptor> {
|
|
145
|
+
const file = path.join(root, DESCRIPTOR_FILE);
|
|
146
|
+
const stat = await lstat(file).catch(() => null);
|
|
147
|
+
if (!stat) {
|
|
148
|
+
throw new SourceLoadError('DESCRIPTOR_NOT_GENERATED', `${DESCRIPTOR_FILE} not found; run \`ontos admin connector codegen\` first`, DESCRIPTOR_FILE);
|
|
149
|
+
}
|
|
150
|
+
if (stat.isSymbolicLink() || !stat.isFile()) throw new SourceLoadError('DESCRIPTOR_INVALID', `${DESCRIPTOR_FILE} must be a regular file; ${REGENERATE_HINT}`, DESCRIPTOR_FILE);
|
|
151
|
+
let value: unknown;
|
|
152
|
+
try {
|
|
153
|
+
value = JSON.parse(await readFile(file, 'utf8'));
|
|
154
|
+
} catch {
|
|
155
|
+
throw new SourceLoadError('DESCRIPTOR_INVALID', `${DESCRIPTOR_FILE} is not valid JSON; ${REGENERATE_HINT}`, DESCRIPTOR_FILE);
|
|
156
|
+
}
|
|
157
|
+
const invalid = (message: string) => new SourceLoadError('DESCRIPTOR_INVALID', `${DESCRIPTOR_FILE}: ${message}; ${REGENERATE_HINT}`, DESCRIPTOR_FILE);
|
|
158
|
+
if (!isRecord(value) || typeof value.name !== 'string' || typeof value.version !== 'string') throw invalid('name and version are required');
|
|
159
|
+
for (const key of ['config', 'streams', 'pushes', 'operations'] as const) {
|
|
160
|
+
if (!isRecord(value[key])) throw invalid(`${key} must be an object`);
|
|
161
|
+
}
|
|
162
|
+
for (const key of ['streams', 'pushes'] as const) {
|
|
163
|
+
for (const name of Object.keys(value[key] as Record<string, unknown>)) {
|
|
164
|
+
if (!SDK_NAME_PATTERN.test(name)) throw invalid(`${key} name ${JSON.stringify(name)} must match ${SDK_NAME_PATTERN.source}`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
for (const name of Object.keys(value.operations as Record<string, unknown>)) {
|
|
168
|
+
if (!OPERATION_NAME_PATTERN.test(name)) throw invalid(`operation name ${JSON.stringify(name)} must match ${OPERATION_NAME_PATTERN.source}`);
|
|
169
|
+
}
|
|
170
|
+
return value as unknown as ConnectorDescriptor;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function resolveModule(files: Record<string, string>, target: string): string | null {
|
|
174
|
+
const base = target === '.' ? 'index' : target;
|
|
175
|
+
const candidates = [base, ...RESOLVE_EXTENSIONS.map((ext) => `${base}${ext}`), ...RESOLVE_EXTENSIONS.map((ext) => `${base}/index${ext}`)];
|
|
176
|
+
for (const candidate of candidates) {
|
|
177
|
+
if (Object.hasOwn(files, candidate) && (CODE_FILE.test(candidate) || candidate.endsWith('.json'))) return candidate;
|
|
178
|
+
}
|
|
179
|
+
return null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
interface Module {
|
|
183
|
+
id: string;
|
|
184
|
+
/** 模块函数体(已转译的 CommonJS 代码)。 */
|
|
185
|
+
body: string;
|
|
186
|
+
/** 该模块的 sourcemap(v3 对象),JSON 模块没有。 */
|
|
187
|
+
map: Record<string, unknown> | null;
|
|
188
|
+
/** 说明符 → 模块 id;SDK 为 null(交给 guest 的 require)。 */
|
|
189
|
+
requires: Record<string, string | null>;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const SOURCE_MAP_COMMENT = /\n?\/\/# sourceMappingURL=[^\n]*\s*$/;
|
|
193
|
+
|
|
194
|
+
function transpile(ts: TypeScriptApi, id: string, content: string): { code: string; map: Record<string, unknown> } {
|
|
195
|
+
const output = ts.transpileModule(content, {
|
|
196
|
+
fileName: id,
|
|
197
|
+
reportDiagnostics: false,
|
|
198
|
+
compilerOptions: {
|
|
199
|
+
module: ts.ModuleKind.CommonJS,
|
|
200
|
+
target: ts.ScriptTarget.ES2022,
|
|
201
|
+
esModuleInterop: true,
|
|
202
|
+
sourceMap: true,
|
|
203
|
+
inlineSourceMap: false,
|
|
204
|
+
inlineSources: true,
|
|
205
|
+
jsx: ts.JsxEmit.React,
|
|
206
|
+
} satisfies TS.CompilerOptions,
|
|
207
|
+
});
|
|
208
|
+
const code = output.outputText.replace(SOURCE_MAP_COMMENT, '');
|
|
209
|
+
let map: Record<string, unknown> = { version: 3, sources: [id], names: [], mappings: '' };
|
|
210
|
+
if (output.sourceMapText) {
|
|
211
|
+
const parsed = JSON.parse(output.sourceMapText) as Record<string, unknown>;
|
|
212
|
+
map = { ...parsed, sources: [id], sourceRoot: '' };
|
|
213
|
+
delete map.file;
|
|
214
|
+
}
|
|
215
|
+
return { code, map };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function loadModule(ts: TypeScriptApi, files: Record<string, string>, id: string): Module {
|
|
219
|
+
const content = files[id] ?? '';
|
|
220
|
+
if (id.endsWith('.json')) {
|
|
221
|
+
try {
|
|
222
|
+
JSON.parse(content);
|
|
223
|
+
} catch (error) {
|
|
224
|
+
throw new SourceLoadError('SYNTAX_ERROR', `${id} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`, id);
|
|
225
|
+
}
|
|
226
|
+
// 走 JSON.parse 而不是内联成对象字面量:字面量里的 `"__proto__"` 键会改原型,JSON 语义下它只是普通键。
|
|
227
|
+
return { id, body: `module.exports = JSON.parse(${JSON.stringify(content)});`, map: null, requires: {} };
|
|
228
|
+
}
|
|
229
|
+
const problems = findForbiddenSourceSyntax(ts, id, content);
|
|
230
|
+
const syntax = problems.find((problem) => problem.kind === 'syntax');
|
|
231
|
+
if (syntax) throw new SourceLoadError('SYNTAX_ERROR', `${id}:${syntax.line}:${syntax.column + 1} ${syntax.message}`, id);
|
|
232
|
+
const forbidden = problems[0];
|
|
233
|
+
if (forbidden) throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}:${forbidden.line}:${forbidden.column + 1} ${forbidden.message}`, id);
|
|
234
|
+
// 源码层:写了但没用到的导入在转译时会被删掉,这里先按源码归类一遍。
|
|
235
|
+
for (const entry of collectSourceSpecifiers(ts, id, content)) {
|
|
236
|
+
const kind = classifySpecifier(entry.specifier, id);
|
|
237
|
+
if (kind.kind === 'escape') throw new SourceLoadError('PATH_ESCAPE', `${id}:${entry.line} import '${entry.specifier}' escapes the connector source root`, id);
|
|
238
|
+
if (kind.kind === 'forbidden') throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}:${entry.line} ${kind.message}`, id);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const { code, map } = transpile(ts, id, content);
|
|
242
|
+
const requires: Record<string, string | null> = {};
|
|
243
|
+
for (const entry of collectRequires(ts, id, code)) {
|
|
244
|
+
if (entry.specifier === null) {
|
|
245
|
+
throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}: imports must use a literal path (computed require / import() is not allowed)`, id);
|
|
246
|
+
}
|
|
247
|
+
const specifier = entry.specifier;
|
|
248
|
+
if (Object.hasOwn(requires, specifier)) continue;
|
|
249
|
+
const kind = classifySpecifier(specifier, id);
|
|
250
|
+
switch (kind.kind) {
|
|
251
|
+
case 'sdk':
|
|
252
|
+
requires[specifier] = null;
|
|
253
|
+
break;
|
|
254
|
+
case 'escape':
|
|
255
|
+
throw new SourceLoadError('PATH_ESCAPE', `${id}: import '${specifier}' escapes the connector source root`, id);
|
|
256
|
+
case 'forbidden':
|
|
257
|
+
throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}: ${kind.message}`, id);
|
|
258
|
+
case 'relative': {
|
|
259
|
+
const resolved = resolveModule(files, kind.target);
|
|
260
|
+
if (resolved === null) throw new SourceLoadError('IMPORT_FORBIDDEN', `${id}: cannot resolve import '${specifier}'`, id);
|
|
261
|
+
requires[specifier] = resolved;
|
|
262
|
+
break;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
return { id, body: code, map, requires };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
function countLines(text: string): number {
|
|
270
|
+
let lines = 0;
|
|
271
|
+
for (let i = 0; i < text.length; i += 1) if (text.charCodeAt(i) === 10) lines += 1;
|
|
272
|
+
return lines;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** 按虚拟入口契约生成入口模块:描述以 JSON 文本嵌入,实现按目录约定引用(只用校验过的名字拼路径)。 */
|
|
276
|
+
function entryModule(descriptor: ConnectorDescriptor, files: Record<string, string>): Module {
|
|
277
|
+
const requires: Record<string, string | null> = {};
|
|
278
|
+
const ref = (file: string): string | null => {
|
|
279
|
+
if (!Object.hasOwn(files, file)) return null;
|
|
280
|
+
const specifier = `./${file}`;
|
|
281
|
+
requires[specifier] = file;
|
|
282
|
+
return `__ontosDefault(require(${JSON.stringify(specifier)}))`;
|
|
283
|
+
};
|
|
284
|
+
const group = (entries: Array<[string, string | null]>): string =>
|
|
285
|
+
`{ ${entries.filter((entry): entry is [string, string] => entry[1] !== null).map(([name, expression]) => `${JSON.stringify(name)}: ${expression}`).join(', ')} }`;
|
|
286
|
+
const streams = Object.keys(descriptor.streams);
|
|
287
|
+
const check = descriptor.check ? ref('check.ts') : null;
|
|
288
|
+
const lines = [
|
|
289
|
+
'var __ontosDefault = function (m) { return m !== null && typeof m === "object" && Object.prototype.hasOwnProperty.call(m, "default") ? m.default : m; };',
|
|
290
|
+
`var descriptor = JSON.parse(${JSON.stringify(JSON.stringify(descriptor))});`,
|
|
291
|
+
'module.exports = { descriptor: descriptor, implementations: {',
|
|
292
|
+
...(check !== null ? [` check: ${check},`] : []),
|
|
293
|
+
` streams: ${group(streams.map((name) => [name, ref(`streams/${name}/index.ts`)]))},`,
|
|
294
|
+
` suggestColumns: ${group(streams.filter((name) => descriptor.streams[name]?.suggestColumns === true).map((name) => [name, ref(`streams/${name}/suggest-columns.ts`)]))},`,
|
|
295
|
+
` pushes: ${group(Object.keys(descriptor.pushes).map((name) => [name, ref(`pushes/${name}/index.ts`)]))},`,
|
|
296
|
+
` operations: ${group(Object.keys(descriptor.operations).map((name) => [name, ref(`operations/${name}/index.ts`)]))},`,
|
|
297
|
+
'} };',
|
|
298
|
+
];
|
|
299
|
+
return { id: ENTRY_ID, body: lines.join('\n'), map: null, requires };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
/** 模块表 + 查表的 `require`;整段作为 guest 会话里 `(function (module, exports, require) {…})` 的函数体。 */
|
|
303
|
+
function assemble(modules: Module[], entry: string): { code: string; sourcemap: string } {
|
|
304
|
+
const lines: string[] = ['var __ontosModules = {'];
|
|
305
|
+
const sections: Array<{ offset: { line: number; column: number }; map: Record<string, unknown> }> = [];
|
|
306
|
+
let line = 1;
|
|
307
|
+
for (const module of modules) {
|
|
308
|
+
lines.push(`${JSON.stringify(module.id)}: [${JSON.stringify(module.requires)}, function (module, exports, require) {`);
|
|
309
|
+
line += 1;
|
|
310
|
+
if (module.map) sections.push({ offset: { line, column: 0 }, map: module.map });
|
|
311
|
+
const body = module.body.endsWith('\n') ? module.body.slice(0, -1) : module.body;
|
|
312
|
+
lines.push(body);
|
|
313
|
+
line += countLines(body) + 1;
|
|
314
|
+
lines.push('}],');
|
|
315
|
+
line += 1;
|
|
316
|
+
}
|
|
317
|
+
lines.push(
|
|
318
|
+
'};',
|
|
319
|
+
'var __ontosCache = {};',
|
|
320
|
+
'function __ontosRequire(id) {',
|
|
321
|
+
' var cached = __ontosCache[id];',
|
|
322
|
+
' if (cached) return cached.exports;',
|
|
323
|
+
' var entry = __ontosModules[id];',
|
|
324
|
+
' var mod = { exports: {} };',
|
|
325
|
+
' __ontosCache[id] = mod;',
|
|
326
|
+
' var table = entry[0];',
|
|
327
|
+
' entry[1].call(mod.exports, mod, mod.exports, function (specifier) {',
|
|
328
|
+
' if (!Object.prototype.hasOwnProperty.call(table, specifier)) throw new Error("cannot require " + String(specifier) + " from " + id);',
|
|
329
|
+
' var target = table[specifier];',
|
|
330
|
+
' return target === null ? require(specifier) : __ontosRequire(target);',
|
|
331
|
+
' });',
|
|
332
|
+
' return mod.exports;',
|
|
333
|
+
'}',
|
|
334
|
+
`module.exports = __ontosRequire(${JSON.stringify(entry)});`,
|
|
335
|
+
);
|
|
336
|
+
return {
|
|
337
|
+
code: lines.join('\n'),
|
|
338
|
+
sourcemap: JSON.stringify({ version: 3, file: 'connector.js', sections }),
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* 本地加载连接器源码目录(见文件头)。`options.typescript` 是项目自己的 TypeScript 编译器 API(`import ts from 'typescript'`)。
|
|
344
|
+
* 没有 `.ontos/descriptor.json` 时抛 `SourceLoadError('DESCRIPTOR_NOT_GENERATED')`。
|
|
345
|
+
*/
|
|
346
|
+
export async function loadLocalConnector(dir: string, options: { typescript: unknown }): Promise<LocalConnector> {
|
|
347
|
+
const ts = asTypeScript(options.typescript);
|
|
348
|
+
const root = path.resolve(dir);
|
|
349
|
+
const stat = await lstat(root).catch(() => null);
|
|
350
|
+
if (!stat) throw new SourceLoadError('MANIFEST_INVALID', `connector source directory not found: ${dir}`);
|
|
351
|
+
if (stat.isSymbolicLink()) throw new SourceLoadError('SYMLINK', `the connector source directory is a symbolic link: ${dir}`);
|
|
352
|
+
if (!stat.isDirectory()) throw new SourceLoadError('MANIFEST_INVALID', `not a directory: ${dir}`);
|
|
353
|
+
|
|
354
|
+
const descriptor = await readDescriptor(root);
|
|
355
|
+
const files = await collectFiles(root);
|
|
356
|
+
const sdkRange = readSdkRange(files);
|
|
357
|
+
const generated = await readGeneratedTypes(root);
|
|
358
|
+
const sources = generated === null ? files : { ...files, [GENERATED_TYPES_FILE]: generated };
|
|
359
|
+
const entry = entryModule(descriptor, files);
|
|
360
|
+
const modules: Module[] = [entry];
|
|
361
|
+
const seen = new Set<string>([entry.id]);
|
|
362
|
+
const queue = Object.values(entry.requires).filter((target): target is string => target !== null);
|
|
363
|
+
while (queue.length > 0) {
|
|
364
|
+
const id = queue.shift() as string;
|
|
365
|
+
if (seen.has(id)) continue;
|
|
366
|
+
seen.add(id);
|
|
367
|
+
const module = loadModule(ts, sources, id);
|
|
368
|
+
modules.push(module);
|
|
369
|
+
for (const target of Object.values(module.requires)) if (target !== null && !seen.has(target)) queue.push(target);
|
|
370
|
+
}
|
|
371
|
+
const { code, sourcemap } = assemble(modules, entry.id);
|
|
372
|
+
const testFiles = await readFixtureDir(root);
|
|
373
|
+
return { dir: root, name: descriptor.name, version: descriptor.version, descriptor, code, sourcemap, files, testFiles, sdkRange };
|
|
374
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `./testing` 的公共类型与错误类(本地运行、本地模拟网关、源码加载器共用)。
|
|
3
|
+
*/
|
|
4
|
+
import type { ConnectorDescriptor, SuggestedColumn } from '../descriptor';
|
|
5
|
+
import type { GuestJob, GuestOperationResult, HostMethod } from '../runtime/protocol';
|
|
6
|
+
|
|
7
|
+
export type { GuestJob, HostMethod };
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* 宿主:guest 的宿主调用(`http` / `db.*` / `crypto` / `cache.*` / `secret.reveal`)交给它应答。
|
|
11
|
+
* `emit` 由执行器自己处理,不交给宿主。实现有本地模拟网关(`createLocalGateway`)与夹具回放。
|
|
12
|
+
* 出站主机范围由调用方事先从连接器描述算好(`hosts` ∪ `url` 配置项的主机),交给本地网关。
|
|
13
|
+
* `call` 抛出的错误回给 guest 作为宿主调用失败(`code` 取错误的 `code`);抛 `LocalRunError('DB_DRIVER_MISSING')` 时整次运行以该码结束。
|
|
14
|
+
*/
|
|
15
|
+
export interface LocalHost {
|
|
16
|
+
call(method: HostMethod, payload: unknown): Promise<unknown>;
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
/** 可选:遮蔽文本里的 Secret 明文(执行器用它遮蔽结果里的日志与错误文案);本地模拟网关提供。 */
|
|
19
|
+
redact?(text: string): string;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** 本地运行的结果;`error` 非 null 时运行失败(code 为 guest 的失败码,或 `TIMEOUT` / `DB_DRIVER_MISSING` 等本地码)。 */
|
|
23
|
+
export interface LocalRunResult {
|
|
24
|
+
records: Array<{ table: string; row: Record<string, unknown> }>;
|
|
25
|
+
removals: Array<{ table: string; key: Record<string, unknown> }>;
|
|
26
|
+
checkpoints: Array<{ seq: number; state: unknown }>;
|
|
27
|
+
logs: Array<{ at: string; level: 'info' | 'warn' | 'error'; message: string }>;
|
|
28
|
+
rejected: Array<{ requestId: string; code: string; message: string }>;
|
|
29
|
+
stopped: boolean;
|
|
30
|
+
error: { code: string; message: string; stack: string | null } | null;
|
|
31
|
+
/** 连接器操作作业的结果(文案已遮蔽);其他作业或运行失败(`error` 非 null)时没有。 */
|
|
32
|
+
operation?: GuestOperationResult;
|
|
33
|
+
/** 建议列作业的结果;其他作业或运行失败时没有。 */
|
|
34
|
+
suggest?: SuggestedColumn[];
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* 执行器:同一个作业交给不同的执行方式;宿主调用一律交给传入的 host。
|
|
39
|
+
* `limit` 由执行器实现:累计记录数达到 limit 时对 `emit` 回 `{ stop: true }`,超出部分截掉。
|
|
40
|
+
*/
|
|
41
|
+
export interface GuestExecutor {
|
|
42
|
+
/** 执行器加载的连接器描述(便捷包装据此取凭证字段与表)。 */
|
|
43
|
+
readonly descriptor?: ConnectorDescriptor;
|
|
44
|
+
run(job: GuestJob, host: LocalHost, options?: { limit?: number; timeoutMs?: number }): Promise<LocalRunResult>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type LocalRunErrorCode = 'DB_DRIVER_MISSING' | 'HOST_DENIED' | 'TIMEOUT';
|
|
48
|
+
|
|
49
|
+
/** 本地运行环境的问题(不是连接器代码的问题)。 */
|
|
50
|
+
export class LocalRunError extends Error {
|
|
51
|
+
readonly code: LocalRunErrorCode;
|
|
52
|
+
|
|
53
|
+
constructor(code: LocalRunErrorCode, message: string) {
|
|
54
|
+
super(message);
|
|
55
|
+
this.name = 'LocalRunError';
|
|
56
|
+
this.code = code;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* - `DESCRIPTOR_NOT_GENERATED`:没有 `.ontos/descriptor.json`(先运行 `ontos admin connector codegen`);
|
|
62
|
+
* - `DESCRIPTOR_INVALID`:`.ontos/descriptor.json` 不是合法的连接器描述(重新运行 codegen)。
|
|
63
|
+
*/
|
|
64
|
+
export type SourceLoadErrorCode =
|
|
65
|
+
| 'IMPORT_FORBIDDEN' | 'PATH_ESCAPE' | 'SYMLINK' | 'MANIFEST_INVALID' | 'SYNTAX_ERROR' | 'TOO_LARGE' | 'DESCRIPTOR_NOT_GENERATED' | 'DESCRIPTOR_INVALID';
|
|
66
|
+
|
|
67
|
+
/** 源码目录加载失败;`file` 为相对源码根目录的路径(与整个目录相关时为 null)。 */
|
|
68
|
+
export class SourceLoadError extends Error {
|
|
69
|
+
readonly code: SourceLoadErrorCode;
|
|
70
|
+
readonly file: string | null;
|
|
71
|
+
|
|
72
|
+
constructor(code: SourceLoadErrorCode, message: string, file: string | null = null) {
|
|
73
|
+
super(message);
|
|
74
|
+
this.name = 'SourceLoadError';
|
|
75
|
+
this.code = code;
|
|
76
|
+
this.file = file;
|
|
77
|
+
}
|
|
78
|
+
}
|