@deepexi/datasense-connector 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -139
- package/package.json +19 -28
- package/scripts/build.ts +188 -0
- package/src/define.ts +267 -0
- package/src/descriptor.ts +116 -0
- package/src/errors.ts +105 -0
- package/src/index.ts +108 -0
- package/src/internal/location.ts +23 -0
- package/src/runtime/guest/bridge.ts +238 -0
- package/src/runtime/guest/cache.ts +43 -0
- package/src/runtime/guest/check.ts +17 -0
- package/src/runtime/guest/crypto.ts +166 -0
- package/src/runtime/guest/ctx.ts +157 -0
- package/src/runtime/guest/db.ts +59 -0
- package/src/runtime/guest/failure.ts +69 -0
- package/src/runtime/guest/http.ts +194 -0
- package/src/runtime/guest/index.ts +102 -0
- package/{runtime/guest/limits.d.ts → src/runtime/guest/limits.ts} +12 -12
- package/src/runtime/guest/loader.ts +210 -0
- package/src/runtime/guest/log.ts +47 -0
- package/src/runtime/guest/map.ts +45 -0
- package/src/runtime/guest/operation.ts +193 -0
- package/src/runtime/guest/paginate.ts +169 -0
- package/src/runtime/guest/pull.ts +177 -0
- package/src/runtime/guest/push.ts +92 -0
- package/src/runtime/guest/signal.ts +14 -0
- package/src/runtime/guest/suggest.ts +51 -0
- package/src/runtime/guest/time.ts +137 -0
- package/src/runtime/guest-runtime.ts +75 -0
- package/src/runtime/index.ts +53 -0
- package/src/runtime/protocol.ts +557 -0
- package/src/runtime/session.ts +216 -0
- package/src/testing/child.ts +21 -0
- package/src/testing/compliance.ts +343 -0
- package/src/testing/engine.ts +445 -0
- package/src/testing/fixtures.ts +591 -0
- package/src/testing/gateway.ts +482 -0
- package/src/testing/import-guard.ts +221 -0
- package/src/testing/index.ts +43 -0
- package/src/testing/local-db.ts +226 -0
- package/src/testing/mock-external-system.ts +249 -0
- package/src/testing/pinned-http.ts +144 -0
- package/src/testing/recorder.ts +140 -0
- package/src/testing/redaction.ts +94 -0
- package/src/testing/replay.ts +437 -0
- package/src/testing/run-tests.ts +115 -0
- package/src/testing/schema.ts +70 -0
- package/src/testing/source-loader.ts +374 -0
- package/src/testing/types.ts +78 -0
- package/src/types.ts +329 -0
- package/src/validate.ts +267 -0
- package/test/build.test.ts +159 -0
- package/test/define.test-d.ts +182 -0
- package/test/fixtures/acceptance.ts +156 -0
- package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
- package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
- package/test/fixtures/connectors/http-pull/package.json +5 -0
- package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
- package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
- package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
- package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
- package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
- package/test/fixtures/connectors/pg-pull/package.json +5 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
- package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
- package/test/fixtures/descriptor.ts +54 -0
- package/test/fixtures/kingdee-example.ts +82 -0
- package/test/fixtures/record-acceptance.ts +32 -0
- package/test/runtime/ctx.test.ts +434 -0
- package/test/runtime/helpers.ts +157 -0
- package/test/runtime/jobs.test.ts +444 -0
- package/test/runtime/operation.test.ts +175 -0
- package/test/runtime/session.test.ts +229 -0
- package/test/testing/acceptance-fixtures.test.ts +69 -0
- package/test/testing/compliance.test.ts +247 -0
- package/test/testing/engine.test.ts +283 -0
- package/test/testing/fixtures.test.ts +221 -0
- package/test/testing/gateway.test.ts +337 -0
- package/test/testing/import-guard.test.ts +45 -0
- package/test/testing/mock-external-system.test.ts +144 -0
- package/test/testing/operation-fixtures.test.ts +160 -0
- package/test/testing/recorder.test.ts +163 -0
- package/test/testing/redaction.test.ts +54 -0
- package/test/testing/replay.test.ts +207 -0
- package/test/testing/run-tests.test.ts +137 -0
- package/test/testing/source-loader.test.ts +220 -0
- package/test/validate-config.test.ts +104 -0
- package/tsconfig.build.json +23 -0
- package/tsconfig.json +19 -0
- package/cjs/index.cjs +0 -1015
- package/cjs/runtime.cjs +0 -752
- package/cjs/testing.cjs +0 -3591
- package/errors.d.ts +0 -17
- package/errors.js +0 -53
- package/guest.cjs +0 -2659
- package/index.d.ts +0 -13
- package/index.js +0 -7
- package/internal/location.d.ts +0 -6
- package/internal/location.js +0 -8
- package/presentation.d.ts +0 -38
- package/presentation.js +0 -268
- package/runtime/guest/bridge.d.ts +0 -53
- package/runtime/guest/bridge.js +0 -187
- package/runtime/guest/cache.d.ts +0 -3
- package/runtime/guest/cache.js +0 -42
- package/runtime/guest/crypto.d.ts +0 -13
- package/runtime/guest/crypto.js +0 -177
- package/runtime/guest/ctx.d.ts +0 -30
- package/runtime/guest/ctx.js +0 -124
- package/runtime/guest/db.d.ts +0 -6
- package/runtime/guest/db.js +0 -59
- package/runtime/guest/discover.d.ts +0 -29
- package/runtime/guest/discover.js +0 -51
- package/runtime/guest/failure.d.ts +0 -24
- package/runtime/guest/failure.js +0 -63
- package/runtime/guest/http.d.ts +0 -4
- package/runtime/guest/http.js +0 -204
- package/runtime/guest/index.d.ts +0 -12
- package/runtime/guest/index.js +0 -89
- package/runtime/guest/limits.js +0 -12
- package/runtime/guest/loader.d.ts +0 -15
- package/runtime/guest/loader.js +0 -42
- package/runtime/guest/log.d.ts +0 -10
- package/runtime/guest/log.js +0 -36
- package/runtime/guest/map.d.ts +0 -4
- package/runtime/guest/map.js +0 -45
- package/runtime/guest/paginate.d.ts +0 -6
- package/runtime/guest/paginate.js +0 -180
- package/runtime/guest/pull.d.ts +0 -12
- package/runtime/guest/pull.js +0 -150
- package/runtime/guest/push.d.ts +0 -18
- package/runtime/guest/push.js +0 -80
- package/runtime/guest/signal.d.ts +0 -3
- package/runtime/guest/signal.js +0 -12
- package/runtime/guest/time.d.ts +0 -5
- package/runtime/guest/time.js +0 -127
- package/runtime/guest-runtime.d.ts +0 -7
- package/runtime/guest-runtime.js +0 -51
- package/runtime/index.d.ts +0 -18
- package/runtime/index.js +0 -3
- package/runtime/protocol.d.ts +0 -276
- package/runtime/protocol.js +0 -409
- package/runtime/session.d.ts +0 -12
- package/runtime/session.js +0 -191
- package/t.d.ts +0 -18
- package/t.js +0 -26
- package/testing/child.cjs +0 -350
- package/testing/child.d.ts +0 -1
- package/testing/child.js +0 -12
- package/testing/compliance.d.ts +0 -41
- package/testing/compliance.js +0 -291
- package/testing/engine.d.ts +0 -66
- package/testing/engine.js +0 -339
- package/testing/fixtures.d.ts +0 -145
- package/testing/fixtures.js +0 -407
- package/testing/gateway.d.ts +0 -65
- package/testing/gateway.js +0 -402
- package/testing/import-guard.d.ts +0 -57
- package/testing/import-guard.js +0 -187
- package/testing/index.d.ts +0 -33
- package/testing/index.js +0 -10
- package/testing/local-db.d.ts +0 -27
- package/testing/local-db.js +0 -181
- package/testing/pinned-http.d.ts +0 -14
- package/testing/pinned-http.js +0 -125
- package/testing/recorder.d.ts +0 -15
- package/testing/recorder.js +0 -116
- package/testing/redaction.d.ts +0 -28
- package/testing/redaction.js +0 -67
- package/testing/replay.d.ts +0 -50
- package/testing/replay.js +0 -313
- package/testing/schema.d.ts +0 -13
- package/testing/schema.js +0 -65
- package/testing/source-loader.d.ts +0 -19
- package/testing/source-loader.js +0 -238
- package/testing/types.d.ts +0 -79
- package/testing/types.js +0 -18
- package/types.d.ts +0 -302
- package/types.js +0 -13
- package/validate.d.ts +0 -78
- package/validate.js +0 -597
package/src/types.ts
ADDED
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 连接器 SDK 的运行时类型:`ctx` 的各项能力、数据流产出、推送与连接器操作的实现形状,以及平台连接上的声明类型
|
|
3
|
+
* (`TableDeclaration` / `ScalarType` 等,connector 数据面在用)。
|
|
4
|
+
*
|
|
5
|
+
* 连接器的元数据写在连接器描述里(见 `descriptor.ts`),代码只写实现(见 `define.ts`)。连接器打包后在 `node:vm` 里执行,
|
|
6
|
+
* `ctx` 由 SDK 的 guest 运行时提供,所以这里不引用任何 Node / Bun 内置模块。
|
|
7
|
+
*
|
|
8
|
+
* 文件来源(`ctx.files`)不在本期:`ctx.files` 的类型是 `never`,访问时抛 `errors.NotAvailable('ctx.files')`。
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export type ScalarKind = 'string' | 'integer' | 'decimal' | 'boolean' | 'date' | 'instant' | 'json';
|
|
12
|
+
|
|
13
|
+
export interface ScalarType {
|
|
14
|
+
readonly kind: ScalarKind;
|
|
15
|
+
readonly precision?: number;
|
|
16
|
+
readonly scale?: number;
|
|
17
|
+
readonly title?: string;
|
|
18
|
+
/** 作为表列时是列说明。 */
|
|
19
|
+
readonly description?: string;
|
|
20
|
+
readonly placeholder?: string;
|
|
21
|
+
readonly optional?: boolean;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** 凭证字段。只能出现在连接配置的顶层;脚本里拿到的是句柄,`exposeToScript` 时可取明文。 */
|
|
25
|
+
export interface SecretType {
|
|
26
|
+
readonly kind: 'secret';
|
|
27
|
+
readonly exposeToScript?: boolean;
|
|
28
|
+
readonly title?: string;
|
|
29
|
+
readonly description?: string;
|
|
30
|
+
readonly placeholder?: string;
|
|
31
|
+
readonly optional?: boolean;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ObjectType {
|
|
35
|
+
readonly kind: 'object';
|
|
36
|
+
readonly shape: Readonly<Record<string, SdkType>>;
|
|
37
|
+
readonly title?: string;
|
|
38
|
+
readonly description?: string;
|
|
39
|
+
readonly optional?: boolean;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface ArrayType {
|
|
43
|
+
readonly kind: 'array';
|
|
44
|
+
readonly item: SdkType;
|
|
45
|
+
readonly title?: string;
|
|
46
|
+
readonly description?: string;
|
|
47
|
+
readonly optional?: boolean;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export type SdkType = ScalarType | SecretType | ObjectType | ArrayType;
|
|
51
|
+
|
|
52
|
+
export interface TableDeclaration {
|
|
53
|
+
columns: Record<string, ScalarType>;
|
|
54
|
+
primaryKey: string[];
|
|
55
|
+
retention?: { column: string; days: number };
|
|
56
|
+
purgeColumns?: { tenant?: string; user?: string };
|
|
57
|
+
indexes?: string[][];
|
|
58
|
+
/** 表的显示名,≤ 40 字符。 */
|
|
59
|
+
title?: string;
|
|
60
|
+
/** 表说明,≤ 200 字符。 */
|
|
61
|
+
description?: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export type SyncMode = 'full' | 'incremental';
|
|
65
|
+
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
// 数据流产出
|
|
68
|
+
|
|
69
|
+
export interface StreamInput<S = unknown> {
|
|
70
|
+
mode: SyncMode;
|
|
71
|
+
/** 上次提交的断点;首次为 undefined。 */
|
|
72
|
+
state: S | undefined;
|
|
73
|
+
streamConfig: Readonly<Record<string, unknown>>;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export interface RecordItem {
|
|
77
|
+
readonly __ontos: 'record';
|
|
78
|
+
table: string;
|
|
79
|
+
row: Record<string, unknown>;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface RemoveItem {
|
|
83
|
+
readonly __ontos: 'remove';
|
|
84
|
+
table: string;
|
|
85
|
+
key: Record<string, unknown>;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export interface CheckpointItem {
|
|
89
|
+
readonly __ontos: 'checkpoint';
|
|
90
|
+
state: unknown;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export type StreamItem = RecordItem | RemoveItem | CheckpointItem;
|
|
94
|
+
|
|
95
|
+
export type StreamFunction = (ctx: PullContext, input: StreamInput) => AsyncIterable<StreamItem>;
|
|
96
|
+
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
// ctx.http
|
|
99
|
+
|
|
100
|
+
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'HEAD';
|
|
101
|
+
|
|
102
|
+
export interface HttpResponse<T = unknown> {
|
|
103
|
+
status: number;
|
|
104
|
+
/** 最终地址(跟随重定向之后)。 */
|
|
105
|
+
url: string;
|
|
106
|
+
headers: Record<string, string>;
|
|
107
|
+
/** `content-type` 为 JSON 时是解析后的值,否则同 `text`。 */
|
|
108
|
+
data: T;
|
|
109
|
+
text: string;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** 登录换来的会话:附加的请求头与相对路径的基址。 */
|
|
113
|
+
export interface HttpAuth {
|
|
114
|
+
headers?: Record<string, string>;
|
|
115
|
+
baseUrl?: string;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
export interface HttpOptions {
|
|
119
|
+
headers?: Record<string, string>;
|
|
120
|
+
query?: Record<string, unknown>;
|
|
121
|
+
body?: unknown;
|
|
122
|
+
auth?: HttpAuth;
|
|
123
|
+
timeoutMs?: number;
|
|
124
|
+
/** 为 true 时不按状态码抛错、不解析 JSON,原样返回。 */
|
|
125
|
+
raw?: boolean;
|
|
126
|
+
/**
|
|
127
|
+
* 上游 502 / 503 / 504 与网关超时是否自动重试(请求可能已被上游处理)。
|
|
128
|
+
* 缺省只对幂等方法(GET / HEAD / PUT / DELETE)重试;POST / PATCH 需显式传 `true`,传 `false` 则一律不重试。
|
|
129
|
+
* 网关限流与上游 429(请求未被处理)不受影响,总会等待后重发。
|
|
130
|
+
*/
|
|
131
|
+
retry?: boolean;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface HttpClient {
|
|
135
|
+
get<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
|
|
136
|
+
post<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
|
|
137
|
+
put<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
|
|
138
|
+
patch<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
|
|
139
|
+
delete<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
|
|
140
|
+
request<T = unknown>(method: HttpMethod, url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// ---------------------------------------------------------------------------
|
|
144
|
+
// ctx.paginate
|
|
145
|
+
|
|
146
|
+
/** 分页迭代产出的一页。 */
|
|
147
|
+
export interface Page<Row = unknown> {
|
|
148
|
+
rows: Row[];
|
|
149
|
+
/** 本页各行取值的最大值(字符串按字典序、数字按大小);空页为 undefined。 */
|
|
150
|
+
maxOf<V>(fn: (row: Row) => V): V | undefined;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** 按页码翻页。没有 `totalPages` 时翻到空页为止。 */
|
|
154
|
+
export interface PagePaginateOptions<Res, Row> {
|
|
155
|
+
/** 缺省 1。 */
|
|
156
|
+
firstPage?: number;
|
|
157
|
+
request(page: number): Promise<HttpResponse<Res>>;
|
|
158
|
+
rows(res: HttpResponse<Res>): Row[];
|
|
159
|
+
totalPages?(res: HttpResponse<Res>): number;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** 按偏移量翻页。没有 `total` 时翻到不满一页为止。 */
|
|
163
|
+
export interface OffsetPaginateOptions<Res, Row> {
|
|
164
|
+
limit: number;
|
|
165
|
+
/** 缺省 0。 */
|
|
166
|
+
startOffset?: number;
|
|
167
|
+
request(offset: number, limit: number): Promise<HttpResponse<Res>>;
|
|
168
|
+
rows(res: HttpResponse<Res>): Row[];
|
|
169
|
+
total?(res: HttpResponse<Res>): number;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** 按游标翻页。`nextCursor` 返回空值时结束。 */
|
|
173
|
+
export interface CursorPaginateOptions<Res, Row> {
|
|
174
|
+
request(cursor: string | undefined): Promise<HttpResponse<Res>>;
|
|
175
|
+
rows(res: HttpResponse<Res>): Row[];
|
|
176
|
+
nextCursor(res: HttpResponse<Res>): string | null | undefined;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** 按下一页链接翻页。缺省取响应头 `Link` 里 `rel="next"` 的地址。 */
|
|
180
|
+
export interface LinkPaginateOptions<Res, Row> {
|
|
181
|
+
url: string;
|
|
182
|
+
options?: HttpOptions;
|
|
183
|
+
rows(res: HttpResponse<Res>): Row[];
|
|
184
|
+
next?(res: HttpResponse<Res>): string | null | undefined;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** 数据库按键分页:`query` 以上一页最后一行的键(首页为 undefined)查下一页,返回空页时结束。 */
|
|
188
|
+
export interface KeysetPaginateOptions<Row, Key> {
|
|
189
|
+
query(last: Key | undefined): AsyncIterable<Row>;
|
|
190
|
+
key(row: Row): Key;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export interface Paginate {
|
|
194
|
+
page<Res = unknown, Row = unknown>(options: PagePaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
|
|
195
|
+
offset<Res = unknown, Row = unknown>(options: OffsetPaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
|
|
196
|
+
cursor<Res = unknown, Row = unknown>(options: CursorPaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
|
|
197
|
+
link<Res = unknown, Row = unknown>(options: LinkPaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
|
|
198
|
+
keyset<Row = Record<string, unknown>, Key = unknown>(options: KeysetPaginateOptions<Row, Key>): AsyncIterable<Page<Row>>;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// ---------------------------------------------------------------------------
|
|
202
|
+
// ctx 的其余成员
|
|
203
|
+
|
|
204
|
+
export interface ConnectorDb {
|
|
205
|
+
/** 一次查询一个只读事务;只接受参数化查询(`$1`、`$2`…),一条语句。 */
|
|
206
|
+
query<R = Record<string, unknown>>(sql: string, params?: unknown[]): AsyncIterable<R>;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export interface ConnectorCache {
|
|
210
|
+
get(key: string): Promise<string | undefined>;
|
|
211
|
+
set(key: string, value: string, ttlSeconds: number): Promise<void>;
|
|
212
|
+
getOrSet(key: string, ttlSeconds: number, fn: () => Promise<string>): Promise<string>;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export type HmacAlgorithm = 'sha1' | 'sha256' | 'sha512';
|
|
216
|
+
export type SignAlgorithm = 'rsa-sha256';
|
|
217
|
+
export type HashAlgorithm = 'md5' | 'sha1' | 'sha256' | 'sha512';
|
|
218
|
+
export type DigestEncoding = 'hex' | 'base64';
|
|
219
|
+
|
|
220
|
+
/** `key` 可以是凭证句柄(`ctx.secrets.<字段>`),由网关代算。 */
|
|
221
|
+
export interface ConnectorCrypto {
|
|
222
|
+
hmac(algorithm: HmacAlgorithm, key: string, data: string, encoding?: DigestEncoding): Promise<string>;
|
|
223
|
+
sign(algorithm: SignAlgorithm, key: string, data: string, encoding?: DigestEncoding): Promise<string>;
|
|
224
|
+
hash(algorithm: HashAlgorithm, data: string, encoding?: DigestEncoding): Promise<string>;
|
|
225
|
+
base64Encode(text: string): string;
|
|
226
|
+
base64Decode(b64: string): string;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export interface ConnectorTime {
|
|
230
|
+
/** 当前时间,ISO 8601 UTC。 */
|
|
231
|
+
now(): string;
|
|
232
|
+
/** 把来源给的本地时间文本按时区换成 ISO 8601 UTC。 */
|
|
233
|
+
toUtc(local: string, timeZone: string, format?: string): string;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export interface ConnectorSignal {
|
|
237
|
+
readonly aborted: boolean;
|
|
238
|
+
onAbort(fn: () => void): void;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export interface ConnectorLog {
|
|
242
|
+
info(message: string, data?: Record<string, unknown>): void;
|
|
243
|
+
warn(message: string, data?: Record<string, unknown>): void;
|
|
244
|
+
error(message: string, data?: Record<string, unknown>): void;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
export interface PullContext {
|
|
248
|
+
readonly config: Readonly<Record<string, unknown>>;
|
|
249
|
+
/** 凭证句柄(`ontos:secret:<字段>`);`exposeToScript` 字段是明文。 */
|
|
250
|
+
readonly secrets: Readonly<Record<string, string>>;
|
|
251
|
+
readonly http: HttpClient;
|
|
252
|
+
readonly paginate: Paginate;
|
|
253
|
+
readonly db: ConnectorDb;
|
|
254
|
+
/** 保留名字;访问即抛 `errors.NotAvailable('ctx.files')`。 */
|
|
255
|
+
readonly files: never;
|
|
256
|
+
map<T, R>(items: readonly T[], fn: (item: T) => Promise<R>, options?: { concurrency?: number }): Promise<R[]>;
|
|
257
|
+
readonly cache: ConnectorCache;
|
|
258
|
+
readonly crypto: ConnectorCrypto;
|
|
259
|
+
readonly time: ConnectorTime;
|
|
260
|
+
record(table: string, row: Record<string, unknown>): RecordItem;
|
|
261
|
+
remove(table: string, key: Record<string, unknown>): RemoveItem;
|
|
262
|
+
checkpoint(state: unknown): CheckpointItem;
|
|
263
|
+
readonly log: ConnectorLog;
|
|
264
|
+
readonly signal: ConnectorSignal;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export type CheckContext = Omit<PullContext, 'record' | 'remove' | 'checkpoint'>;
|
|
268
|
+
|
|
269
|
+
// ---------------------------------------------------------------------------
|
|
270
|
+
// 推送
|
|
271
|
+
|
|
272
|
+
export interface PushRequest {
|
|
273
|
+
readonly id: string;
|
|
274
|
+
readonly receivedAt: string;
|
|
275
|
+
readonly headers: Readonly<Record<string, string>>;
|
|
276
|
+
readonly body: string;
|
|
277
|
+
json<T = unknown>(): T;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
export interface PushContext {
|
|
281
|
+
readonly config: Readonly<Record<string, unknown>>;
|
|
282
|
+
record(table: string, row: Record<string, unknown>): void;
|
|
283
|
+
remove(table: string, key: Record<string, unknown>): void;
|
|
284
|
+
readonly log: ConnectorLog;
|
|
285
|
+
readonly time: { now(): string };
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export interface PushHandler {
|
|
289
|
+
verify?(ctx: PushContext, req: PushRequest): void | Promise<void>;
|
|
290
|
+
map(ctx: PushContext, req: PushRequest): void | Promise<void>;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// ---------------------------------------------------------------------------
|
|
294
|
+
// 连接器操作
|
|
295
|
+
|
|
296
|
+
/** 连接器操作的 `ctx`:只有出站请求、凭证、加密、缓存、时间与日志,没有 `record`、`db`。 */
|
|
297
|
+
export interface OperationContext {
|
|
298
|
+
readonly config: Readonly<Record<string, unknown>>;
|
|
299
|
+
/** 凭证句柄(`ontos:secret:<字段>`);`exposeToScript` 字段是明文。 */
|
|
300
|
+
readonly secrets: Readonly<Record<string, string>>;
|
|
301
|
+
readonly http: HttpClient;
|
|
302
|
+
readonly crypto: ConnectorCrypto;
|
|
303
|
+
readonly cache: ConnectorCache;
|
|
304
|
+
readonly time: ConnectorTime;
|
|
305
|
+
readonly log: ConnectorLog;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/** `lookup` 的回答:`applied` 带输出(按 `output` 声明校验);抛错按 `unknown` 处理。 */
|
|
309
|
+
export type LookupResult<O> = { status: 'applied'; output: O } | { status: 'not_applied' } | { status: 'unknown' };
|
|
310
|
+
|
|
311
|
+
export type ConnectorOperationEffect = 'create' | 'update' | 'delete';
|
|
312
|
+
export type ConnectorOperationRisk = 'low' | 'medium' | 'high' | 'critical';
|
|
313
|
+
|
|
314
|
+
/**
|
|
315
|
+
* 一个连接器操作的实现(契约写在 `operations/<名>/operation.yaml` 里):
|
|
316
|
+
* - `idempotency: 'supported'`:外部系统按 `idempotencyKey` 去重,平台超时后可用同一个键重发;
|
|
317
|
+
* - `idempotency: 'unsupported'`:平台不重发,结果不明时调 `lookup` 查询,所以必须实现 `lookup`;
|
|
318
|
+
* - `simulate` 是纯函数(拿不到 `ctx`),推演与试运行时代替 `run` 返回模拟输出。
|
|
319
|
+
*/
|
|
320
|
+
export interface OperationHandlers<I = Record<string, unknown>, O = Record<string, unknown>> {
|
|
321
|
+
run(ctx: OperationContext, params: I, meta: { idempotencyKey: string }): O | Promise<O>;
|
|
322
|
+
lookup?(ctx: OperationContext, input: { idempotencyKey: string; params: I }): LookupResult<O> | Promise<LookupResult<O>>;
|
|
323
|
+
simulate?(params: I): O;
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
export interface ConnectorRateLimit {
|
|
327
|
+
perSecond: number;
|
|
328
|
+
concurrency: number;
|
|
329
|
+
}
|
package/src/validate.ts
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 名字规则、展示文字的长度上限,以及按连接器描述校验的值:连接配置(`validateConfig`)、连接器操作的入参与输出
|
|
3
|
+
* (`validateOperationValue`)、`suggestColumns` 的建议列(`validateSuggestedColumns`)。
|
|
4
|
+
* 输入可能来自 vm 里的不可信代码或租户数据,所以按 unknown 逐层检查,只返回问题列表、不抛异常。
|
|
5
|
+
*
|
|
6
|
+
* 连接器描述本身的校验(YAML 解析、必填说明、跨文件规则)在平台(shared)里,SDK 不重复。
|
|
7
|
+
*/
|
|
8
|
+
import type { ColumnType, ConfigField, SuggestedColumn, ValueField } from './descriptor';
|
|
9
|
+
|
|
10
|
+
/** 表名、连接器名、数据流名与推送名的规则,与 shared 的 DATA_FOUNDATION_NAME_PATTERN 一致。 */
|
|
11
|
+
export const SDK_NAME_PATTERN: RegExp = /^[a-z][a-z0-9_]{0,27}$/;
|
|
12
|
+
|
|
13
|
+
/** 保留的推送名:`dev` 会和接收地址的 `/dev` 后缀冲突。与 shared 同值,SDK 不依赖 shared,自己定义。 */
|
|
14
|
+
export const RESERVED_PUSH_NAMES: readonly string[] = Object.freeze(['dev']);
|
|
15
|
+
|
|
16
|
+
/** 缺省推送名(接收地址不带推送名后缀时对应它)。 */
|
|
17
|
+
export const DEFAULT_PUSH_NAME = 'default';
|
|
18
|
+
|
|
19
|
+
/** 连接器操作名的规则。 */
|
|
20
|
+
export const OPERATION_NAME_PATTERN: RegExp = /^[a-z][a-z0-9_]{0,62}$/;
|
|
21
|
+
|
|
22
|
+
const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
|
|
23
|
+
const INSTANT_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2})$/;
|
|
24
|
+
const DECIMAL_TEXT_PATTERN = /^-?\d+(?:\.\d+)?$/;
|
|
25
|
+
const COLUMN_TYPES: ReadonlySet<string> = new Set<ColumnType>(['string', 'integer', 'decimal', 'boolean', 'date', 'instant', 'json']);
|
|
26
|
+
/** 建议列(将成为扩展列)的名字:不得以 `_` 开头(`_ontos_` 前缀留给平台系统列),最长 63(PostgreSQL 标识符上限)。 */
|
|
27
|
+
const SUGGESTED_COLUMN_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,62}$/;
|
|
28
|
+
/** 一次建议最多的列数。 */
|
|
29
|
+
export const SUGGESTED_COLUMNS_MAX = 1000;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* 展示文字的长度上限(字符数按 Unicode 码点计,字节数按解码后的字节计;多语言文字每种语言分别计)。
|
|
33
|
+
* `title` / `description` / `readmeBytes` / `iconBytes` / `iconMinPx` 是连接器本身(标题、说明、README.md、图标)的上限;
|
|
34
|
+
* 其余是配置项、字段、表与列的上限。连接器描述的校验在 shared 里,沿用这里的值。
|
|
35
|
+
*/
|
|
36
|
+
export const PRESENTATION_LIMITS: {
|
|
37
|
+
readonly title: 40;
|
|
38
|
+
readonly description: 120;
|
|
39
|
+
readonly readmeBytes: 65536;
|
|
40
|
+
readonly iconBytes: 262144;
|
|
41
|
+
readonly iconMinPx: 128;
|
|
42
|
+
readonly fieldTitle: 40;
|
|
43
|
+
readonly fieldDescription: 200;
|
|
44
|
+
readonly placeholder: 100;
|
|
45
|
+
readonly tableTitle: 40;
|
|
46
|
+
readonly tableDescription: 200;
|
|
47
|
+
} = Object.freeze({
|
|
48
|
+
title: 40,
|
|
49
|
+
description: 120,
|
|
50
|
+
readmeBytes: 65536,
|
|
51
|
+
iconBytes: 262144,
|
|
52
|
+
iconMinPx: 128,
|
|
53
|
+
fieldTitle: 40,
|
|
54
|
+
fieldDescription: 200,
|
|
55
|
+
placeholder: 100,
|
|
56
|
+
tableTitle: 40,
|
|
57
|
+
tableDescription: 200,
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
export interface DeclarationIssue {
|
|
61
|
+
path: string;
|
|
62
|
+
message: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
66
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function isValidDate(text: string): boolean {
|
|
70
|
+
if (!DATE_PATTERN.test(text)) return false;
|
|
71
|
+
const date = new Date(`${text}T00:00:00Z`);
|
|
72
|
+
return !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === text;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** `http(s)://` + 非空主机(可带用户信息与端口)+ 可选的路径、查询、片段;用正则判断,guest 的 vm 里没有 `URL`。 */
|
|
76
|
+
const HTTP_URL_PATTERN = /^https?:\/\/(?:[^\s@/?#]*@)?(?:[^\s:/?#@[\]]+|\[[0-9a-f:.]+\])(?::\d{1,5})?(?:[/?#]\S*)?$/i;
|
|
77
|
+
|
|
78
|
+
function isHttpUrl(text: string): boolean {
|
|
79
|
+
return HTTP_URL_PATTERN.test(text);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** 标量值是否符合类型;不符合时返回期望的说明。 */
|
|
83
|
+
function scalarProblem(type: string, value: unknown): string | null {
|
|
84
|
+
switch (type) {
|
|
85
|
+
case 'string':
|
|
86
|
+
return typeof value === 'string' ? null : 'a string';
|
|
87
|
+
case 'integer':
|
|
88
|
+
return typeof value === 'number' && Number.isSafeInteger(value) ? null : 'an integer';
|
|
89
|
+
case 'decimal':
|
|
90
|
+
return (typeof value === 'number' && Number.isFinite(value)) || (typeof value === 'string' && DECIMAL_TEXT_PATTERN.test(value)) ? null : 'a decimal number';
|
|
91
|
+
case 'boolean':
|
|
92
|
+
return typeof value === 'boolean' ? null : 'a boolean';
|
|
93
|
+
case 'date':
|
|
94
|
+
return typeof value === 'string' && isValidDate(value) ? null : 'a date like 2026-01-31';
|
|
95
|
+
case 'instant':
|
|
96
|
+
return typeof value === 'string' && INSTANT_PATTERN.test(value) && !Number.isNaN(Date.parse(value)) ? null : 'an ISO 8601 timestamp with time zone';
|
|
97
|
+
case 'json':
|
|
98
|
+
return null;
|
|
99
|
+
default:
|
|
100
|
+
return `of a known type (got ${type})`;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// ---------------------------------------------------------------------------
|
|
105
|
+
// 连接配置
|
|
106
|
+
|
|
107
|
+
function configProblem(field: ConfigField, value: unknown): string | null {
|
|
108
|
+
switch (field.type) {
|
|
109
|
+
case 'url':
|
|
110
|
+
return typeof value === 'string' && isHttpUrl(value) ? null : 'an http or https URL';
|
|
111
|
+
case 'enum': {
|
|
112
|
+
const values = Array.isArray(field.values) ? field.values : [];
|
|
113
|
+
return typeof value === 'string' && values.includes(value) ? null : `one of ${values.map((item) => JSON.stringify(item)).join(', ')}`;
|
|
114
|
+
}
|
|
115
|
+
default:
|
|
116
|
+
return scalarProblem(field.type, value);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* 按描述的配置项校验连接配置(或数据流参数)的值,路径是配置项名。
|
|
122
|
+
* - 缺省或为 null 的值:`optional: true` 或有 `default` 的项可以缺省,其余报 `<名> is required`;
|
|
123
|
+
* - `url` 只接受 http / https,`enum` 只接受列出的取值;
|
|
124
|
+
* - `secret` 项的值走 Secret 引用,不在配置值里,出现了也忽略;
|
|
125
|
+
* - 没有声明的键报错。
|
|
126
|
+
*/
|
|
127
|
+
export function validateConfig(fields: Record<string, ConfigField>, value: Record<string, unknown>): Array<{ path: string; message: string }> {
|
|
128
|
+
if (!isRecord(value)) return [{ path: '', message: 'config must be an object' }];
|
|
129
|
+
const declared = isRecord(fields) ? fields : {};
|
|
130
|
+
const issues: DeclarationIssue[] = [];
|
|
131
|
+
for (const [name, field] of Object.entries(declared)) {
|
|
132
|
+
if (!isRecord(field)) continue;
|
|
133
|
+
if (field.type === 'secret') continue;
|
|
134
|
+
const item = value[name];
|
|
135
|
+
if (item === undefined || item === null) {
|
|
136
|
+
if (field.optional !== true && field.default === undefined) issues.push({ path: name, message: `${name} is required` });
|
|
137
|
+
continue;
|
|
138
|
+
}
|
|
139
|
+
const problem = configProblem(field, item);
|
|
140
|
+
if (problem) issues.push({ path: name, message: `${name} must be ${problem}` });
|
|
141
|
+
}
|
|
142
|
+
for (const name of Object.keys(value)) {
|
|
143
|
+
if (!Object.hasOwn(declared, name)) issues.push({ path: name, message: `${name} is not declared` });
|
|
144
|
+
}
|
|
145
|
+
return issues;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// ---------------------------------------------------------------------------
|
|
149
|
+
// 连接器操作的入参与输出
|
|
150
|
+
|
|
151
|
+
function validateValue(path: string, field: ValueField, value: unknown, issues: DeclarationIssue[]): void {
|
|
152
|
+
if (!isRecord(field) || typeof field.type !== 'string') {
|
|
153
|
+
issues.push({ path, message: `${path} has an invalid field declaration` });
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
if (field.type === 'object') {
|
|
157
|
+
if (!isRecord(value)) {
|
|
158
|
+
issues.push({ path, message: `${path} must be an object` });
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
validateFields(path, isRecord(field.fields) ? field.fields : {}, value, issues);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
if (field.type === 'array') {
|
|
165
|
+
if (!Array.isArray(value)) {
|
|
166
|
+
issues.push({ path, message: `${path} must be an array` });
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
value.forEach((item: unknown, i) => {
|
|
170
|
+
const itemPath = `${path}[${i}]`;
|
|
171
|
+
if (item === undefined || item === null) issues.push({ path: itemPath, message: `${itemPath} is required` });
|
|
172
|
+
else if (field.item) validateValue(itemPath, field.item, item, issues);
|
|
173
|
+
});
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
const problem = scalarProblem(field.type, value);
|
|
177
|
+
if (problem) issues.push({ path, message: `${path} must be ${problem}` });
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function validateFields(path: string, fields: Record<string, ValueField>, value: Record<string, unknown>, issues: DeclarationIssue[]): void {
|
|
181
|
+
for (const [name, field] of Object.entries(fields)) {
|
|
182
|
+
const fieldPath = `${path}.${name}`;
|
|
183
|
+
const item = value[name];
|
|
184
|
+
if (item === undefined || item === null) {
|
|
185
|
+
if (isRecord(field) && field.optional === true) continue;
|
|
186
|
+
// json 字段的 null 是合法值。
|
|
187
|
+
if (!(isRecord(field) && field.type === 'json' && item === null)) issues.push({ path: fieldPath, message: `${fieldPath} is required` });
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
validateValue(fieldPath, field, item, issues);
|
|
191
|
+
}
|
|
192
|
+
for (const name of Object.keys(value)) {
|
|
193
|
+
if (!Object.hasOwn(fields, name)) issues.push({ path: `${path}.${name}`, message: `${path}.${name} is not declared` });
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/**
|
|
198
|
+
* 按连接器操作的 `input` / `output` 字段校验一个值(`path` 为 `input` 或 `output`):
|
|
199
|
+
* `optional: true` 的字段可缺省或为 null,其余必填;没有声明的字段报错。
|
|
200
|
+
*/
|
|
201
|
+
export function validateOperationValue(fields: Record<string, ValueField>, value: unknown, path: 'input' | 'output'): DeclarationIssue[] {
|
|
202
|
+
if (!isRecord(value)) return [{ path, message: `${path} must be an object` }];
|
|
203
|
+
const issues: DeclarationIssue[] = [];
|
|
204
|
+
validateFields(path, isRecord(fields) ? fields : {}, value, issues);
|
|
205
|
+
return issues;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// ---------------------------------------------------------------------------
|
|
209
|
+
// 建议列
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* 校验 `suggestColumns` 的返回值(租户代码产出,按 unknown 处理):必须是数组、不超过 `SUGGESTED_COLUMNS_MAX` 项;
|
|
213
|
+
* 每项 `name` 符合扩展列名规则、不重复,`type` 是列类型(`decimal` 要给 precision / scale),`title` 非空,`description` 可选。
|
|
214
|
+
* 合法时返回复制出的干净列表。
|
|
215
|
+
*/
|
|
216
|
+
export function validateSuggestedColumns(value: unknown): { ok: true; columns: SuggestedColumn[] } | { ok: false; issues: DeclarationIssue[] } {
|
|
217
|
+
if (!Array.isArray(value)) return { ok: false, issues: [{ path: '', message: 'suggestColumns must return an array of columns' }] };
|
|
218
|
+
if (value.length > SUGGESTED_COLUMNS_MAX) return { ok: false, issues: [{ path: '', message: `suggestColumns returned more than ${SUGGESTED_COLUMNS_MAX} columns` }] };
|
|
219
|
+
const issues: DeclarationIssue[] = [];
|
|
220
|
+
const columns: SuggestedColumn[] = [];
|
|
221
|
+
const seen = new Set<string>();
|
|
222
|
+
value.forEach((item: unknown, i) => {
|
|
223
|
+
const path = `[${i}]`;
|
|
224
|
+
if (!isRecord(item)) {
|
|
225
|
+
issues.push({ path, message: `${path} must be an object` });
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
const { name, type, precision, scale, title, description } = item;
|
|
229
|
+
if (typeof name !== 'string' || !SUGGESTED_COLUMN_NAME_PATTERN.test(name) || name.toLowerCase().startsWith('_ontos_')) {
|
|
230
|
+
issues.push({ path: `${path}.name`, message: `column name must match ${SUGGESTED_COLUMN_NAME_PATTERN.source}` });
|
|
231
|
+
return;
|
|
232
|
+
}
|
|
233
|
+
if (seen.has(name)) {
|
|
234
|
+
issues.push({ path: `${path}.name`, message: `column ${name} is suggested more than once` });
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
seen.add(name);
|
|
238
|
+
if (typeof type !== 'string' || !COLUMN_TYPES.has(type)) {
|
|
239
|
+
issues.push({ path: `${path}.type`, message: `column ${name} must have a column type (string, integer, decimal, boolean, date, instant or json)` });
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
if (typeof title !== 'string' || title.trim() === '') {
|
|
243
|
+
issues.push({ path: `${path}.title`, message: `column ${name} needs a non-empty title` });
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
if (description !== undefined && typeof description !== 'string') {
|
|
247
|
+
issues.push({ path: `${path}.description`, message: `column ${name} description must be a string` });
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
const column: SuggestedColumn = { name, type: type as ColumnType, title };
|
|
251
|
+
if (type === 'decimal') {
|
|
252
|
+
const validPrecision = typeof precision === 'number' && Number.isInteger(precision) && precision > 0;
|
|
253
|
+
if (!validPrecision || typeof scale !== 'number' || !Number.isInteger(scale) || scale < 0 || scale > precision) {
|
|
254
|
+
issues.push({ path, message: `column ${name}: decimal needs an integer precision > 0 and an integer scale between 0 and precision` });
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
column.precision = precision;
|
|
258
|
+
column.scale = scale;
|
|
259
|
+
} else if (precision !== undefined || scale !== undefined) {
|
|
260
|
+
issues.push({ path, message: `column ${name}: precision and scale are only allowed on decimal columns` });
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
if (description !== undefined) column.description = description;
|
|
264
|
+
columns.push(column);
|
|
265
|
+
});
|
|
266
|
+
return issues.length > 0 ? { ok: false, issues } : { ok: true, columns };
|
|
267
|
+
}
|