@deepexi/datasense-connector 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -139
- package/package.json +19 -28
- package/scripts/build.ts +188 -0
- package/src/define.ts +267 -0
- package/src/descriptor.ts +116 -0
- package/src/errors.ts +105 -0
- package/src/index.ts +108 -0
- package/src/internal/location.ts +23 -0
- package/src/runtime/guest/bridge.ts +238 -0
- package/src/runtime/guest/cache.ts +43 -0
- package/src/runtime/guest/check.ts +17 -0
- package/src/runtime/guest/crypto.ts +166 -0
- package/src/runtime/guest/ctx.ts +157 -0
- package/src/runtime/guest/db.ts +59 -0
- package/src/runtime/guest/failure.ts +69 -0
- package/src/runtime/guest/http.ts +194 -0
- package/src/runtime/guest/index.ts +102 -0
- package/{runtime/guest/limits.d.ts → src/runtime/guest/limits.ts} +12 -12
- package/src/runtime/guest/loader.ts +210 -0
- package/src/runtime/guest/log.ts +47 -0
- package/src/runtime/guest/map.ts +45 -0
- package/src/runtime/guest/operation.ts +193 -0
- package/src/runtime/guest/paginate.ts +169 -0
- package/src/runtime/guest/pull.ts +177 -0
- package/src/runtime/guest/push.ts +92 -0
- package/src/runtime/guest/signal.ts +14 -0
- package/src/runtime/guest/suggest.ts +51 -0
- package/src/runtime/guest/time.ts +137 -0
- package/src/runtime/guest-runtime.ts +75 -0
- package/src/runtime/index.ts +53 -0
- package/src/runtime/protocol.ts +557 -0
- package/src/runtime/session.ts +216 -0
- package/src/testing/child.ts +21 -0
- package/src/testing/compliance.ts +343 -0
- package/src/testing/engine.ts +445 -0
- package/src/testing/fixtures.ts +591 -0
- package/src/testing/gateway.ts +482 -0
- package/src/testing/import-guard.ts +221 -0
- package/src/testing/index.ts +43 -0
- package/src/testing/local-db.ts +226 -0
- package/src/testing/mock-external-system.ts +249 -0
- package/src/testing/pinned-http.ts +144 -0
- package/src/testing/recorder.ts +140 -0
- package/src/testing/redaction.ts +94 -0
- package/src/testing/replay.ts +437 -0
- package/src/testing/run-tests.ts +115 -0
- package/src/testing/schema.ts +70 -0
- package/src/testing/source-loader.ts +374 -0
- package/src/testing/types.ts +78 -0
- package/src/types.ts +329 -0
- package/src/validate.ts +267 -0
- package/test/build.test.ts +159 -0
- package/test/define.test-d.ts +182 -0
- package/test/fixtures/acceptance.ts +156 -0
- package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
- package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
- package/test/fixtures/connectors/http-pull/package.json +5 -0
- package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
- package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
- package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
- package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
- package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
- package/test/fixtures/connectors/pg-pull/package.json +5 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
- package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
- package/test/fixtures/descriptor.ts +54 -0
- package/test/fixtures/kingdee-example.ts +82 -0
- package/test/fixtures/record-acceptance.ts +32 -0
- package/test/runtime/ctx.test.ts +434 -0
- package/test/runtime/helpers.ts +157 -0
- package/test/runtime/jobs.test.ts +444 -0
- package/test/runtime/operation.test.ts +175 -0
- package/test/runtime/session.test.ts +229 -0
- package/test/testing/acceptance-fixtures.test.ts +69 -0
- package/test/testing/compliance.test.ts +247 -0
- package/test/testing/engine.test.ts +283 -0
- package/test/testing/fixtures.test.ts +221 -0
- package/test/testing/gateway.test.ts +337 -0
- package/test/testing/import-guard.test.ts +45 -0
- package/test/testing/mock-external-system.test.ts +144 -0
- package/test/testing/operation-fixtures.test.ts +160 -0
- package/test/testing/recorder.test.ts +163 -0
- package/test/testing/redaction.test.ts +54 -0
- package/test/testing/replay.test.ts +207 -0
- package/test/testing/run-tests.test.ts +137 -0
- package/test/testing/source-loader.test.ts +220 -0
- package/test/validate-config.test.ts +104 -0
- package/tsconfig.build.json +23 -0
- package/tsconfig.json +19 -0
- package/cjs/index.cjs +0 -1015
- package/cjs/runtime.cjs +0 -752
- package/cjs/testing.cjs +0 -3591
- package/errors.d.ts +0 -17
- package/errors.js +0 -53
- package/guest.cjs +0 -2659
- package/index.d.ts +0 -13
- package/index.js +0 -7
- package/internal/location.d.ts +0 -6
- package/internal/location.js +0 -8
- package/presentation.d.ts +0 -38
- package/presentation.js +0 -268
- package/runtime/guest/bridge.d.ts +0 -53
- package/runtime/guest/bridge.js +0 -187
- package/runtime/guest/cache.d.ts +0 -3
- package/runtime/guest/cache.js +0 -42
- package/runtime/guest/crypto.d.ts +0 -13
- package/runtime/guest/crypto.js +0 -177
- package/runtime/guest/ctx.d.ts +0 -30
- package/runtime/guest/ctx.js +0 -124
- package/runtime/guest/db.d.ts +0 -6
- package/runtime/guest/db.js +0 -59
- package/runtime/guest/discover.d.ts +0 -29
- package/runtime/guest/discover.js +0 -51
- package/runtime/guest/failure.d.ts +0 -24
- package/runtime/guest/failure.js +0 -63
- package/runtime/guest/http.d.ts +0 -4
- package/runtime/guest/http.js +0 -204
- package/runtime/guest/index.d.ts +0 -12
- package/runtime/guest/index.js +0 -89
- package/runtime/guest/limits.js +0 -12
- package/runtime/guest/loader.d.ts +0 -15
- package/runtime/guest/loader.js +0 -42
- package/runtime/guest/log.d.ts +0 -10
- package/runtime/guest/log.js +0 -36
- package/runtime/guest/map.d.ts +0 -4
- package/runtime/guest/map.js +0 -45
- package/runtime/guest/paginate.d.ts +0 -6
- package/runtime/guest/paginate.js +0 -180
- package/runtime/guest/pull.d.ts +0 -12
- package/runtime/guest/pull.js +0 -150
- package/runtime/guest/push.d.ts +0 -18
- package/runtime/guest/push.js +0 -80
- package/runtime/guest/signal.d.ts +0 -3
- package/runtime/guest/signal.js +0 -12
- package/runtime/guest/time.d.ts +0 -5
- package/runtime/guest/time.js +0 -127
- package/runtime/guest-runtime.d.ts +0 -7
- package/runtime/guest-runtime.js +0 -51
- package/runtime/index.d.ts +0 -18
- package/runtime/index.js +0 -3
- package/runtime/protocol.d.ts +0 -276
- package/runtime/protocol.js +0 -409
- package/runtime/session.d.ts +0 -12
- package/runtime/session.js +0 -191
- package/t.d.ts +0 -18
- package/t.js +0 -26
- package/testing/child.cjs +0 -350
- package/testing/child.d.ts +0 -1
- package/testing/child.js +0 -12
- package/testing/compliance.d.ts +0 -41
- package/testing/compliance.js +0 -291
- package/testing/engine.d.ts +0 -66
- package/testing/engine.js +0 -339
- package/testing/fixtures.d.ts +0 -145
- package/testing/fixtures.js +0 -407
- package/testing/gateway.d.ts +0 -65
- package/testing/gateway.js +0 -402
- package/testing/import-guard.d.ts +0 -57
- package/testing/import-guard.js +0 -187
- package/testing/index.d.ts +0 -33
- package/testing/index.js +0 -10
- package/testing/local-db.d.ts +0 -27
- package/testing/local-db.js +0 -181
- package/testing/pinned-http.d.ts +0 -14
- package/testing/pinned-http.js +0 -125
- package/testing/recorder.d.ts +0 -15
- package/testing/recorder.js +0 -116
- package/testing/redaction.d.ts +0 -28
- package/testing/redaction.js +0 -67
- package/testing/replay.d.ts +0 -50
- package/testing/replay.js +0 -313
- package/testing/schema.d.ts +0 -13
- package/testing/schema.js +0 -65
- package/testing/source-loader.d.ts +0 -19
- package/testing/source-loader.js +0 -238
- package/testing/types.d.ts +0 -79
- package/testing/types.js +0 -18
- package/types.d.ts +0 -302
- package/types.js +0 -13
- package/validate.d.ts +0 -78
- package/validate.js +0 -597
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 上游条目 → 表 `items` 的一行。上游契约:每个条目都有字符串 `id`、`name` 与 ISO 8601 的 `updatedAt`;
|
|
3
|
+
* 不符合契约的条目说明上游改了接口,直接抛错让运行失败(调用栈经 sourcemap 映射回本文件)。
|
|
4
|
+
*/
|
|
5
|
+
export interface RawItem { id?: unknown; name?: unknown; updatedAt?: unknown }
|
|
6
|
+
|
|
7
|
+
export function mapItem(raw: RawItem): { id: string; name: string; updated_at: string } {
|
|
8
|
+
if (typeof raw.id !== 'string' || typeof raw.updatedAt !== 'string') {
|
|
9
|
+
throw new TypeError(`upstream item is missing id or updatedAt: ${JSON.stringify(raw)}`);
|
|
10
|
+
}
|
|
11
|
+
return { id: raw.id, name: typeof raw.name === 'string' ? raw.name : '', updated_at: raw.updatedAt };
|
|
12
|
+
}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
{"seq":1,"kind":"http","request":{"method":"GET","url":"https://api.example.com/v1/items","headers":{"authorization":"***"},"body":null,"bodyEncoding":"utf8"},"response":{"status":200,"headers":{"content-type":"application/json;charset=utf-8"},"body":"{\"items\":[{\"id\":\"a\",\"name\":\"Alpha\",\"updatedAt\":\"2026-10-01T00:00:00.000Z\"},{\"id\":\"b\",\"name\":\"Beta\",\"updatedAt\":\"2026-10-02T00:00:00.000Z\"}],\"nextCursor\":\"p2\"}","bodyEncoding":"utf8"},"error":null}
|
|
2
|
+
{"seq":2,"kind":"http","request":{"method":"GET","url":"https://api.example.com/v1/items?cursor=p2","headers":{"authorization":"***"},"body":null,"bodyEncoding":"utf8"},"response":{"status":200,"headers":{"content-type":"application/json;charset=utf-8"},"body":"{\"items\":[{\"id\":\"c\",\"name\":\"Gamma\",\"updatedAt\":\"2026-10-03T00:00:00.000Z\"}],\"nextCursor\":null}","bodyEncoding":"utf8"},"error":null}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "pg_pull",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"title": "PostgreSQL pull",
|
|
5
|
+
"description": "Acceptance fixture: pull from an existing PostgreSQL database",
|
|
6
|
+
"icon": null,
|
|
7
|
+
"config": {
|
|
8
|
+
"schema": {
|
|
9
|
+
"type": "string",
|
|
10
|
+
"title": "Schema",
|
|
11
|
+
"description": "Schema",
|
|
12
|
+
"placeholder": null,
|
|
13
|
+
"optional": false
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"hosts": [],
|
|
17
|
+
"db": {
|
|
18
|
+
"engine": "postgresql"
|
|
19
|
+
},
|
|
20
|
+
"rateLimit": null,
|
|
21
|
+
"check": false,
|
|
22
|
+
"streams": {
|
|
23
|
+
"orders": {
|
|
24
|
+
"title": "Orders",
|
|
25
|
+
"description": "Orders",
|
|
26
|
+
"syncModes": [
|
|
27
|
+
"incremental"
|
|
28
|
+
],
|
|
29
|
+
"snapshot": "partial",
|
|
30
|
+
"params": {
|
|
31
|
+
"pageSize": {
|
|
32
|
+
"type": "integer",
|
|
33
|
+
"title": "Page size",
|
|
34
|
+
"description": "Page size",
|
|
35
|
+
"placeholder": null,
|
|
36
|
+
"optional": true,
|
|
37
|
+
"default": 1000
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"tables": {
|
|
41
|
+
"orders": {
|
|
42
|
+
"title": "Orders",
|
|
43
|
+
"description": "Orders",
|
|
44
|
+
"primaryKey": [
|
|
45
|
+
"id"
|
|
46
|
+
],
|
|
47
|
+
"extensible": false,
|
|
48
|
+
"columns": {
|
|
49
|
+
"id": {
|
|
50
|
+
"type": "string",
|
|
51
|
+
"title": "ID",
|
|
52
|
+
"description": null
|
|
53
|
+
},
|
|
54
|
+
"customer_id": {
|
|
55
|
+
"type": "string",
|
|
56
|
+
"title": "Customer",
|
|
57
|
+
"description": null
|
|
58
|
+
},
|
|
59
|
+
"amount": {
|
|
60
|
+
"type": "integer",
|
|
61
|
+
"title": "Amount",
|
|
62
|
+
"description": null
|
|
63
|
+
},
|
|
64
|
+
"updated_at": {
|
|
65
|
+
"type": "instant",
|
|
66
|
+
"title": "Updated at",
|
|
67
|
+
"description": null
|
|
68
|
+
}
|
|
69
|
+
},
|
|
70
|
+
"retention": null,
|
|
71
|
+
"purgeColumns": null,
|
|
72
|
+
"indexes": []
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
"suggestColumns": false
|
|
76
|
+
},
|
|
77
|
+
"customers": {
|
|
78
|
+
"title": "Customers",
|
|
79
|
+
"description": "Customers",
|
|
80
|
+
"syncModes": [
|
|
81
|
+
"full"
|
|
82
|
+
],
|
|
83
|
+
"snapshot": "complete",
|
|
84
|
+
"params": {
|
|
85
|
+
"pageSize": {
|
|
86
|
+
"type": "integer",
|
|
87
|
+
"title": "Page size",
|
|
88
|
+
"description": "Page size",
|
|
89
|
+
"placeholder": null,
|
|
90
|
+
"optional": true,
|
|
91
|
+
"default": 1000
|
|
92
|
+
}
|
|
93
|
+
},
|
|
94
|
+
"tables": {
|
|
95
|
+
"customers": {
|
|
96
|
+
"title": "Customers",
|
|
97
|
+
"description": "Customers",
|
|
98
|
+
"primaryKey": [
|
|
99
|
+
"id"
|
|
100
|
+
],
|
|
101
|
+
"extensible": true,
|
|
102
|
+
"columns": {
|
|
103
|
+
"id": {
|
|
104
|
+
"type": "string",
|
|
105
|
+
"title": "ID",
|
|
106
|
+
"description": null
|
|
107
|
+
},
|
|
108
|
+
"name": {
|
|
109
|
+
"type": "string",
|
|
110
|
+
"title": "Name",
|
|
111
|
+
"description": null
|
|
112
|
+
},
|
|
113
|
+
"balance": {
|
|
114
|
+
"type": "decimal",
|
|
115
|
+
"title": "Balance",
|
|
116
|
+
"description": null,
|
|
117
|
+
"precision": 12,
|
|
118
|
+
"scale": 2
|
|
119
|
+
},
|
|
120
|
+
"active": {
|
|
121
|
+
"type": "boolean",
|
|
122
|
+
"title": "Active",
|
|
123
|
+
"description": null
|
|
124
|
+
},
|
|
125
|
+
"joined": {
|
|
126
|
+
"type": "date",
|
|
127
|
+
"title": "Joined",
|
|
128
|
+
"description": null
|
|
129
|
+
},
|
|
130
|
+
"updated_at": {
|
|
131
|
+
"type": "instant",
|
|
132
|
+
"title": "Updated at",
|
|
133
|
+
"description": null
|
|
134
|
+
},
|
|
135
|
+
"profile": {
|
|
136
|
+
"type": "json",
|
|
137
|
+
"title": "Profile",
|
|
138
|
+
"description": null
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
"retention": null,
|
|
142
|
+
"purgeColumns": null,
|
|
143
|
+
"indexes": []
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
"suggestColumns": false
|
|
147
|
+
}
|
|
148
|
+
},
|
|
149
|
+
"pushes": {},
|
|
150
|
+
"operations": {}
|
|
151
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/* eslint-disable @typescript-eslint/no-empty-object-type -- 原样保留 shared codegen 的产物(空能力组生成 `{}`) */
|
|
2
|
+
// 由 ontos admin connector codegen 根据连接器描述生成。不要手改。
|
|
3
|
+
import {
|
|
4
|
+
defineCheck as defineCheckFor,
|
|
5
|
+
defineOperation as defineOperationFor,
|
|
6
|
+
definePush as definePushFor,
|
|
7
|
+
defineStream as defineStreamFor,
|
|
8
|
+
defineSuggestColumns as defineSuggestColumnsFor,
|
|
9
|
+
} from '@deepexi/datasense-connector';
|
|
10
|
+
|
|
11
|
+
/** 这份描述绑定到编写 API 上的类型参数。 */
|
|
12
|
+
export interface GeneratedDescriptor {
|
|
13
|
+
readonly streams: {
|
|
14
|
+
readonly orders: { readonly tables: { readonly orders: { readonly id: string; readonly customer_id: string; readonly amount: number; readonly updated_at: string; }; }; readonly params: { readonly pageSize?: number; }; };
|
|
15
|
+
readonly customers: { readonly tables: { readonly customers: { readonly id: string; readonly name: string; readonly balance: string; readonly active: boolean; readonly joined: string; readonly updated_at: string; readonly profile: unknown; }; }; readonly params: { readonly pageSize?: number; }; };
|
|
16
|
+
};
|
|
17
|
+
readonly pushes: {
|
|
18
|
+
|
|
19
|
+
};
|
|
20
|
+
readonly operations: {
|
|
21
|
+
|
|
22
|
+
};
|
|
23
|
+
readonly extensibleStreams: {
|
|
24
|
+
readonly customers: GeneratedDescriptor['streams']['customers'];
|
|
25
|
+
};
|
|
26
|
+
readonly config: { readonly schema: string; };
|
|
27
|
+
readonly secrets: { };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function defineStream<N extends keyof GeneratedDescriptor['streams'] & string>(
|
|
31
|
+
name: N,
|
|
32
|
+
fn: Parameters<typeof defineStreamFor<GeneratedDescriptor, N>>[1],
|
|
33
|
+
): ReturnType<typeof defineStreamFor<GeneratedDescriptor, N>> {
|
|
34
|
+
return defineStreamFor<GeneratedDescriptor, N>(name, fn);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function definePush<N extends keyof GeneratedDescriptor['pushes'] & string>(
|
|
38
|
+
name: N,
|
|
39
|
+
fn: Parameters<typeof definePushFor<GeneratedDescriptor, N>>[1],
|
|
40
|
+
): ReturnType<typeof definePushFor<GeneratedDescriptor, N>> {
|
|
41
|
+
return definePushFor<GeneratedDescriptor, N>(name, fn);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function defineOperation<N extends keyof GeneratedDescriptor['operations'] & string>(
|
|
45
|
+
name: N,
|
|
46
|
+
fn: Parameters<typeof defineOperationFor<GeneratedDescriptor, N>>[1],
|
|
47
|
+
): ReturnType<typeof defineOperationFor<GeneratedDescriptor, N>> {
|
|
48
|
+
return defineOperationFor<GeneratedDescriptor, N>(name, fn);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function defineCheck(
|
|
52
|
+
fn: Parameters<typeof defineCheckFor<GeneratedDescriptor>>[0],
|
|
53
|
+
): ReturnType<typeof defineCheckFor<GeneratedDescriptor>> {
|
|
54
|
+
return defineCheckFor<GeneratedDescriptor>(fn);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function defineSuggestColumns<N extends keyof GeneratedDescriptor['extensibleStreams'] & string>(
|
|
58
|
+
name: N,
|
|
59
|
+
fn: Parameters<typeof defineSuggestColumnsFor<GeneratedDescriptor, N>>[1],
|
|
60
|
+
): ReturnType<typeof defineSuggestColumnsFor<GeneratedDescriptor, N>> {
|
|
61
|
+
return defineSuggestColumnsFor<GeneratedDescriptor, N>(name, fn);
|
|
62
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/** 双引号引用标识符(模式名、表名不能走参数)。 */
|
|
2
|
+
export function quoteIdent(name: string): string {
|
|
3
|
+
return `"${name.replaceAll('"', '""')}"`;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export function relationOf(schema: string, table: string): string {
|
|
7
|
+
return `${quoteIdent(schema)}.${quoteIdent(table)}`;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
const DEFAULT_PAGE_SIZE = 1000;
|
|
11
|
+
|
|
12
|
+
export function pageSizeOf(value: unknown): number {
|
|
13
|
+
return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : DEFAULT_PAGE_SIZE;
|
|
14
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `customers`:全量完整快照,按 `id` 键分页,断点 `{ id }` 让一次全量中途失败后从断点续跑;本轮没出现的主键被平台软删除。
|
|
3
|
+
* 表可扩展:来源表新增的列原样带上,建连接时加了扩展列就落库。
|
|
4
|
+
*/
|
|
5
|
+
import { defineStream, type GeneratedDescriptor } from '../../.ontos/types';
|
|
6
|
+
import { pageSizeOf, relationOf } from '../../lib/sql';
|
|
7
|
+
|
|
8
|
+
interface CustomerKey { id: string }
|
|
9
|
+
|
|
10
|
+
/** 数据库行按来源库的列原样带上(可能有 null),生成的行类型不含 null,这里按行类型交给 `ctx.record`。 */
|
|
11
|
+
type Row = GeneratedDescriptor['streams']['customers']['tables']['customers'];
|
|
12
|
+
|
|
13
|
+
export default defineStream('customers', async function* (ctx, input) {
|
|
14
|
+
const relation = relationOf(ctx.config.schema, 'customers');
|
|
15
|
+
const limit = pageSizeOf(input.streamConfig.pageSize);
|
|
16
|
+
const start = input.state as CustomerKey | undefined;
|
|
17
|
+
const pages = ctx.paginate.keyset<Record<string, unknown>, CustomerKey>({
|
|
18
|
+
query: (last) => {
|
|
19
|
+
const after = last ?? start;
|
|
20
|
+
return after
|
|
21
|
+
? ctx.db.query(`select * from ${relation} where id > $1 order by id limit $2`, [after.id, limit])
|
|
22
|
+
: ctx.db.query(`select * from ${relation} order by id limit $1`, [limit]);
|
|
23
|
+
},
|
|
24
|
+
key: (row) => ({ id: String(row.id) }),
|
|
25
|
+
});
|
|
26
|
+
for await (const page of pages) {
|
|
27
|
+
for (const row of page.rows) yield ctx.record('customers', row as Row);
|
|
28
|
+
yield ctx.checkpoint({ id: String((page.rows[page.rows.length - 1] as Record<string, unknown>).id) } satisfies CustomerKey);
|
|
29
|
+
}
|
|
30
|
+
});
|
package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
{"seq":1,"kind":"db","sql":"select * from \"sdk_fixture_pg_pull\".\"customers\" order by id limit $1","params":[2],"fields":[{"name":"id","dataTypeId":25},{"name":"name","dataTypeId":25},{"name":"balance","dataTypeId":1700},{"name":"active","dataTypeId":16},{"name":"joined","dataTypeId":1082},{"name":"updated_at","dataTypeId":1184},{"name":"profile","dataTypeId":3802}],"rows":[{"id":"c1","name":"Ada","balance":"120.50","active":true,"joined":"2026-01-02","updated_at":"2026-10-01T08:00:00.000Z","profile":{"tier":"gold"}},{"id":"c2","name":"Bob","balance":"0.00","active":false,"joined":"2026-02-03","updated_at":"2026-10-02T08:00:00.000Z","profile":null}],"error":null}
|
|
2
|
+
{"seq":2,"kind":"db","sql":"select * from \"sdk_fixture_pg_pull\".\"customers\" where id > $1 order by id limit $2","params":["c2",2],"fields":[{"name":"id","dataTypeId":25},{"name":"name","dataTypeId":25},{"name":"balance","dataTypeId":1700},{"name":"active","dataTypeId":16},{"name":"joined","dataTypeId":1082},{"name":"updated_at","dataTypeId":1184},{"name":"profile","dataTypeId":3802}],"rows":[{"id":"c3","name":"Cyd","balance":null,"active":null,"joined":null,"updated_at":null,"profile":{"tags":["x"]}}],"error":null}
|
|
3
|
+
{"seq":3,"kind":"db","sql":"select * from \"sdk_fixture_pg_pull\".\"customers\" where id > $1 order by id limit $2","params":["c3",2],"fields":[],"rows":[],"error":null}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `orders`:增量,按 `(updated_at, id)` 键分页,每页之后提交断点 `{ updatedAt, id }`(最后一行的键)。
|
|
3
|
+
* 来源库由连接每个环境的平台字段 `db` 决定,配置 `schema` 是来源表所在的模式;表结构固定写在描述里。
|
|
4
|
+
*/
|
|
5
|
+
import { defineStream, type GeneratedDescriptor } from '../../.ontos/types';
|
|
6
|
+
import { pageSizeOf, relationOf } from '../../lib/sql';
|
|
7
|
+
|
|
8
|
+
interface OrderKey { updatedAt: string; id: string }
|
|
9
|
+
|
|
10
|
+
/** 数据库行按来源库的列原样带上(可能有 null),生成的行类型不含 null,这里按行类型交给 `ctx.record`。 */
|
|
11
|
+
type Row = GeneratedDescriptor['streams']['orders']['tables']['orders'];
|
|
12
|
+
|
|
13
|
+
export default defineStream('orders', async function* (ctx, input) {
|
|
14
|
+
const relation = relationOf(ctx.config.schema, 'orders');
|
|
15
|
+
const limit = pageSizeOf(input.streamConfig.pageSize);
|
|
16
|
+
// 重新同步(全量)时平台不给断点,从头读。
|
|
17
|
+
const start = input.state as OrderKey | undefined;
|
|
18
|
+
const pages = ctx.paginate.keyset<Record<string, unknown>, OrderKey>({
|
|
19
|
+
query: (last) => {
|
|
20
|
+
const after = last ?? start;
|
|
21
|
+
return after
|
|
22
|
+
? ctx.db.query(`select * from ${relation} where (updated_at, id) > ($1::timestamptz, $2) order by updated_at, id limit $3`, [after.updatedAt, after.id, limit])
|
|
23
|
+
: ctx.db.query(`select * from ${relation} order by updated_at, id limit $1`, [limit]);
|
|
24
|
+
},
|
|
25
|
+
key: (row) => ({ updatedAt: String(row.updated_at), id: String(row.id) }),
|
|
26
|
+
});
|
|
27
|
+
for await (const page of pages) {
|
|
28
|
+
for (const row of page.rows) yield ctx.record('orders', row as Row);
|
|
29
|
+
const last = page.rows[page.rows.length - 1] as Record<string, unknown>;
|
|
30
|
+
yield ctx.checkpoint({ updatedAt: String(last.updated_at), id: String(last.id) } satisfies OrderKey);
|
|
31
|
+
}
|
|
32
|
+
});
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/** 测试用的连接器描述构造器:只写关心的部分,其余按规整后的缺省值补齐。 */
|
|
2
|
+
import type {
|
|
3
|
+
ColumnDescriptor, ColumnType, ConfigField, ConnectorDescriptor, OperationDescriptor, PushDescriptor, StreamDescriptor, TableDescriptor, ValueField,
|
|
4
|
+
} from '../../src/index';
|
|
5
|
+
|
|
6
|
+
export function column(type: ColumnType, extra: Partial<ColumnDescriptor> = {}): ColumnDescriptor {
|
|
7
|
+
return { type, title: extra.title ?? type, description: null, ...extra };
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function table(columns: Record<string, ColumnType | ColumnDescriptor>, primaryKey: string[], extra: Partial<TableDescriptor> = {}): TableDescriptor {
|
|
11
|
+
const expanded: Record<string, ColumnDescriptor> = {};
|
|
12
|
+
for (const [name, value] of Object.entries(columns)) expanded[name] = typeof value === 'string' ? { type: value, title: name, description: null } : value;
|
|
13
|
+
return {
|
|
14
|
+
title: 'Table', description: 'A table', primaryKey, extensible: false, columns: expanded, retention: null, purgeColumns: null, indexes: [], ...extra,
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function stream(tables: Record<string, TableDescriptor>, extra: Partial<StreamDescriptor> = {}): StreamDescriptor {
|
|
19
|
+
return { title: 'Stream', description: 'A stream', syncModes: ['full', 'incremental'], snapshot: 'partial', params: {}, tables, suggestColumns: false, ...extra };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function push(tables: Record<string, TableDescriptor>): PushDescriptor {
|
|
23
|
+
return { title: 'Push', description: 'A push', tables };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function configField(type: ConfigField['type'], extra: Partial<ConfigField> = {}): ConfigField {
|
|
27
|
+
return { type, title: 'Field', description: 'A field', placeholder: null, optional: false, ...extra };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function valueField(type: ValueField['type'], extra: Partial<ValueField> = {}): ValueField {
|
|
31
|
+
return { type, title: 'Field', description: 'A field', optional: false, ...extra };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function operation(input: Record<string, ValueField>, output: Record<string, ValueField>, extra: Partial<OperationDescriptor> = {}): OperationDescriptor {
|
|
35
|
+
return { title: 'Operation', description: 'An operation', effect: 'create', risk: 'low', idempotency: 'supported', input, output, ...extra };
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function descriptor(overrides: Partial<ConnectorDescriptor> = {}): ConnectorDescriptor {
|
|
39
|
+
return {
|
|
40
|
+
name: 'demo', version: '1.0.0', title: 'Demo', description: 'Demo connector', icon: null,
|
|
41
|
+
config: {}, hosts: [], db: null, rateLimit: null, check: false, streams: {}, pushes: {}, operations: {},
|
|
42
|
+
...overrides,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* 手写的 CommonJS 连接器产物(与虚拟入口同形,避开打包器的常量折叠),从 guest 的 `require` 取 SDK。
|
|
48
|
+
* `implementations` 是 `implementations` 对象字面量里的成员(如 `streams: { items: defineStream('items', async function* () {}) }`)。
|
|
49
|
+
*/
|
|
50
|
+
export function rawConnector(desc: ConnectorDescriptor, implementations: string, header = ''): string {
|
|
51
|
+
return `const { defineStream, definePush, defineOperation, defineCheck, defineSuggestColumns, errors } = require('@deepexi/datasense-connector');
|
|
52
|
+
${header}
|
|
53
|
+
module.exports = { descriptor: ${JSON.stringify(desc)}, implementations: { ${implementations} } };`;
|
|
54
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 设计 `connector-runtime.md` §4.2 的金蝶云星辰示例,按描述 + 编写 API 改写,用来确认带类型的编写 API 能写出真实连接器:
|
|
3
|
+
* 包的 `ts-check` 覆盖本文件。`D` 按 codegen(`.ontos/types.ts`)的形状手写:顶层 interface、各层 readonly、列与配置按类型映射。
|
|
4
|
+
* HTTP 响应体、断点与会话用显式类型(原文是无类型的 JS 写法)。
|
|
5
|
+
*/
|
|
6
|
+
import { defineCheck, defineStream, errors, type ConnectorContext, type HttpAuth } from '../../src/index';
|
|
7
|
+
|
|
8
|
+
/** 描述 `kingdee_jdy@1.0.0` 的生成类型(`hosts: ['api.kingdee.com', '*.jdy.com']`,`rateLimit: { perSecond: 10, concurrency: 4 }`)。 */
|
|
9
|
+
interface D {
|
|
10
|
+
readonly streams: {
|
|
11
|
+
readonly sales_order: {
|
|
12
|
+
readonly tables: {
|
|
13
|
+
readonly sales_order: {
|
|
14
|
+
readonly id: string; readonly bill_no: string; readonly biz_date: string; readonly modified_at: string; readonly amount: string; readonly raw: unknown;
|
|
15
|
+
};
|
|
16
|
+
readonly sales_order_line: { readonly order_id: string; readonly line_no: number; readonly sku_id: string; readonly qty: string };
|
|
17
|
+
};
|
|
18
|
+
readonly params: Record<never, never>;
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
readonly pushes: Record<never, never>;
|
|
22
|
+
readonly operations: Record<never, never>;
|
|
23
|
+
readonly extensibleStreams: Record<never, never>;
|
|
24
|
+
readonly config: { readonly accountId: string };
|
|
25
|
+
readonly secrets: { readonly clientId: string; readonly clientSecret: string; readonly appKey: string; readonly appSecret: string };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
interface SalesOrderPage { rows: Array<{ id: string; modify_time: string }>; total_page: number }
|
|
29
|
+
interface SalesOrderDetail {
|
|
30
|
+
id: string; bill_no: string; bill_date: string; modify_time: string; total_amount: string;
|
|
31
|
+
material_entity: Array<{ seq: number; material_id: string; qty: string }>;
|
|
32
|
+
}
|
|
33
|
+
interface TokenResponse { errcode: number; data: { 'app-token': string; domain: string } }
|
|
34
|
+
|
|
35
|
+
export const check = defineCheck<D>(async (ctx) => {
|
|
36
|
+
await session(ctx);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
export const salesOrder = defineStream<D, 'sales_order'>('sales_order', async function* (ctx, { mode, state }) {
|
|
40
|
+
const auth = await session(ctx);
|
|
41
|
+
const since = mode === 'incremental' ? (state as { modifiedSince?: string } | undefined)?.modifiedSince : undefined;
|
|
42
|
+
|
|
43
|
+
for await (const page of ctx.paginate.page({
|
|
44
|
+
firstPage: 1,
|
|
45
|
+
request: (n) => ctx.http.get<SalesOrderPage>('/jdy/v2/scm/sal_order', {
|
|
46
|
+
query: { page: n, page_size: 100, modify_start_time: since }, auth,
|
|
47
|
+
}),
|
|
48
|
+
rows: (res) => res.data.rows,
|
|
49
|
+
totalPages: (res) => res.data.total_page,
|
|
50
|
+
})) {
|
|
51
|
+
const details = await ctx.map(page.rows, (row) =>
|
|
52
|
+
ctx.http.get<SalesOrderDetail>('/jdy/v2/scm/sal_order_detail', { query: { id: row.id }, auth }), { concurrency: 4 });
|
|
53
|
+
|
|
54
|
+
for (const d of details) {
|
|
55
|
+
yield ctx.record('sales_order', {
|
|
56
|
+
id: d.data.id, bill_no: d.data.bill_no,
|
|
57
|
+
biz_date: d.data.bill_date,
|
|
58
|
+
modified_at: ctx.time.toUtc(d.data.modify_time, 'Asia/Shanghai'),
|
|
59
|
+
amount: d.data.total_amount, raw: d.data,
|
|
60
|
+
});
|
|
61
|
+
for (const line of d.data.material_entity) {
|
|
62
|
+
yield ctx.record('sales_order_line', {
|
|
63
|
+
order_id: d.data.id, line_no: line.seq, sku_id: line.material_id, qty: line.qty,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
yield ctx.checkpoint({ modifiedSince: page.maxOf((row) => row.modify_time) });
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// 登录换会话:签名由网关用句柄代算,脚本看不到 appSecret 明文
|
|
72
|
+
async function session(ctx: ConnectorContext<D>): Promise<HttpAuth> {
|
|
73
|
+
const cached = await ctx.cache.getOrSet('session', 3000, async () => {
|
|
74
|
+
const signature = await ctx.crypto.hmac('sha256', ctx.secrets.appSecret, ctx.secrets.appKey);
|
|
75
|
+
const res = await ctx.http.get<TokenResponse>('https://api.kingdee.com/jdyconnector/app_management/kingdee_auth_token', {
|
|
76
|
+
query: { app_key: ctx.secrets.appKey, app_signature: signature },
|
|
77
|
+
});
|
|
78
|
+
if (res.data.errcode !== 0) throw new errors.AuthError(String(res.data.errcode));
|
|
79
|
+
return JSON.stringify({ headers: { 'app-token': res.data.data['app-token'] }, baseUrl: `https://${res.data.data.domain}` });
|
|
80
|
+
});
|
|
81
|
+
return JSON.parse(cached) as HttpAuth;
|
|
82
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 重新录制两个验收连接器的测试用例(需要本机 :15432 的 Postgres):
|
|
3
|
+
*
|
|
4
|
+
* bun packages/ontos-connector-sdk/test/fixtures/record-acceptance.ts
|
|
5
|
+
*
|
|
6
|
+
* 输出到 `test/fixtures/connectors/{http-pull,pg-pull}/streams/<数据流>/tests/<用例名>/`,提交进仓库,测试只回放不重录(http-pull 另有一条重录对比)。
|
|
7
|
+
*/
|
|
8
|
+
import { mkdir, rm, writeFile } from 'node:fs/promises';
|
|
9
|
+
import path from 'node:path';
|
|
10
|
+
import type { ConnectorFixtureFile } from '../../src/testing';
|
|
11
|
+
import {
|
|
12
|
+
dropPgSchema, HTTP_CASE_NAME, HTTP_PULL_DIR, PG_CASE_NAME, PG_PULL_DIR, preparePgSchema, recordHttpPull, recordPgPull,
|
|
13
|
+
} from './acceptance';
|
|
14
|
+
|
|
15
|
+
async function write(root: string, name: string, recorded: { files: ConnectorFixtureFile[]; result: { error: unknown; records: unknown[] } }): Promise<void> {
|
|
16
|
+
if (recorded.result.error) throw new Error(`recording ${name} failed: ${JSON.stringify(recorded.result.error)}`);
|
|
17
|
+
for (const file of recorded.files) await rm(path.dirname(path.join(root, file.path)), { recursive: true, force: true });
|
|
18
|
+
for (const file of recorded.files) {
|
|
19
|
+
const target = path.join(root, file.path);
|
|
20
|
+
await mkdir(path.dirname(target), { recursive: true });
|
|
21
|
+
await writeFile(target, file.content);
|
|
22
|
+
}
|
|
23
|
+
console.log(`${name}: ${recorded.result.records.length} record(s), ${recorded.files.map((file) => file.path).join(', ')}`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
await write(HTTP_PULL_DIR, HTTP_CASE_NAME, await recordHttpPull());
|
|
27
|
+
if (!await preparePgSchema()) throw new Error('Postgres on localhost:15432 is not reachable');
|
|
28
|
+
try {
|
|
29
|
+
await write(PG_PULL_DIR, PG_CASE_NAME, await recordPgPull());
|
|
30
|
+
} finally {
|
|
31
|
+
await dropPgSchema();
|
|
32
|
+
}
|