@deepexi/datasense-connector 3.0.1 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -139
- package/package.json +19 -28
- package/scripts/build.ts +188 -0
- package/src/define.ts +267 -0
- package/src/descriptor.ts +116 -0
- package/src/errors.ts +105 -0
- package/src/index.ts +108 -0
- package/src/internal/location.ts +23 -0
- package/src/runtime/guest/bridge.ts +238 -0
- package/src/runtime/guest/cache.ts +43 -0
- package/src/runtime/guest/check.ts +17 -0
- package/src/runtime/guest/crypto.ts +166 -0
- package/src/runtime/guest/ctx.ts +157 -0
- package/src/runtime/guest/db.ts +59 -0
- package/src/runtime/guest/failure.ts +69 -0
- package/src/runtime/guest/http.ts +194 -0
- package/src/runtime/guest/index.ts +102 -0
- package/{runtime/guest/limits.d.ts → src/runtime/guest/limits.ts} +12 -12
- package/src/runtime/guest/loader.ts +210 -0
- package/src/runtime/guest/log.ts +47 -0
- package/src/runtime/guest/map.ts +45 -0
- package/src/runtime/guest/operation.ts +193 -0
- package/src/runtime/guest/paginate.ts +169 -0
- package/src/runtime/guest/pull.ts +177 -0
- package/src/runtime/guest/push.ts +92 -0
- package/src/runtime/guest/signal.ts +14 -0
- package/src/runtime/guest/suggest.ts +51 -0
- package/src/runtime/guest/time.ts +137 -0
- package/src/runtime/guest-runtime.ts +75 -0
- package/src/runtime/index.ts +53 -0
- package/src/runtime/protocol.ts +557 -0
- package/src/runtime/session.ts +216 -0
- package/src/testing/child.ts +21 -0
- package/src/testing/compliance.ts +343 -0
- package/src/testing/engine.ts +445 -0
- package/src/testing/fixtures.ts +591 -0
- package/src/testing/gateway.ts +482 -0
- package/src/testing/import-guard.ts +221 -0
- package/src/testing/index.ts +43 -0
- package/src/testing/local-db.ts +226 -0
- package/src/testing/mock-external-system.ts +249 -0
- package/src/testing/pinned-http.ts +144 -0
- package/src/testing/recorder.ts +140 -0
- package/src/testing/redaction.ts +94 -0
- package/src/testing/replay.ts +437 -0
- package/src/testing/run-tests.ts +115 -0
- package/src/testing/schema.ts +70 -0
- package/src/testing/source-loader.ts +374 -0
- package/src/testing/types.ts +78 -0
- package/src/types.ts +329 -0
- package/src/validate.ts +267 -0
- package/test/build.test.ts +159 -0
- package/test/define.test-d.ts +182 -0
- package/test/fixtures/acceptance.ts +156 -0
- package/test/fixtures/connectors/http-pull/.ontos/descriptor.json +72 -0
- package/test/fixtures/connectors/http-pull/.ontos/types.ts +61 -0
- package/test/fixtures/connectors/http-pull/package.json +5 -0
- package/test/fixtures/connectors/http-pull/streams/items/index.ts +37 -0
- package/test/fixtures/connectors/http-pull/streams/items/map.ts +12 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/case.json +15 -0
- package/test/fixtures/connectors/http-pull/streams/items/tests/basic/exchanges.ndjson +2 -0
- package/test/fixtures/connectors/pg-pull/.ontos/descriptor.json +151 -0
- package/test/fixtures/connectors/pg-pull/.ontos/types.ts +62 -0
- package/test/fixtures/connectors/pg-pull/lib/sql.ts +14 -0
- package/test/fixtures/connectors/pg-pull/package.json +5 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/index.ts +30 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/case.json +15 -0
- package/test/fixtures/connectors/pg-pull/streams/customers/tests/customers-full/exchanges.ndjson +3 -0
- package/test/fixtures/connectors/pg-pull/streams/orders/index.ts +32 -0
- package/test/fixtures/descriptor.ts +54 -0
- package/test/fixtures/kingdee-example.ts +82 -0
- package/test/fixtures/record-acceptance.ts +32 -0
- package/test/runtime/ctx.test.ts +434 -0
- package/test/runtime/helpers.ts +157 -0
- package/test/runtime/jobs.test.ts +444 -0
- package/test/runtime/operation.test.ts +175 -0
- package/test/runtime/session.test.ts +229 -0
- package/test/testing/acceptance-fixtures.test.ts +69 -0
- package/test/testing/compliance.test.ts +247 -0
- package/test/testing/engine.test.ts +283 -0
- package/test/testing/fixtures.test.ts +221 -0
- package/test/testing/gateway.test.ts +337 -0
- package/test/testing/import-guard.test.ts +45 -0
- package/test/testing/mock-external-system.test.ts +144 -0
- package/test/testing/operation-fixtures.test.ts +160 -0
- package/test/testing/recorder.test.ts +163 -0
- package/test/testing/redaction.test.ts +54 -0
- package/test/testing/replay.test.ts +207 -0
- package/test/testing/run-tests.test.ts +137 -0
- package/test/testing/source-loader.test.ts +220 -0
- package/test/validate-config.test.ts +104 -0
- package/tsconfig.build.json +23 -0
- package/tsconfig.json +19 -0
- package/cjs/index.cjs +0 -1015
- package/cjs/runtime.cjs +0 -752
- package/cjs/testing.cjs +0 -3591
- package/errors.d.ts +0 -17
- package/errors.js +0 -53
- package/guest.cjs +0 -2659
- package/index.d.ts +0 -13
- package/index.js +0 -7
- package/internal/location.d.ts +0 -6
- package/internal/location.js +0 -8
- package/presentation.d.ts +0 -38
- package/presentation.js +0 -268
- package/runtime/guest/bridge.d.ts +0 -53
- package/runtime/guest/bridge.js +0 -187
- package/runtime/guest/cache.d.ts +0 -3
- package/runtime/guest/cache.js +0 -42
- package/runtime/guest/crypto.d.ts +0 -13
- package/runtime/guest/crypto.js +0 -177
- package/runtime/guest/ctx.d.ts +0 -30
- package/runtime/guest/ctx.js +0 -124
- package/runtime/guest/db.d.ts +0 -6
- package/runtime/guest/db.js +0 -59
- package/runtime/guest/discover.d.ts +0 -29
- package/runtime/guest/discover.js +0 -51
- package/runtime/guest/failure.d.ts +0 -24
- package/runtime/guest/failure.js +0 -63
- package/runtime/guest/http.d.ts +0 -4
- package/runtime/guest/http.js +0 -204
- package/runtime/guest/index.d.ts +0 -12
- package/runtime/guest/index.js +0 -89
- package/runtime/guest/limits.js +0 -12
- package/runtime/guest/loader.d.ts +0 -15
- package/runtime/guest/loader.js +0 -42
- package/runtime/guest/log.d.ts +0 -10
- package/runtime/guest/log.js +0 -36
- package/runtime/guest/map.d.ts +0 -4
- package/runtime/guest/map.js +0 -45
- package/runtime/guest/paginate.d.ts +0 -6
- package/runtime/guest/paginate.js +0 -180
- package/runtime/guest/pull.d.ts +0 -12
- package/runtime/guest/pull.js +0 -150
- package/runtime/guest/push.d.ts +0 -18
- package/runtime/guest/push.js +0 -80
- package/runtime/guest/signal.d.ts +0 -3
- package/runtime/guest/signal.js +0 -12
- package/runtime/guest/time.d.ts +0 -5
- package/runtime/guest/time.js +0 -127
- package/runtime/guest-runtime.d.ts +0 -7
- package/runtime/guest-runtime.js +0 -51
- package/runtime/index.d.ts +0 -18
- package/runtime/index.js +0 -3
- package/runtime/protocol.d.ts +0 -276
- package/runtime/protocol.js +0 -409
- package/runtime/session.d.ts +0 -12
- package/runtime/session.js +0 -191
- package/t.d.ts +0 -18
- package/t.js +0 -26
- package/testing/child.cjs +0 -350
- package/testing/child.d.ts +0 -1
- package/testing/child.js +0 -12
- package/testing/compliance.d.ts +0 -41
- package/testing/compliance.js +0 -291
- package/testing/engine.d.ts +0 -66
- package/testing/engine.js +0 -339
- package/testing/fixtures.d.ts +0 -145
- package/testing/fixtures.js +0 -407
- package/testing/gateway.d.ts +0 -65
- package/testing/gateway.js +0 -402
- package/testing/import-guard.d.ts +0 -57
- package/testing/import-guard.js +0 -187
- package/testing/index.d.ts +0 -33
- package/testing/index.js +0 -10
- package/testing/local-db.d.ts +0 -27
- package/testing/local-db.js +0 -181
- package/testing/pinned-http.d.ts +0 -14
- package/testing/pinned-http.js +0 -125
- package/testing/recorder.d.ts +0 -15
- package/testing/recorder.js +0 -116
- package/testing/redaction.d.ts +0 -28
- package/testing/redaction.js +0 -67
- package/testing/replay.d.ts +0 -50
- package/testing/replay.js +0 -313
- package/testing/schema.d.ts +0 -13
- package/testing/schema.js +0 -65
- package/testing/source-loader.d.ts +0 -19
- package/testing/source-loader.js +0 -238
- package/testing/types.d.ts +0 -79
- package/testing/types.js +0 -18
- package/types.d.ts +0 -302
- package/types.js +0 -13
- package/validate.d.ts +0 -78
- package/validate.js +0 -597
package/validate.js
DELETED
|
@@ -1,597 +0,0 @@
|
|
|
1
|
-
import { isSingleObjectPush, SINGLE_OBJECT_PUSH_MESSAGE } from './types.js';
|
|
2
|
-
export const SDK_NAME_PATTERN = /^[a-z][a-z0-9_]{0,27}$/;
|
|
3
|
-
export const RESERVED_PUSH_NAMES = Object.freeze(['dev']);
|
|
4
|
-
export const DEFAULT_PUSH_NAME = 'default';
|
|
5
|
-
const VERSION_PATTERN = /^\d+\.\d+\.\d+$/;
|
|
6
|
-
const SCALAR_KINDS = new Set(['string', 'integer', 'decimal', 'boolean', 'date', 'instant', 'json']);
|
|
7
|
-
const RETENTION_KINDS = new Set(['instant', 'date']);
|
|
8
|
-
const SYNC_MODES = new Set(['full', 'incremental']);
|
|
9
|
-
const SNAPSHOTS = new Set(['complete', 'partial']);
|
|
10
|
-
const PURGE_KEYS = ['tenant', 'user'];
|
|
11
|
-
const FIELD_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
12
|
-
const HOST_PLACEHOLDER = /\{config\.([^{}]*)\}/g;
|
|
13
|
-
const HOST_LABEL = '[a-z0-9](?:[a-z0-9-]*[a-z0-9])?';
|
|
14
|
-
const HOST_PATTERN = new RegExp(`^(?:\\*\\.)?${HOST_LABEL}(?:\\.${HOST_LABEL})*$`, 'i');
|
|
15
|
-
const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
|
|
16
|
-
const INSTANT_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2})$/;
|
|
17
|
-
const DECIMAL_TEXT_PATTERN = /^-?\d+(?:\.\d+)?$/;
|
|
18
|
-
export const PRESENTATION_LIMITS = Object.freeze({
|
|
19
|
-
title: 40,
|
|
20
|
-
description: 120,
|
|
21
|
-
readmeBytes: 65536,
|
|
22
|
-
iconBytes: 262144,
|
|
23
|
-
iconMinPx: 128,
|
|
24
|
-
fieldTitle: 40,
|
|
25
|
-
fieldDescription: 200,
|
|
26
|
-
placeholder: 100,
|
|
27
|
-
tableTitle: 40,
|
|
28
|
-
tableDescription: 200,
|
|
29
|
-
});
|
|
30
|
-
function isRecord(value) {
|
|
31
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
32
|
-
}
|
|
33
|
-
function isPositiveInteger(value) {
|
|
34
|
-
return typeof value === 'number' && Number.isInteger(value) && value > 0;
|
|
35
|
-
}
|
|
36
|
-
function isStringArray(value) {
|
|
37
|
-
return Array.isArray(value) && value.every((item) => typeof item === 'string');
|
|
38
|
-
}
|
|
39
|
-
function validateText(path, owner, key, max, issues) {
|
|
40
|
-
const value = owner[key];
|
|
41
|
-
if (value === undefined)
|
|
42
|
-
return;
|
|
43
|
-
if (typeof value !== 'string') {
|
|
44
|
-
issues.push({ path: `${path}.${key}`, message: `${key} must be a string` });
|
|
45
|
-
}
|
|
46
|
-
else if ([...value].length > max) {
|
|
47
|
-
issues.push({ path: `${path}.${key}`, message: `${key} must be at most ${max} characters` });
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
function validateFieldMeta(path, type, issues) {
|
|
51
|
-
validateText(path, type, 'title', PRESENTATION_LIMITS.fieldTitle, issues);
|
|
52
|
-
validateText(path, type, 'description', PRESENTATION_LIMITS.fieldDescription, issues);
|
|
53
|
-
if (type.placeholder !== undefined && (type.kind === 'object' || type.kind === 'array')) {
|
|
54
|
-
issues.push({ path: `${path}.placeholder`, message: 'placeholder is only allowed on scalar and secret fields' });
|
|
55
|
-
return;
|
|
56
|
-
}
|
|
57
|
-
validateText(path, type, 'placeholder', PRESENTATION_LIMITS.placeholder, issues);
|
|
58
|
-
}
|
|
59
|
-
function columnTypeProblem(type) {
|
|
60
|
-
if (!isRecord(type) || typeof type.kind !== 'string' || !SCALAR_KINDS.has(type.kind)) {
|
|
61
|
-
return 'column type must be a scalar type (string, integer, decimal, boolean, date, instant or json)';
|
|
62
|
-
}
|
|
63
|
-
if (type.kind === 'decimal')
|
|
64
|
-
return decimalProblem(type);
|
|
65
|
-
return null;
|
|
66
|
-
}
|
|
67
|
-
function decimalProblem(type) {
|
|
68
|
-
const { precision, scale } = type;
|
|
69
|
-
if (!isPositiveInteger(precision) || typeof scale !== 'number' || !Number.isInteger(scale) || scale < 0 || scale > precision) {
|
|
70
|
-
return 'decimal needs an integer precision > 0 and an integer scale between 0 and precision';
|
|
71
|
-
}
|
|
72
|
-
return null;
|
|
73
|
-
}
|
|
74
|
-
function validateTable(path, table, issues) {
|
|
75
|
-
if (!isRecord(table)) {
|
|
76
|
-
issues.push({ path, message: 'table declaration must be an object' });
|
|
77
|
-
return;
|
|
78
|
-
}
|
|
79
|
-
const columns = isRecord(table.columns) ? table.columns : null;
|
|
80
|
-
if (!columns) {
|
|
81
|
-
issues.push({ path: `${path}.columns`, message: 'columns must be an object' });
|
|
82
|
-
return;
|
|
83
|
-
}
|
|
84
|
-
const columnKind = (name) => {
|
|
85
|
-
const type = columns[name];
|
|
86
|
-
return isRecord(type) && typeof type.kind === 'string' ? type.kind : undefined;
|
|
87
|
-
};
|
|
88
|
-
const isDeclared = (name) => typeof name === 'string' && Object.hasOwn(columns, name);
|
|
89
|
-
for (const [name, type] of Object.entries(columns)) {
|
|
90
|
-
const problem = columnTypeProblem(type);
|
|
91
|
-
if (problem)
|
|
92
|
-
issues.push({ path: `${path}.columns.${name}`, message: problem });
|
|
93
|
-
else
|
|
94
|
-
validateFieldMeta(`${path}.columns.${name}`, type, issues);
|
|
95
|
-
}
|
|
96
|
-
validateText(path, table, 'title', PRESENTATION_LIMITS.tableTitle, issues);
|
|
97
|
-
validateText(path, table, 'description', PRESENTATION_LIMITS.tableDescription, issues);
|
|
98
|
-
const primaryKey = table.primaryKey;
|
|
99
|
-
if (!isStringArray(primaryKey) || primaryKey.length === 0) {
|
|
100
|
-
issues.push({ path: `${path}.primaryKey`, message: 'primaryKey must be a non-empty list of column names' });
|
|
101
|
-
}
|
|
102
|
-
else {
|
|
103
|
-
const missing = primaryKey.filter((name) => !isDeclared(name));
|
|
104
|
-
if (missing.length > 0) {
|
|
105
|
-
issues.push({ path: `${path}.primaryKey`, message: `primaryKey references undeclared columns: ${missing.join(', ')}` });
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
if (table.retention !== undefined) {
|
|
109
|
-
const retention = table.retention;
|
|
110
|
-
if (!isRecord(retention)) {
|
|
111
|
-
issues.push({ path: `${path}.retention`, message: 'retention must be an object' });
|
|
112
|
-
}
|
|
113
|
-
else {
|
|
114
|
-
const column = retention.column;
|
|
115
|
-
if (!isDeclared(column)) {
|
|
116
|
-
issues.push({ path: `${path}.retention.column`, message: 'retention column must be a declared column' });
|
|
117
|
-
}
|
|
118
|
-
else if (!RETENTION_KINDS.has(columnKind(column) ?? '')) {
|
|
119
|
-
issues.push({ path: `${path}.retention.column`, message: 'retention column must be of type instant or date' });
|
|
120
|
-
}
|
|
121
|
-
else if (!isStringArray(primaryKey) || !primaryKey.includes(column)) {
|
|
122
|
-
issues.push({ path: `${path}.retention.column`, message: 'retention column must be part of primaryKey' });
|
|
123
|
-
}
|
|
124
|
-
if (!isPositiveInteger(retention.days)) {
|
|
125
|
-
issues.push({ path: `${path}.retention.days`, message: 'retention days must be a positive integer' });
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
if (table.purgeColumns !== undefined) {
|
|
130
|
-
const purgeColumns = table.purgeColumns;
|
|
131
|
-
if (!isRecord(purgeColumns)) {
|
|
132
|
-
issues.push({ path: `${path}.purgeColumns`, message: 'purgeColumns must be an object' });
|
|
133
|
-
}
|
|
134
|
-
else {
|
|
135
|
-
for (const key of PURGE_KEYS) {
|
|
136
|
-
const column = purgeColumns[key];
|
|
137
|
-
if (column !== undefined && !isDeclared(column)) {
|
|
138
|
-
issues.push({ path: `${path}.purgeColumns.${key}`, message: 'purge column must be a declared column' });
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
if (table.indexes !== undefined) {
|
|
144
|
-
const indexes = table.indexes;
|
|
145
|
-
if (!Array.isArray(indexes)) {
|
|
146
|
-
issues.push({ path: `${path}.indexes`, message: 'indexes must be a list of column lists' });
|
|
147
|
-
}
|
|
148
|
-
else {
|
|
149
|
-
indexes.forEach((index, i) => {
|
|
150
|
-
if (!isStringArray(index) || index.length === 0 || !index.every(isDeclared)) {
|
|
151
|
-
issues.push({ path: `${path}.indexes[${i}]`, message: 'index must be a non-empty list of declared columns' });
|
|
152
|
-
}
|
|
153
|
-
});
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
function validateTables(path, tables, owners, owner, issues) {
|
|
158
|
-
if (!isRecord(tables)) {
|
|
159
|
-
issues.push({ path, message: 'tables must be an object' });
|
|
160
|
-
return;
|
|
161
|
-
}
|
|
162
|
-
for (const [name, table] of Object.entries(tables)) {
|
|
163
|
-
const tablePath = `${path}.${name}`;
|
|
164
|
-
if (!SDK_NAME_PATTERN.test(name)) {
|
|
165
|
-
issues.push({ path: tablePath, message: `table name must match ${SDK_NAME_PATTERN.source}` });
|
|
166
|
-
}
|
|
167
|
-
const previous = owners.get(name);
|
|
168
|
-
if (previous !== undefined) {
|
|
169
|
-
issues.push({ path: tablePath, message: `table ${name} is already declared by ${previous}` });
|
|
170
|
-
}
|
|
171
|
-
else {
|
|
172
|
-
owners.set(name, owner);
|
|
173
|
-
}
|
|
174
|
-
validateTable(tablePath, table, issues);
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
function validateSdkType(path, type, rules, issues) {
|
|
178
|
-
if (!isRecord(type) || typeof type.kind !== 'string') {
|
|
179
|
-
issues.push({ path, message: 'type must be declared with t.*()' });
|
|
180
|
-
return;
|
|
181
|
-
}
|
|
182
|
-
const nested = { secretAllowed: false, secretMessage: rules.secretMessage };
|
|
183
|
-
if (type.kind === 'secret' || type.kind === 'object' || type.kind === 'array' || SCALAR_KINDS.has(type.kind)) {
|
|
184
|
-
validateFieldMeta(path, type, issues);
|
|
185
|
-
}
|
|
186
|
-
switch (type.kind) {
|
|
187
|
-
case 'secret':
|
|
188
|
-
if (!rules.secretAllowed)
|
|
189
|
-
issues.push({ path, message: rules.secretMessage });
|
|
190
|
-
else if (type.exposeToScript !== undefined && typeof type.exposeToScript !== 'boolean') {
|
|
191
|
-
issues.push({ path: `${path}.exposeToScript`, message: 'exposeToScript must be a boolean' });
|
|
192
|
-
}
|
|
193
|
-
return;
|
|
194
|
-
case 'object':
|
|
195
|
-
validateShape(path, type, nested, issues);
|
|
196
|
-
return;
|
|
197
|
-
case 'array':
|
|
198
|
-
validateSdkType(`${path}[]`, type.item, nested, issues);
|
|
199
|
-
return;
|
|
200
|
-
case 'decimal': {
|
|
201
|
-
const problem = decimalProblem(type);
|
|
202
|
-
if (problem)
|
|
203
|
-
issues.push({ path, message: problem });
|
|
204
|
-
return;
|
|
205
|
-
}
|
|
206
|
-
default:
|
|
207
|
-
if (!SCALAR_KINDS.has(type.kind))
|
|
208
|
-
issues.push({ path, message: `unknown type kind ${type.kind}` });
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
function validateShape(path, type, fieldRules, issues) {
|
|
212
|
-
if (!isRecord(type.shape)) {
|
|
213
|
-
issues.push({ path, message: 'object type needs a shape' });
|
|
214
|
-
return;
|
|
215
|
-
}
|
|
216
|
-
for (const [field, fieldType] of Object.entries(type.shape)) {
|
|
217
|
-
const fieldPath = `${path}.${field}`;
|
|
218
|
-
if (!FIELD_NAME_PATTERN.test(field)) {
|
|
219
|
-
issues.push({ path: fieldPath, message: `field name must match ${FIELD_NAME_PATTERN.source}` });
|
|
220
|
-
}
|
|
221
|
-
validateSdkType(fieldPath, fieldType, fieldRules, issues);
|
|
222
|
-
}
|
|
223
|
-
}
|
|
224
|
-
function validateObjectDeclaration(path, type, rules, issues) {
|
|
225
|
-
if (!isRecord(type) || type.kind !== 'object') {
|
|
226
|
-
issues.push({ path, message: `${path} must be declared with t.object()` });
|
|
227
|
-
return false;
|
|
228
|
-
}
|
|
229
|
-
validateFieldMeta(path, type, issues);
|
|
230
|
-
validateShape(path, type, rules, issues);
|
|
231
|
-
return true;
|
|
232
|
-
}
|
|
233
|
-
function configFieldType(config, fieldPath) {
|
|
234
|
-
let current = config;
|
|
235
|
-
for (const segment of fieldPath.split('.')) {
|
|
236
|
-
if (!isRecord(current) || current.kind !== 'object' || !isRecord(current.shape) || !Object.hasOwn(current.shape, segment))
|
|
237
|
-
return undefined;
|
|
238
|
-
current = current.shape[segment];
|
|
239
|
-
}
|
|
240
|
-
return isRecord(current) ? current : undefined;
|
|
241
|
-
}
|
|
242
|
-
function pushNames(push) {
|
|
243
|
-
return isRecord(push) ? Object.keys(push) : [];
|
|
244
|
-
}
|
|
245
|
-
function validatePushHandler(path, handler, issues) {
|
|
246
|
-
if (!isRecord(handler)) {
|
|
247
|
-
issues.push({ path, message: 'push handler must be an object with map()' });
|
|
248
|
-
return;
|
|
249
|
-
}
|
|
250
|
-
if (handler.verify !== undefined && typeof handler.verify !== 'function') {
|
|
251
|
-
issues.push({ path: `${path}.verify`, message: `${path}.verify must be a function` });
|
|
252
|
-
}
|
|
253
|
-
if (typeof handler.map !== 'function') {
|
|
254
|
-
issues.push({ path: `${path}.map`, message: `${path}.map must be a function` });
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
function validatePush(push, issues) {
|
|
258
|
-
if (push === undefined)
|
|
259
|
-
return 0;
|
|
260
|
-
if (!isRecord(push)) {
|
|
261
|
-
issues.push({ path: 'push', message: 'push must be an object' });
|
|
262
|
-
return 0;
|
|
263
|
-
}
|
|
264
|
-
if (isSingleObjectPush(push)) {
|
|
265
|
-
issues.push({ path: 'push', message: SINGLE_OBJECT_PUSH_MESSAGE });
|
|
266
|
-
return 1;
|
|
267
|
-
}
|
|
268
|
-
const names = Object.keys(push);
|
|
269
|
-
if (names.length === 0) {
|
|
270
|
-
issues.push({ path: 'push', message: 'push must declare at least one handler, e.g. push: { default: { map } }' });
|
|
271
|
-
return 0;
|
|
272
|
-
}
|
|
273
|
-
for (const name of names) {
|
|
274
|
-
const path = `push.${name}`;
|
|
275
|
-
if (!SDK_NAME_PATTERN.test(name)) {
|
|
276
|
-
issues.push({ path, message: `push name must match ${SDK_NAME_PATTERN.source}` });
|
|
277
|
-
}
|
|
278
|
-
else if (RESERVED_PUSH_NAMES.includes(name)) {
|
|
279
|
-
issues.push({ path, message: `push name ${name} is reserved` });
|
|
280
|
-
}
|
|
281
|
-
validatePushHandler(path, push[name], issues);
|
|
282
|
-
}
|
|
283
|
-
return names.length;
|
|
284
|
-
}
|
|
285
|
-
function validateHosts(hosts, config, issues) {
|
|
286
|
-
if (hosts === undefined)
|
|
287
|
-
return 0;
|
|
288
|
-
if (!Array.isArray(hosts)) {
|
|
289
|
-
issues.push({ path: 'hosts', message: 'hosts must be a list of host names' });
|
|
290
|
-
return 0;
|
|
291
|
-
}
|
|
292
|
-
hosts.forEach((host, i) => {
|
|
293
|
-
const path = `hosts[${i}]`;
|
|
294
|
-
if (typeof host !== 'string' || host.length === 0 || host.length > 253) {
|
|
295
|
-
issues.push({ path, message: 'host must be a non-empty string' });
|
|
296
|
-
return;
|
|
297
|
-
}
|
|
298
|
-
for (const match of host.matchAll(HOST_PLACEHOLDER)) {
|
|
299
|
-
const fieldPath = match[1] ?? '';
|
|
300
|
-
const fieldType = configFieldType(config, fieldPath);
|
|
301
|
-
if (!fieldType) {
|
|
302
|
-
issues.push({ path, message: `host template references undeclared config field ${fieldPath}` });
|
|
303
|
-
}
|
|
304
|
-
else if (fieldType.kind !== 'string') {
|
|
305
|
-
issues.push({ path, message: `host template field ${fieldPath} must be declared with t.string()` });
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
if (!HOST_PATTERN.test(host.replace(HOST_PLACEHOLDER, 'x'))) {
|
|
309
|
-
issues.push({ path, message: 'host must be an exact host name or *.suffix, without scheme, port or path' });
|
|
310
|
-
}
|
|
311
|
-
});
|
|
312
|
-
return hosts.length;
|
|
313
|
-
}
|
|
314
|
-
function validateRateLimit(rateLimit, issues) {
|
|
315
|
-
if (rateLimit === undefined)
|
|
316
|
-
return;
|
|
317
|
-
if (!isRecord(rateLimit)) {
|
|
318
|
-
issues.push({ path: 'rateLimit', message: 'rateLimit must be { perSecond, concurrency }' });
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
|
-
const { perSecond, concurrency } = rateLimit;
|
|
322
|
-
if (typeof perSecond !== 'number' || !Number.isFinite(perSecond) || perSecond <= 0) {
|
|
323
|
-
issues.push({ path: 'rateLimit.perSecond', message: 'rateLimit.perSecond must be a positive number' });
|
|
324
|
-
}
|
|
325
|
-
if (!isPositiveInteger(concurrency)) {
|
|
326
|
-
issues.push({ path: 'rateLimit.concurrency', message: 'rateLimit.concurrency must be a positive integer' });
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
function validateStreams(streams, issues) {
|
|
330
|
-
if (streams === undefined)
|
|
331
|
-
return 0;
|
|
332
|
-
if (!isRecord(streams)) {
|
|
333
|
-
issues.push({ path: 'streams', message: 'streams must be an object of async generator functions' });
|
|
334
|
-
return 0;
|
|
335
|
-
}
|
|
336
|
-
for (const [name, stream] of Object.entries(streams)) {
|
|
337
|
-
const path = `streams.${name}`;
|
|
338
|
-
if (!SDK_NAME_PATTERN.test(name))
|
|
339
|
-
issues.push({ path, message: `stream name must match ${SDK_NAME_PATTERN.source}` });
|
|
340
|
-
if (typeof stream !== 'function')
|
|
341
|
-
issues.push({ path, message: `${path} must be an async generator function` });
|
|
342
|
-
}
|
|
343
|
-
return Object.keys(streams).length;
|
|
344
|
-
}
|
|
345
|
-
export function validateConnectorDefinition(definition) {
|
|
346
|
-
if (!isRecord(definition)) {
|
|
347
|
-
return [{ path: '', message: 'connector definition must be an object' }];
|
|
348
|
-
}
|
|
349
|
-
const issues = [];
|
|
350
|
-
if (typeof definition.name !== 'string' || !SDK_NAME_PATTERN.test(definition.name)) {
|
|
351
|
-
issues.push({ path: 'name', message: `name must match ${SDK_NAME_PATTERN.source}` });
|
|
352
|
-
}
|
|
353
|
-
if (typeof definition.version !== 'string' || !VERSION_PATTERN.test(definition.version)) {
|
|
354
|
-
issues.push({ path: 'version', message: 'version must be a semver string like 1.0.0' });
|
|
355
|
-
}
|
|
356
|
-
validateObjectDeclaration('config', definition.config, {
|
|
357
|
-
secretAllowed: true,
|
|
358
|
-
secretMessage: 'secret fields must be top-level config fields',
|
|
359
|
-
}, issues);
|
|
360
|
-
const hostCount = validateHosts(definition.hosts, definition.config, issues);
|
|
361
|
-
const hasDb = definition.db !== undefined;
|
|
362
|
-
if (hasDb && (!isRecord(definition.db) || definition.db.engine !== 'postgresql')) {
|
|
363
|
-
issues.push({ path: 'db.engine', message: "db.engine must be 'postgresql'" });
|
|
364
|
-
}
|
|
365
|
-
validateRateLimit(definition.rateLimit, issues);
|
|
366
|
-
if (definition.check !== undefined && typeof definition.check !== 'function') {
|
|
367
|
-
issues.push({ path: 'check', message: 'check must be a function' });
|
|
368
|
-
}
|
|
369
|
-
if (typeof definition.discover !== 'function') {
|
|
370
|
-
issues.push({ path: 'discover', message: 'discover must be a function' });
|
|
371
|
-
}
|
|
372
|
-
if (definition.writes !== undefined) {
|
|
373
|
-
issues.push({ path: 'writes', message: 'external write operations are not available yet' });
|
|
374
|
-
}
|
|
375
|
-
const streamCount = validateStreams(definition.streams, issues);
|
|
376
|
-
const pushCount = validatePush(definition.push, issues);
|
|
377
|
-
if (streamCount === 0 && pushCount === 0) {
|
|
378
|
-
issues.push({ path: 'streams', message: 'connector must declare at least one stream or push' });
|
|
379
|
-
}
|
|
380
|
-
if (streamCount > 0 && hostCount === 0 && !hasDb) {
|
|
381
|
-
issues.push({ path: 'hosts', message: 'connectors with streams must declare hosts (for ctx.http) or db (for ctx.db)' });
|
|
382
|
-
}
|
|
383
|
-
return issues;
|
|
384
|
-
}
|
|
385
|
-
function validateStreamDiscovery(path, stream, owners, issues) {
|
|
386
|
-
if (!isRecord(stream)) {
|
|
387
|
-
issues.push({ path, message: 'stream declaration must be an object' });
|
|
388
|
-
return;
|
|
389
|
-
}
|
|
390
|
-
validateTables(`${path}.tables`, stream.tables, owners, path, issues);
|
|
391
|
-
const syncModes = stream.syncModes;
|
|
392
|
-
if (!isStringArray(syncModes) || syncModes.length === 0 || !syncModes.every((mode) => SYNC_MODES.has(mode)) || new Set(syncModes).size !== syncModes.length) {
|
|
393
|
-
issues.push({ path: `${path}.syncModes`, message: "syncModes must be a non-empty list of 'full' and/or 'incremental'" });
|
|
394
|
-
}
|
|
395
|
-
if (stream.snapshot !== undefined && (typeof stream.snapshot !== 'string' || !SNAPSHOTS.has(stream.snapshot))) {
|
|
396
|
-
issues.push({ path: `${path}.snapshot`, message: "snapshot must be 'complete' or 'partial'" });
|
|
397
|
-
}
|
|
398
|
-
if (stream.params !== undefined) {
|
|
399
|
-
validateObjectDeclaration(`${path}.params`, stream.params, {
|
|
400
|
-
secretAllowed: false,
|
|
401
|
-
secretMessage: 'stream params cannot contain secret fields',
|
|
402
|
-
}, issues);
|
|
403
|
-
}
|
|
404
|
-
}
|
|
405
|
-
function definitionStreamNames(definition) {
|
|
406
|
-
if (!isRecord(definition) || !isRecord(definition.streams))
|
|
407
|
-
return [];
|
|
408
|
-
return Object.keys(definition.streams);
|
|
409
|
-
}
|
|
410
|
-
function definitionPushNames(definition) {
|
|
411
|
-
if (!isRecord(definition))
|
|
412
|
-
return [];
|
|
413
|
-
return pushNames(definition.push);
|
|
414
|
-
}
|
|
415
|
-
export function validateDiscoverResult(result, definition) {
|
|
416
|
-
if (!isRecord(result)) {
|
|
417
|
-
return [{ path: '', message: 'discover result must be an object' }];
|
|
418
|
-
}
|
|
419
|
-
const issues = [];
|
|
420
|
-
const owners = new Map();
|
|
421
|
-
const declaredStreams = definitionStreamNames(definition);
|
|
422
|
-
const streams = result.streams === undefined ? {} : result.streams;
|
|
423
|
-
if (!isRecord(streams)) {
|
|
424
|
-
issues.push({ path: 'streams', message: 'streams must be an object' });
|
|
425
|
-
}
|
|
426
|
-
else {
|
|
427
|
-
for (const [name, stream] of Object.entries(streams)) {
|
|
428
|
-
const path = `streams.${name}`;
|
|
429
|
-
if (!declaredStreams.includes(name)) {
|
|
430
|
-
issues.push({ path, message: `stream ${name} is not defined in the connector's streams` });
|
|
431
|
-
}
|
|
432
|
-
validateStreamDiscovery(path, stream, owners, issues);
|
|
433
|
-
}
|
|
434
|
-
for (const name of declaredStreams) {
|
|
435
|
-
if (!Object.hasOwn(streams, name)) {
|
|
436
|
-
issues.push({ path: `streams.${name}`, message: `stream ${name} is defined but missing from the discover result` });
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
const declaredPushes = definitionPushNames(definition);
|
|
441
|
-
const pushes = result.push === undefined ? {} : result.push;
|
|
442
|
-
if (!isRecord(pushes)) {
|
|
443
|
-
issues.push({ path: 'push', message: 'push must be an object of { tables } by push name' });
|
|
444
|
-
}
|
|
445
|
-
else {
|
|
446
|
-
for (const [name, push] of Object.entries(pushes)) {
|
|
447
|
-
const path = `push.${name}`;
|
|
448
|
-
if (!declaredPushes.includes(name)) {
|
|
449
|
-
issues.push({ path, message: `push ${name} is not defined in the connector's push handlers` });
|
|
450
|
-
}
|
|
451
|
-
if (!isRecord(push)) {
|
|
452
|
-
issues.push({ path, message: 'push declaration must be { tables }' });
|
|
453
|
-
continue;
|
|
454
|
-
}
|
|
455
|
-
validateTables(`${path}.tables`, push.tables, owners, path, issues);
|
|
456
|
-
}
|
|
457
|
-
for (const name of declaredPushes) {
|
|
458
|
-
if (!Object.hasOwn(pushes, name)) {
|
|
459
|
-
issues.push({ path: `push.${name}`, message: `push ${name} is defined but missing from the discover result` });
|
|
460
|
-
}
|
|
461
|
-
}
|
|
462
|
-
}
|
|
463
|
-
return issues;
|
|
464
|
-
}
|
|
465
|
-
export function normalizeDiscoverResult(result) {
|
|
466
|
-
const streams = {};
|
|
467
|
-
for (const [name, stream] of Object.entries(result.streams ?? {})) {
|
|
468
|
-
streams[name] = {
|
|
469
|
-
tables: stream.tables,
|
|
470
|
-
syncModes: [...stream.syncModes],
|
|
471
|
-
snapshot: stream.snapshot ?? 'partial',
|
|
472
|
-
...(stream.params ? { params: stream.params } : {}),
|
|
473
|
-
};
|
|
474
|
-
}
|
|
475
|
-
const pushes = {};
|
|
476
|
-
for (const [name, push] of Object.entries(result.push ?? {}))
|
|
477
|
-
pushes[name] = { tables: push.tables };
|
|
478
|
-
return { streams, pushes };
|
|
479
|
-
}
|
|
480
|
-
function topLevelSecrets(config) {
|
|
481
|
-
if (!isRecord(config) || !isRecord(config.shape))
|
|
482
|
-
return [];
|
|
483
|
-
const secrets = [];
|
|
484
|
-
for (const [field, type] of Object.entries(config.shape)) {
|
|
485
|
-
if (isRecord(type) && type.kind === 'secret')
|
|
486
|
-
secrets.push({ field, exposeToScript: type.exposeToScript === true });
|
|
487
|
-
}
|
|
488
|
-
return secrets;
|
|
489
|
-
}
|
|
490
|
-
export function summarizeDefinition(definition) {
|
|
491
|
-
const secrets = topLevelSecrets(definition.config);
|
|
492
|
-
const rateLimit = definition.rateLimit;
|
|
493
|
-
return {
|
|
494
|
-
name: definition.name,
|
|
495
|
-
version: definition.version,
|
|
496
|
-
hosts: Array.isArray(definition.hosts) ? definition.hosts.filter((host) => typeof host === 'string') : [],
|
|
497
|
-
db: definition.db?.engine === 'postgresql' ? { engine: 'postgresql' } : null,
|
|
498
|
-
rateLimit: rateLimit ? { perSecond: rateLimit.perSecond, concurrency: rateLimit.concurrency } : null,
|
|
499
|
-
pushes: pushNames(definition.push),
|
|
500
|
-
streams: definitionStreamNames(definition),
|
|
501
|
-
secretFields: secrets.map((secret) => secret.field),
|
|
502
|
-
exposeToScript: secrets.filter((secret) => secret.exposeToScript).map((secret) => secret.field),
|
|
503
|
-
...(isRecord(definition.config) ? { config: JSON.parse(JSON.stringify(definition.config)) } : {}),
|
|
504
|
-
};
|
|
505
|
-
}
|
|
506
|
-
function isValidDate(text) {
|
|
507
|
-
if (!DATE_PATTERN.test(text))
|
|
508
|
-
return false;
|
|
509
|
-
const date = new Date(`${text}T00:00:00Z`);
|
|
510
|
-
return !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === text;
|
|
511
|
-
}
|
|
512
|
-
function validateValue(path, type, value, issues) {
|
|
513
|
-
if (!isRecord(type) || typeof type.kind !== 'string') {
|
|
514
|
-
issues.push({ path, message: 'field type is not a valid t.*() declaration' });
|
|
515
|
-
return;
|
|
516
|
-
}
|
|
517
|
-
const fail = (expected) => issues.push({ path, message: `${path} must be ${expected}` });
|
|
518
|
-
switch (type.kind) {
|
|
519
|
-
case 'string':
|
|
520
|
-
if (typeof value !== 'string')
|
|
521
|
-
fail('a string');
|
|
522
|
-
return;
|
|
523
|
-
case 'integer':
|
|
524
|
-
if (typeof value !== 'number' || !Number.isSafeInteger(value))
|
|
525
|
-
fail('an integer');
|
|
526
|
-
return;
|
|
527
|
-
case 'decimal':
|
|
528
|
-
if (!(typeof value === 'number' && Number.isFinite(value)) && !(typeof value === 'string' && DECIMAL_TEXT_PATTERN.test(value)))
|
|
529
|
-
fail('a decimal number');
|
|
530
|
-
return;
|
|
531
|
-
case 'boolean':
|
|
532
|
-
if (typeof value !== 'boolean')
|
|
533
|
-
fail('a boolean');
|
|
534
|
-
return;
|
|
535
|
-
case 'date':
|
|
536
|
-
if (typeof value !== 'string' || !isValidDate(value))
|
|
537
|
-
fail('a date like 2026-01-31');
|
|
538
|
-
return;
|
|
539
|
-
case 'instant':
|
|
540
|
-
if (typeof value !== 'string' || !INSTANT_PATTERN.test(value) || Number.isNaN(Date.parse(value)))
|
|
541
|
-
fail('an ISO 8601 timestamp with time zone');
|
|
542
|
-
return;
|
|
543
|
-
case 'json':
|
|
544
|
-
return;
|
|
545
|
-
case 'object':
|
|
546
|
-
if (!isRecord(value)) {
|
|
547
|
-
fail('an object');
|
|
548
|
-
return;
|
|
549
|
-
}
|
|
550
|
-
validateFields(path, type, value, issues, false);
|
|
551
|
-
return;
|
|
552
|
-
case 'array':
|
|
553
|
-
if (!Array.isArray(value)) {
|
|
554
|
-
fail('an array');
|
|
555
|
-
return;
|
|
556
|
-
}
|
|
557
|
-
value.forEach((item, i) => {
|
|
558
|
-
if (item === undefined || item === null)
|
|
559
|
-
issues.push({ path: `${path}[${i}]`, message: `${path}[${i}] is required` });
|
|
560
|
-
else
|
|
561
|
-
validateValue(`${path}[${i}]`, type.item, item, issues);
|
|
562
|
-
});
|
|
563
|
-
return;
|
|
564
|
-
case 'secret':
|
|
565
|
-
issues.push({ path, message: 'secret fields must be top-level config fields' });
|
|
566
|
-
return;
|
|
567
|
-
default:
|
|
568
|
-
issues.push({ path, message: `unknown type kind ${type.kind}` });
|
|
569
|
-
}
|
|
570
|
-
}
|
|
571
|
-
function validateFields(path, type, value, issues, topLevel) {
|
|
572
|
-
const shape = isRecord(type.shape) ? type.shape : {};
|
|
573
|
-
for (const [field, fieldType] of Object.entries(shape)) {
|
|
574
|
-
const fieldPath = `${path}.${field}`;
|
|
575
|
-
if (topLevel && isRecord(fieldType) && fieldType.kind === 'secret')
|
|
576
|
-
continue;
|
|
577
|
-
const fieldValue = value[field];
|
|
578
|
-
if (fieldValue === undefined || fieldValue === null) {
|
|
579
|
-
if (!(isRecord(fieldType) && fieldType.kind === 'json' && fieldValue === null)) {
|
|
580
|
-
issues.push({ path: fieldPath, message: `${fieldPath} is required` });
|
|
581
|
-
}
|
|
582
|
-
continue;
|
|
583
|
-
}
|
|
584
|
-
validateValue(fieldPath, fieldType, fieldValue, issues);
|
|
585
|
-
}
|
|
586
|
-
for (const field of Object.keys(value)) {
|
|
587
|
-
if (!Object.hasOwn(shape, field))
|
|
588
|
-
issues.push({ path: `${path}.${field}`, message: `${path}.${field} is not declared` });
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
export function validateConfig(schema, config) {
|
|
592
|
-
if (!isRecord(config))
|
|
593
|
-
return [{ path: 'config', message: 'config must be an object' }];
|
|
594
|
-
const issues = [];
|
|
595
|
-
validateFields('config', schema, config, issues, true);
|
|
596
|
-
return issues;
|
|
597
|
-
}
|