@ingestron/core 0.12.4 → 0.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/dist/compiler-fingerprint.json +10 -10
- package/dist/core/authoring.d.ts +40 -0
- package/dist/core/authoring.js +163 -0
- package/dist/core/connections.d.ts +1 -0
- package/dist/core/connections.js +7 -0
- package/dist/core/contracts.d.ts +1 -0
- package/dist/core/contracts.js +6 -0
- package/dist/core/operations.d.ts +41 -0
- package/dist/core/operations.js +80 -1
- package/dist/core/workflows.js +1 -2
- package/package.json +12 -15
package/README.md
CHANGED
|
@@ -44,6 +44,11 @@ selects the output columns. Connector packages define the allowed table source
|
|
|
44
44
|
shape, so the same project layout works across source types. Older stream-based
|
|
45
45
|
connector releases remain supported.
|
|
46
46
|
|
|
47
|
+
The `contract_scaffold` operation creates a minimal draft ODCS contract from a
|
|
48
|
+
user-supplied first field. `connection_flow_add` accepts an optional provider
|
|
49
|
+
execution mapping; when omitted it authors local execution. Providers and the
|
|
50
|
+
compiler still validate the selected execution mode during project checks.
|
|
51
|
+
|
|
47
52
|
## Integration guides
|
|
48
53
|
|
|
49
54
|
- [Operation and editor API](docs/api.md): requests, results, permissions, errors
|
|
@@ -3,11 +3,11 @@
|
|
|
3
3
|
"dist/assets/schemas/ODCS-LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
|
|
4
4
|
"dist/assets/schemas/odcs-3.1.0.json": "1a35de14c688b400fef306564f66e2af3304f057cd5cd6e7b847c1a8bf252c5b",
|
|
5
5
|
"dist/assets/schemas/provenance.json": "4b5eb1fd89b81223f7946113b651d5a3a18adaa28948e43c3615dd851fb6844e",
|
|
6
|
-
"dist/core/authoring.js": "
|
|
6
|
+
"dist/core/authoring.js": "84d848f46e612c241b615d3bd6868563696ca1f6504b27b76440da9bb94036fc",
|
|
7
7
|
"dist/core/config.js": "b7e7138f7cc4a7b4ec8f6d0f0616c8802f2d802ec29aa09b40c1fc5c9324e016",
|
|
8
|
-
"dist/core/connections.js": "
|
|
8
|
+
"dist/core/connections.js": "48a06440a2d82eee36ef8a22a9343b52194e567263a5062f06fddd5def2c3c3e",
|
|
9
9
|
"dist/core/connector-package.js": "de9729e9e4b0cf05bd1ef347e57ffd489245bb68efce5c313ccba72b78409f41",
|
|
10
|
-
"dist/core/contracts.js": "
|
|
10
|
+
"dist/core/contracts.js": "a4af0b29ef8277a19d6e0c1d5de3059e09cdd3d45baefe9841cab1fb60985ecc",
|
|
11
11
|
"dist/core/deliveries.js": "a4d2cb1213bc4c02cffb962dcb943ba98e261d37cfb21041948ec637599db080",
|
|
12
12
|
"dist/core/delivery-exports.js": "d895b044c08f2059f0275724205120c72dd4bf5b74170bb01bdf42dd9d526d56",
|
|
13
13
|
"dist/core/development.js": "2b55b15d545172e527cf66868d96d97c5ca081d0a83892ffd253eadaa3cc1b39",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"dist/core/installed-plugins.js": "817ed411aef6f5b966a1828893081406805d2c8764d2ca20fd7e250e8a2168a8",
|
|
22
22
|
"dist/core/model-pack-schema.js": "e3eb7ff33ddc652643e3f896637de0034004cd0cf26b0e296a074f830ae0b870",
|
|
23
23
|
"dist/core/model-packs.js": "fee776ea0804dc66c9f140e57d94a3922d69c52d03c72079842ec43417616b0d",
|
|
24
|
-
"dist/core/operations.js": "
|
|
24
|
+
"dist/core/operations.js": "c3d295df7450c6f0f730bf3ab05c223361fb4015f2e4cbffac7c09a16ef0f903",
|
|
25
25
|
"dist/core/package-references.js": "bd3085eb6dcdac399d79af109d0a9e710a0532eba61e5298dc842aec9c7ef10a",
|
|
26
26
|
"dist/core/packages.js": "c6bcbc90aa3597fcd4205b27136221229d9efedc9bfa68a83a134cb5e8f51d2b",
|
|
27
27
|
"dist/core/planner.js": "974c6bac5c533a32a7c45f797ef4d857c644448aa243a929caca5917fb9f3fb8",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"dist/core/provider-starter.js": "69795c1b79360e48664e91db0768f813fe4efc91aede003e1bf032f71731754c",
|
|
32
32
|
"dist/core/report-pack-schema.js": "fe8ecdc2b8deacd8b2a1e46588ee06744a7b6d8e7342786e320f74cc848665fa",
|
|
33
33
|
"dist/core/schema.js": "821e79cfc56dbd8c206959c53c0cda6d81507ae78010da1ec614e204c17d96eb",
|
|
34
|
-
"dist/core/workflows.js": "
|
|
34
|
+
"dist/core/workflows.js": "6573e1a2fd5873947bcba2809b5a4fb93e05db9a97ee5e0c294d67dae5947ff1",
|
|
35
35
|
"dist/plugins/authoring.js": "ff985a7173c7aeda4a431a8603a4ab8631dc5134486578ede7d1301ebc3e6a19",
|
|
36
36
|
"dist/plugins/compatibility.js": "a6591cfa4ca5a324fd29856e51319e39be470d6ed835ee268b293727ce11f8b7",
|
|
37
37
|
"dist/plugins/generation.js": "7b0bae4de23edab4c2596d1f8ed2244d70b30fcae60395994b3964ffc845e9d5",
|
|
@@ -43,11 +43,11 @@
|
|
|
43
43
|
"dist/version.js": "48af2aa1e425f1e387c082bbf0ad2c0c8d757c468101a614c3864153a94e6889"
|
|
44
44
|
},
|
|
45
45
|
"sources": {
|
|
46
|
-
"src/core/authoring.ts": "
|
|
46
|
+
"src/core/authoring.ts": "e01136d3b54410ea0f44302bb43eefe2c2431138bee47600f0c97680785b8cab",
|
|
47
47
|
"src/core/config.ts": "f282eb246438608e1951556dfeab9372daf65330a15a9d159d36d1f940370344",
|
|
48
|
-
"src/core/connections.ts": "
|
|
48
|
+
"src/core/connections.ts": "608dd839421b08b612fb1e6f861bd934e2443dce72543b002f23b38b7dafc5b7",
|
|
49
49
|
"src/core/connector-package.ts": "a36151f6ffb49836547984eae871023ad1ad69a136f33cd6ad608496f84c85f8",
|
|
50
|
-
"src/core/contracts.ts": "
|
|
50
|
+
"src/core/contracts.ts": "f10e2d689709665d7aea27d0c8861ae6d8b7039676d361389bc59a8f00e86aa9",
|
|
51
51
|
"src/core/deliveries.ts": "6be0a7059c057420ba22a5515703164a892fac9e79a5bdb0e0871ea0b92ddd15",
|
|
52
52
|
"src/core/delivery-exports.ts": "45a4d13e73a3809340fcd0f018dd06e2e360c67c1a117c672cd48be463c82d58",
|
|
53
53
|
"src/core/development.ts": "decd47d8f6ca3289fea45331f27b99b0feb7188ae0a9e152c6459cc00658e7bd",
|
|
@@ -61,7 +61,7 @@
|
|
|
61
61
|
"src/core/installed-plugins.ts": "951d15cdd8ca4a2c7d97fb01a46838822fe1b2fcf083ff8cf4bdca20d0997a2d",
|
|
62
62
|
"src/core/model-pack-schema.ts": "bc9b30ab257fd25766cba63aa2fb6891c05e4b3eccf56d820caaab2dcc4147b4",
|
|
63
63
|
"src/core/model-packs.ts": "a110d319f6feeff22a2d0d3a99ce5e49e56817d0cc27ab77d831d04a945ddfd3",
|
|
64
|
-
"src/core/operations.ts": "
|
|
64
|
+
"src/core/operations.ts": "bdb6b3dfaa5acd8367853dbe2d08013f5d68f6443afaacbaec8973f3d96424f0",
|
|
65
65
|
"src/core/package-references.ts": "5c5cee7e140158928e2e5a37f426520c7c2f042546a37eed4507d79326181f5a",
|
|
66
66
|
"src/core/packages.ts": "8a9b6612325de68ab2610f540bd39e4f899b06d1cc1fbd54ecfd2abb8b4e9c40",
|
|
67
67
|
"src/core/planner.ts": "2c6f0d53e21cbb60fc2433e7da1245707133336274c60b9a5678614dabf09bca",
|
|
@@ -71,7 +71,7 @@
|
|
|
71
71
|
"src/core/provider-starter.ts": "eeb504e347638bb933d204dd4d5a7e3e94ff45db136c5edd3fb457061342e405",
|
|
72
72
|
"src/core/report-pack-schema.ts": "cf2c68f3725cef847627ce61acd0887f04d0067bc324841d8a552bcb12d274b0",
|
|
73
73
|
"src/core/schema.ts": "a99a90e51be10ed56713573df6f238e6aca5b897a9679fe7bd2a8c0ad1e7f586",
|
|
74
|
-
"src/core/workflows.ts": "
|
|
74
|
+
"src/core/workflows.ts": "52f755e429a80e3283be030af58469158fdd19a357981b653e879d97194ec59a",
|
|
75
75
|
"src/plugins/authoring.ts": "a0d68d406b4569c0b48eec458407d24c3d4eeed5b7918c6c4fcb3828ada93fe5",
|
|
76
76
|
"src/plugins/compatibility.ts": "b5d357a1dc2a972cf8209311602f9fd816ba0a59cbe747f86f1b5567754073ea",
|
|
77
77
|
"src/plugins/generation.ts": "a495906af09cf7bf820c7aba390bf742d7e94328ce708a2ff40f613f451fe237",
|
package/dist/core/authoring.d.ts
CHANGED
|
@@ -34,6 +34,46 @@ export declare function addFlow(root: string, options: {
|
|
|
34
34
|
inputDataset?: string;
|
|
35
35
|
provider?: string;
|
|
36
36
|
}): Proposal;
|
|
37
|
+
/** Author a connector instance without requiring the editor to edit project YAML. */
|
|
38
|
+
export declare function addConnection(root: string, options: {
|
|
39
|
+
id: string;
|
|
40
|
+
package: string;
|
|
41
|
+
sourceId: string;
|
|
42
|
+
tenantId: string;
|
|
43
|
+
binding?: string;
|
|
44
|
+
settings: Record<string, unknown>;
|
|
45
|
+
}): Proposal;
|
|
46
|
+
/** Create a minimal reviewed-by-human starting point without reading source data. */
|
|
47
|
+
export declare function scaffoldContract(root: string, options: {
|
|
48
|
+
id: string;
|
|
49
|
+
table: string;
|
|
50
|
+
field: string;
|
|
51
|
+
type: "string" | "integer" | "number" | "boolean";
|
|
52
|
+
}): Proposal;
|
|
53
|
+
/** Add a connection-backed ingestion flow with one selected ODCS table. */
|
|
54
|
+
export declare function addConnectionFlow(root: string, options: {
|
|
55
|
+
id: string;
|
|
56
|
+
provider: string;
|
|
57
|
+
connection: string;
|
|
58
|
+
table: string;
|
|
59
|
+
contract: string;
|
|
60
|
+
source: Record<string, unknown>;
|
|
61
|
+
execution?: Record<string, unknown>;
|
|
62
|
+
}): Proposal;
|
|
63
|
+
/** Apply a reviewed source-field selection to one ODCS table. Approval remains separate. */
|
|
64
|
+
export declare function mapContractFields(root: string, options: {
|
|
65
|
+
apiVersion: "ingestron.field-mapping/v1";
|
|
66
|
+
flow: string;
|
|
67
|
+
environment: string;
|
|
68
|
+
contract: string;
|
|
69
|
+
discovery: string;
|
|
70
|
+
stream: string;
|
|
71
|
+
table: string;
|
|
72
|
+
fields: {
|
|
73
|
+
source: string;
|
|
74
|
+
target: string;
|
|
75
|
+
}[];
|
|
76
|
+
}): Proposal;
|
|
37
77
|
export declare function addTable(root: string, options: {
|
|
38
78
|
flow: string;
|
|
39
79
|
id: string;
|
package/dist/core/authoring.js
CHANGED
|
@@ -244,6 +244,169 @@ export function addFlow(root, options) {
|
|
|
244
244
|
files["project.yaml"] = project.toString();
|
|
245
245
|
return preview(root, files);
|
|
246
246
|
}
|
|
247
|
+
/** Author a connector instance without requiring the editor to edit project YAML. */
|
|
248
|
+
export function addConnection(root, options) {
|
|
249
|
+
const reader = new Configuration(root);
|
|
250
|
+
const project = yaml(reader.text("project.yaml"), "project.yaml");
|
|
251
|
+
const parsed = projectSchema.parse(reader.load());
|
|
252
|
+
check(!Object.hasOwn(parsed.connections, options.id), "OWNER", `Connection ${options.id} already exists`);
|
|
253
|
+
const pkg = projectPackages(parsed)[options.package];
|
|
254
|
+
check(pkg, "PACKAGE", `Unknown connector package ${options.package}`);
|
|
255
|
+
const locked = resolvePackage(root, configuredPackageReference(pkg));
|
|
256
|
+
const manifest = packageYaml(locked.file);
|
|
257
|
+
check(manifest.apiVersion === "ingestron.connector/v1", "PACKAGE", "Select an installed connector package");
|
|
258
|
+
project.setIn(["connections", options.id], {
|
|
259
|
+
package: options.package,
|
|
260
|
+
sourceId: options.sourceId,
|
|
261
|
+
tenantId: options.tenantId,
|
|
262
|
+
...(options.binding ? { binding: options.binding } : {}),
|
|
263
|
+
settings: options.settings,
|
|
264
|
+
});
|
|
265
|
+
return preview(root, { "project.yaml": project.toString() });
|
|
266
|
+
}
|
|
267
|
+
/** Create a minimal reviewed-by-human starting point without reading source data. */
|
|
268
|
+
export function scaffoldContract(root, options) {
|
|
269
|
+
const path = `contracts/${options.id}.odcs.yaml`;
|
|
270
|
+
check(!existsSync(fence(root, path)), "OWNER", `Contract ${path} already exists`);
|
|
271
|
+
const physicalTypes = {
|
|
272
|
+
string: "STRING",
|
|
273
|
+
integer: "BIGINT",
|
|
274
|
+
number: "DOUBLE",
|
|
275
|
+
boolean: "BOOLEAN",
|
|
276
|
+
};
|
|
277
|
+
const contract = {
|
|
278
|
+
apiVersion: "v3.1.0",
|
|
279
|
+
kind: "DataContract",
|
|
280
|
+
id: options.id,
|
|
281
|
+
name: options.id.replaceAll("-", " "),
|
|
282
|
+
version: "0.1.0",
|
|
283
|
+
status: "draft",
|
|
284
|
+
description: {
|
|
285
|
+
purpose: "Review this draft against source metadata before approval.",
|
|
286
|
+
},
|
|
287
|
+
schema: [
|
|
288
|
+
{
|
|
289
|
+
name: options.table,
|
|
290
|
+
logicalType: "object",
|
|
291
|
+
physicalType: "table",
|
|
292
|
+
properties: [
|
|
293
|
+
{
|
|
294
|
+
name: options.field,
|
|
295
|
+
logicalType: options.type,
|
|
296
|
+
physicalType: physicalTypes[options.type],
|
|
297
|
+
required: false,
|
|
298
|
+
},
|
|
299
|
+
],
|
|
300
|
+
},
|
|
301
|
+
],
|
|
302
|
+
};
|
|
303
|
+
contractColumns(contract, path);
|
|
304
|
+
return preview(root, { [path]: stringify(contract) });
|
|
305
|
+
}
|
|
306
|
+
/** Add a connection-backed ingestion flow with one selected ODCS table. */
|
|
307
|
+
export function addConnectionFlow(root, options) {
|
|
308
|
+
const reader = new Configuration(root);
|
|
309
|
+
const project = yaml(reader.text("project.yaml"), "project.yaml");
|
|
310
|
+
const raw = project.toJS();
|
|
311
|
+
const parsed = projectSchema.parse(reader.load());
|
|
312
|
+
check(!parsed.flows.some((flow) => flow?.id === options.id), "OWNER", `Flow ${options.id} already exists`);
|
|
313
|
+
check(parsed.connections[options.connection], "CONNECTION", "Select a configured connection");
|
|
314
|
+
check(parsed.providers.configurations[options.provider], "PROVIDER", "Select a configured provider");
|
|
315
|
+
check(!resolve(root, options.contract).startsWith(resolve(root, "flows", options.id) + "/"), "PATH", "Contract must be outside the new flow directory");
|
|
316
|
+
const contractFile = fence(root, options.contract);
|
|
317
|
+
check(existsSync(contractFile), "ODCS", "Contract file is missing");
|
|
318
|
+
contractColumns(reader.load(options.contract), options.contract);
|
|
319
|
+
const path = `flows/${options.id}/flow.yaml`;
|
|
320
|
+
check(!existsSync(fence(root, path)), "OWNER", `Flow ${options.id} already exists`);
|
|
321
|
+
const contractRef = relative(dirname(resolve(root, path)), contractFile).replaceAll("\\", "/");
|
|
322
|
+
const flow = {
|
|
323
|
+
apiVersion: "ingestron.flow/v1",
|
|
324
|
+
kind: "ingestion",
|
|
325
|
+
id: options.id,
|
|
326
|
+
provider: options.provider,
|
|
327
|
+
ingestion: {
|
|
328
|
+
connection: options.connection,
|
|
329
|
+
execution: options.execution ?? { mode: "local" },
|
|
330
|
+
},
|
|
331
|
+
tables: {
|
|
332
|
+
[options.table]: {
|
|
333
|
+
source: options.source,
|
|
334
|
+
contract: {
|
|
335
|
+
$resolve: contractRef.startsWith(".")
|
|
336
|
+
? contractRef
|
|
337
|
+
: `./${contractRef}`,
|
|
338
|
+
},
|
|
339
|
+
},
|
|
340
|
+
},
|
|
341
|
+
};
|
|
342
|
+
project.set("flows", [...raw.flows, { $resolve: `./${path}` }]);
|
|
343
|
+
return preview(root, {
|
|
344
|
+
"project.yaml": project.toString(),
|
|
345
|
+
[path]: stringify(flow),
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
/** Apply a reviewed source-field selection to one ODCS table. Approval remains separate. */
|
|
349
|
+
export function mapContractFields(root, options) {
|
|
350
|
+
const reader = new Configuration(root);
|
|
351
|
+
const discovered = JSON.parse(reader.text(options.discovery));
|
|
352
|
+
check(discovered.apiVersion === "ingestron.singer-discovery/v1", "DISCOVERY", "Select saved Singer discovery evidence");
|
|
353
|
+
const stream = discovered.catalog?.streams?.find((entry) => entry?.stream === options.stream ||
|
|
354
|
+
entry?.tap_stream_id === options.stream);
|
|
355
|
+
check(stream, "DISCOVERY", "Selected source dataset is absent from discovery");
|
|
356
|
+
const sourceFields = stream.schema?.properties;
|
|
357
|
+
check(sourceFields &&
|
|
358
|
+
typeof sourceFields === "object" &&
|
|
359
|
+
!Array.isArray(sourceFields), "DISCOVERY", "Discovery has no field schema");
|
|
360
|
+
const doc = yaml(reader.text(options.contract), options.contract);
|
|
361
|
+
const contract = doc.toJS();
|
|
362
|
+
contractColumns(contract, options.contract);
|
|
363
|
+
check(contract.schema[0].name === options.table, "ODCS", "Selected contract table differs from the workflow table");
|
|
364
|
+
const previous = new Map(contract.schema[0].properties.map((property) => [
|
|
365
|
+
property.physicalName ?? property.name,
|
|
366
|
+
property,
|
|
367
|
+
]));
|
|
368
|
+
const previousTargets = new Map(contract.schema[0].properties.map((property) => [
|
|
369
|
+
property.name,
|
|
370
|
+
property,
|
|
371
|
+
]));
|
|
372
|
+
const sources = new Set(), targets = new Set();
|
|
373
|
+
const properties = options.fields.map(({ source, target }) => {
|
|
374
|
+
check(Object.hasOwn(sourceFields, source) && !sources.has(source), "MAPPING", `Unknown or duplicate source field ${source}`);
|
|
375
|
+
check(!targets.has(target), "MAPPING", `Duplicate target field ${target}`);
|
|
376
|
+
sources.add(source);
|
|
377
|
+
targets.add(target);
|
|
378
|
+
const old = (previous.get(source) ?? previousTargets.get(target));
|
|
379
|
+
if (old) {
|
|
380
|
+
const next = { ...old, name: target };
|
|
381
|
+
if (source !== target)
|
|
382
|
+
next.physicalName = source;
|
|
383
|
+
else
|
|
384
|
+
delete next.physicalName;
|
|
385
|
+
return next;
|
|
386
|
+
}
|
|
387
|
+
const shape = sourceFields[source], types = Array.isArray(shape?.type) ? shape.type : [shape?.type];
|
|
388
|
+
const type = types.find((value) => value !== "null");
|
|
389
|
+
const physicalType = {
|
|
390
|
+
string: "STRING",
|
|
391
|
+
integer: "BIGINT",
|
|
392
|
+
number: "DOUBLE",
|
|
393
|
+
boolean: "BOOLEAN",
|
|
394
|
+
}[String(type)];
|
|
395
|
+
check(physicalType, "MAPPING", `Source field ${source} needs an explicitly authored ODCS type`);
|
|
396
|
+
return {
|
|
397
|
+
name: target,
|
|
398
|
+
...(source !== target ? { physicalName: source } : {}),
|
|
399
|
+
logicalType: type,
|
|
400
|
+
physicalType,
|
|
401
|
+
required: !types.includes("null"),
|
|
402
|
+
};
|
|
403
|
+
});
|
|
404
|
+
check(properties.length > 0, "MAPPING", "Select at least one source field");
|
|
405
|
+
doc.setIn(["schema", 0, "properties"], properties);
|
|
406
|
+
doc.set("status", "draft");
|
|
407
|
+
contractColumns(doc.toJS(), options.contract);
|
|
408
|
+
return preview(root, { [options.contract]: doc.toString() });
|
|
409
|
+
}
|
|
247
410
|
export function addTable(root, options) {
|
|
248
411
|
const { file, doc } = flowFile(root, options.flow);
|
|
249
412
|
check(doc.get("kind") === "ingestion", "FLOW", "table add requires an ingestion flow");
|
package/dist/core/connections.js
CHANGED
|
@@ -207,6 +207,13 @@ export function prepareConnection(root, environment, flowId, validateOnly = fals
|
|
|
207
207
|
? {
|
|
208
208
|
valid: true,
|
|
209
209
|
specificationSha256: input.specificationSha256,
|
|
210
|
+
sourceBoundarySha256: digest(canonical({
|
|
211
|
+
...specification,
|
|
212
|
+
tables: Object.fromEntries(Object.entries(tables).map(([name, table]) => [
|
|
213
|
+
name,
|
|
214
|
+
table.source,
|
|
215
|
+
])),
|
|
216
|
+
})),
|
|
210
217
|
connection: input.connection,
|
|
211
218
|
flow: flowId,
|
|
212
219
|
}
|
package/dist/core/contracts.d.ts
CHANGED
package/dist/core/contracts.js
CHANGED
|
@@ -14,12 +14,17 @@ export function contractColumns(contract, file) {
|
|
|
14
14
|
validateContract(contract, file);
|
|
15
15
|
check(contract.schema?.length === 1 && contract.schema[0].properties?.length, "ODCS", "A dataset contract must contain one table schema with columns", file);
|
|
16
16
|
const names = new Set();
|
|
17
|
+
const targets = new Set();
|
|
17
18
|
return contract.schema[0].properties.map((p) => {
|
|
18
19
|
const name = p.physicalName ?? p.name;
|
|
19
20
|
check(typeof name === "string" &&
|
|
20
21
|
/^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
|
|
21
22
|
!names.has(name), "ODCS", `Invalid or duplicate column ${name}`, file);
|
|
22
23
|
names.add(name);
|
|
24
|
+
check(typeof p.name === "string" &&
|
|
25
|
+
/^[A-Za-z_][A-Za-z0-9_]*$/.test(p.name) &&
|
|
26
|
+
!targets.has(p.name), "ODCS", `Invalid or duplicate target column ${p.name}`, file);
|
|
27
|
+
targets.add(p.name);
|
|
23
28
|
const type = (p.physicalType ??
|
|
24
29
|
{
|
|
25
30
|
string: "STRING",
|
|
@@ -37,6 +42,7 @@ export function contractColumns(contract, file) {
|
|
|
37
42
|
}
|
|
38
43
|
return {
|
|
39
44
|
name,
|
|
45
|
+
...(p.name !== name ? { target: p.name } : {}),
|
|
40
46
|
type,
|
|
41
47
|
required: p.required === true,
|
|
42
48
|
key: p.primaryKey === true,
|
|
@@ -1,5 +1,46 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
export declare const operationSchemas: {
|
|
3
|
+
readonly connection_add: z.ZodObject<{
|
|
4
|
+
id: z.ZodString;
|
|
5
|
+
package: z.ZodString;
|
|
6
|
+
sourceId: z.ZodString;
|
|
7
|
+
tenantId: z.ZodString;
|
|
8
|
+
binding: z.ZodOptional<z.ZodString>;
|
|
9
|
+
settings: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
10
|
+
}, z.core.$strict>;
|
|
11
|
+
readonly connection_flow_add: z.ZodObject<{
|
|
12
|
+
id: z.ZodString;
|
|
13
|
+
provider: z.ZodString;
|
|
14
|
+
connection: z.ZodString;
|
|
15
|
+
table: z.ZodString;
|
|
16
|
+
contract: z.ZodString;
|
|
17
|
+
source: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
18
|
+
execution: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
19
|
+
}, z.core.$strict>;
|
|
20
|
+
readonly contract_scaffold: z.ZodObject<{
|
|
21
|
+
id: z.ZodString;
|
|
22
|
+
table: z.ZodString;
|
|
23
|
+
field: z.ZodString;
|
|
24
|
+
type: z.ZodEnum<{
|
|
25
|
+
boolean: "boolean";
|
|
26
|
+
integer: "integer";
|
|
27
|
+
number: "number";
|
|
28
|
+
string: "string";
|
|
29
|
+
}>;
|
|
30
|
+
}, z.core.$strict>;
|
|
31
|
+
readonly contract_map_fields: z.ZodObject<{
|
|
32
|
+
apiVersion: z.ZodLiteral<"ingestron.field-mapping/v1">;
|
|
33
|
+
flow: z.ZodString;
|
|
34
|
+
environment: z.ZodString;
|
|
35
|
+
contract: z.ZodString;
|
|
36
|
+
discovery: z.ZodString;
|
|
37
|
+
stream: z.ZodString;
|
|
38
|
+
table: z.ZodString;
|
|
39
|
+
fields: z.ZodArray<z.ZodObject<{
|
|
40
|
+
source: z.ZodString;
|
|
41
|
+
target: z.ZodString;
|
|
42
|
+
}, z.core.$strict>>;
|
|
43
|
+
}, z.core.$strict>;
|
|
3
44
|
readonly provider_scaffold: z.ZodObject<{
|
|
4
45
|
id: z.ZodString;
|
|
5
46
|
out: z.ZodString;
|
package/dist/core/operations.js
CHANGED
|
@@ -7,7 +7,7 @@ import { deliveryIndexSchema } from "./deliveries.js";
|
|
|
7
7
|
import { existsSync, realpathSync } from "node:fs";
|
|
8
8
|
import { resolve } from "node:path";
|
|
9
9
|
import { z } from "zod";
|
|
10
|
-
import { check, Problem, canonical } from "./errors.js";
|
|
10
|
+
import { check, Problem, canonical, digest } from "./errors.js";
|
|
11
11
|
import { Configuration } from "./config.js";
|
|
12
12
|
import { projectSchema, flowSchema, stepSchema, environmentSchema, } from "./schema.js";
|
|
13
13
|
import { inspectProject, planProject } from "./planner.js";
|
|
@@ -24,6 +24,55 @@ import { providerVersions, friendlyReference } from "./package-references.js";
|
|
|
24
24
|
import { browseProviders, pluginDetails } from "./installed-plugins.js";
|
|
25
25
|
export const operationSchemas = {
|
|
26
26
|
...executionSchemas,
|
|
27
|
+
connection_add: z
|
|
28
|
+
.object({
|
|
29
|
+
id: workflows.identifier,
|
|
30
|
+
package: workflows.identifier,
|
|
31
|
+
sourceId: workflows.identifier,
|
|
32
|
+
tenantId: workflows.identifier,
|
|
33
|
+
binding: workflows.identifier.optional(),
|
|
34
|
+
settings: z.record(z.string(), z.unknown()).default({}),
|
|
35
|
+
})
|
|
36
|
+
.strict(),
|
|
37
|
+
connection_flow_add: z
|
|
38
|
+
.object({
|
|
39
|
+
id: workflows.identifier,
|
|
40
|
+
provider: workflows.identifier,
|
|
41
|
+
connection: workflows.identifier,
|
|
42
|
+
table: workflows.identifier,
|
|
43
|
+
contract: z.string().min(1),
|
|
44
|
+
source: z.record(z.string(), z.unknown()).default({}),
|
|
45
|
+
execution: z.record(z.string(), z.unknown()).optional(),
|
|
46
|
+
})
|
|
47
|
+
.strict(),
|
|
48
|
+
contract_scaffold: z
|
|
49
|
+
.object({
|
|
50
|
+
id: workflows.identifier,
|
|
51
|
+
table: z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/),
|
|
52
|
+
field: z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/),
|
|
53
|
+
type: z.enum(["string", "integer", "number", "boolean"]),
|
|
54
|
+
})
|
|
55
|
+
.strict(),
|
|
56
|
+
contract_map_fields: z
|
|
57
|
+
.object({
|
|
58
|
+
apiVersion: z.literal("ingestron.field-mapping/v1"),
|
|
59
|
+
flow: workflows.identifier,
|
|
60
|
+
environment: workflows.identifier,
|
|
61
|
+
contract: z.string().min(1),
|
|
62
|
+
discovery: z.string().min(1),
|
|
63
|
+
stream: z.string().min(1),
|
|
64
|
+
table: workflows.identifier,
|
|
65
|
+
fields: z
|
|
66
|
+
.array(z
|
|
67
|
+
.object({
|
|
68
|
+
source: workflows.identifier,
|
|
69
|
+
target: workflows.identifier,
|
|
70
|
+
})
|
|
71
|
+
.strict())
|
|
72
|
+
.min(1)
|
|
73
|
+
.max(500),
|
|
74
|
+
})
|
|
75
|
+
.strict(),
|
|
27
76
|
provider_scaffold: z
|
|
28
77
|
.object({ id: workflows.identifier, out: z.string() })
|
|
29
78
|
.strict(),
|
|
@@ -308,6 +357,33 @@ export function execute(context, operation, request = {}) {
|
|
|
308
357
|
const write = () => check(context.allowWrite, "PERMISSION", "This operation requires local-write access");
|
|
309
358
|
let result;
|
|
310
359
|
switch (operation) {
|
|
360
|
+
case "connection_add":
|
|
361
|
+
result = author.addConnection(root, args);
|
|
362
|
+
break;
|
|
363
|
+
case "connection_flow_add":
|
|
364
|
+
result = author.addConnectionFlow(root, args);
|
|
365
|
+
break;
|
|
366
|
+
case "contract_map_fields": {
|
|
367
|
+
check(args.environment === environment, "ENVIRONMENT", "Mapping draft is for another environment");
|
|
368
|
+
const prepared = prepareConnection(root, environment, args.flow, true);
|
|
369
|
+
const reader = new Configuration(root);
|
|
370
|
+
const discovery = JSON.parse(reader.text(args.discovery));
|
|
371
|
+
const discoveredSpec = discovery.identity?.projectSpecSha256;
|
|
372
|
+
if (discoveredSpec !== prepared.specificationSha256) {
|
|
373
|
+
const saved = JSON.parse(reader.text(`build/generated/flows/${args.flow}/project-connection.lock.json`));
|
|
374
|
+
const { specificationSha256: savedDigest, ...savedSpec } = saved;
|
|
375
|
+
const sourceBoundary = {
|
|
376
|
+
...savedSpec,
|
|
377
|
+
tables: Object.fromEntries(Object.entries(savedSpec.tables ?? {}).map(([name, table]) => [name, table.source])),
|
|
378
|
+
};
|
|
379
|
+
check(savedDigest === discoveredSpec &&
|
|
380
|
+
savedDigest === digest(canonical(savedSpec)) &&
|
|
381
|
+
digest(canonical(sourceBoundary)) ===
|
|
382
|
+
prepared.sourceBoundarySha256, "DISCOVERY", "Source configuration changed since discovery; rebuild and rediscover before mapping fields");
|
|
383
|
+
}
|
|
384
|
+
result = author.mapContractFields(root, args);
|
|
385
|
+
break;
|
|
386
|
+
}
|
|
311
387
|
case "runtime_prepare":
|
|
312
388
|
case "run":
|
|
313
389
|
case "run_status":
|
|
@@ -392,6 +468,9 @@ export function execute(context, operation, request = {}) {
|
|
|
392
468
|
case "contract_create":
|
|
393
469
|
result = workflows.draftContract(root, environment, args);
|
|
394
470
|
break;
|
|
471
|
+
case "contract_scaffold":
|
|
472
|
+
result = author.scaffoldContract(root, args);
|
|
473
|
+
break;
|
|
395
474
|
case "flow_create":
|
|
396
475
|
result = workflows.flowCreate(root, environment, args);
|
|
397
476
|
break;
|
package/dist/core/workflows.js
CHANGED
|
@@ -182,12 +182,11 @@ export function pluginConfigure(root, args) {
|
|
|
182
182
|
const connector = connectorPackageSchema.parse(sourceManifest);
|
|
183
183
|
const { project, finish } = packageEditor(root);
|
|
184
184
|
const name = identifier.parse(args.name ?? connector.id);
|
|
185
|
-
const at = reference.lastIndexOf("@");
|
|
186
185
|
const raw = project.toJS();
|
|
187
186
|
const friendly = preferredReference(root, raw, reference, connector.id, connector.version);
|
|
188
187
|
const value = {
|
|
189
188
|
source: friendly.slice(0, friendly.lastIndexOf("@")),
|
|
190
|
-
version:
|
|
189
|
+
version: friendly.slice(friendly.lastIndexOf("@") + 1),
|
|
191
190
|
};
|
|
192
191
|
const previous = packageValue(raw, name);
|
|
193
192
|
if (previous) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@ingestron/core",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Shared Ingestron project compiler, operations and editor schemas",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -45,6 +45,16 @@
|
|
|
45
45
|
"type": "git",
|
|
46
46
|
"url": "git+https://github.com/ingestron/core.git"
|
|
47
47
|
},
|
|
48
|
+
"devDependencies": {
|
|
49
|
+
"@types/node": "22.20.1",
|
|
50
|
+
"prettier": "3.9.8",
|
|
51
|
+
"tsx": "4.23.13",
|
|
52
|
+
"typescript": "7.0.2"
|
|
53
|
+
},
|
|
54
|
+
"homepage": "https://ingestron.io",
|
|
55
|
+
"bugs": {
|
|
56
|
+
"url": "https://github.com/ingestron/core/issues"
|
|
57
|
+
},
|
|
48
58
|
"scripts": {
|
|
49
59
|
"build": "node scripts/build-assets.mjs && tsc && node scripts/compiler-fingerprint.mjs",
|
|
50
60
|
"test": "tsx --test --test-concurrency=1 test/unit/*.test.ts test/providers/*.test.ts test/integration/*.test.ts",
|
|
@@ -54,19 +64,6 @@
|
|
|
54
64
|
"release:check": "node scripts/release-check.mjs",
|
|
55
65
|
"package:check": "tsx scripts/package-check.ts",
|
|
56
66
|
"validate": "pnpm format:check && pnpm build && pnpm test && pnpm secrets:check && pnpm release:check && pnpm package:check",
|
|
57
|
-
"prepack": "npm run build",
|
|
58
|
-
"prepublishOnly": "node scripts/release-check.mjs --publish",
|
|
59
67
|
"licences:inventory": "node scripts/licence-inventory.mjs"
|
|
60
|
-
},
|
|
61
|
-
"packageManager": "pnpm@10.15.0",
|
|
62
|
-
"devDependencies": {
|
|
63
|
-
"@types/node": "22.20.1",
|
|
64
|
-
"prettier": "3.9.8",
|
|
65
|
-
"tsx": "4.23.13",
|
|
66
|
-
"typescript": "7.0.2"
|
|
67
|
-
},
|
|
68
|
-
"homepage": "https://ingestron.io",
|
|
69
|
-
"bugs": {
|
|
70
|
-
"url": "https://github.com/ingestron/core/issues"
|
|
71
68
|
}
|
|
72
|
-
}
|
|
69
|
+
}
|