@ingestron/core 0.12.4 → 0.12.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -44,6 +44,11 @@ selects the output columns. Connector packages define the allowed table source
44
44
  shape, so the same project layout works across source types. Older stream-based
45
45
  connector releases remain supported.
46
46
 
47
+ The `contract_scaffold` operation creates a minimal draft ODCS contract from a
48
+ user-supplied first field. `connection_flow_add` accepts an optional provider
49
+ execution mapping; when omitted it authors local execution. Providers and the
50
+ compiler still validate the selected execution mode during project checks.
51
+
47
52
  ## Integration guides
48
53
 
49
54
  - [Operation and editor API](docs/api.md): requests, results, permissions, errors
@@ -3,11 +3,11 @@
3
3
  "dist/assets/schemas/ODCS-LICENSE": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4",
4
4
  "dist/assets/schemas/odcs-3.1.0.json": "1a35de14c688b400fef306564f66e2af3304f057cd5cd6e7b847c1a8bf252c5b",
5
5
  "dist/assets/schemas/provenance.json": "4b5eb1fd89b81223f7946113b651d5a3a18adaa28948e43c3615dd851fb6844e",
6
- "dist/core/authoring.js": "c144374cccbb512be7b3cd26700bc29538236eeeb35493eeec3bab938e08ff2a",
6
+ "dist/core/authoring.js": "84d848f46e612c241b615d3bd6868563696ca1f6504b27b76440da9bb94036fc",
7
7
  "dist/core/config.js": "b7e7138f7cc4a7b4ec8f6d0f0616c8802f2d802ec29aa09b40c1fc5c9324e016",
8
- "dist/core/connections.js": "37b797b402788e00c51f3cac93b87342d158ed9b279753db91006d2d6c66bb4a",
8
+ "dist/core/connections.js": "48a06440a2d82eee36ef8a22a9343b52194e567263a5062f06fddd5def2c3c3e",
9
9
  "dist/core/connector-package.js": "de9729e9e4b0cf05bd1ef347e57ffd489245bb68efce5c313ccba72b78409f41",
10
- "dist/core/contracts.js": "a1b7fd28f776117cca86a25c6c70e852ff57ffea2bb9e7accbf2d583948d9dde",
10
+ "dist/core/contracts.js": "a4af0b29ef8277a19d6e0c1d5de3059e09cdd3d45baefe9841cab1fb60985ecc",
11
11
  "dist/core/deliveries.js": "a4d2cb1213bc4c02cffb962dcb943ba98e261d37cfb21041948ec637599db080",
12
12
  "dist/core/delivery-exports.js": "d895b044c08f2059f0275724205120c72dd4bf5b74170bb01bdf42dd9d526d56",
13
13
  "dist/core/development.js": "2b55b15d545172e527cf66868d96d97c5ca081d0a83892ffd253eadaa3cc1b39",
@@ -21,7 +21,7 @@
21
21
  "dist/core/installed-plugins.js": "817ed411aef6f5b966a1828893081406805d2c8764d2ca20fd7e250e8a2168a8",
22
22
  "dist/core/model-pack-schema.js": "e3eb7ff33ddc652643e3f896637de0034004cd0cf26b0e296a074f830ae0b870",
23
23
  "dist/core/model-packs.js": "fee776ea0804dc66c9f140e57d94a3922d69c52d03c72079842ec43417616b0d",
24
- "dist/core/operations.js": "7f4a9eeee19d3e1e4c1beecc0350e38d386f012edfb31c3248efcd22ebd40a77",
24
+ "dist/core/operations.js": "c3d295df7450c6f0f730bf3ab05c223361fb4015f2e4cbffac7c09a16ef0f903",
25
25
  "dist/core/package-references.js": "bd3085eb6dcdac399d79af109d0a9e710a0532eba61e5298dc842aec9c7ef10a",
26
26
  "dist/core/packages.js": "c6bcbc90aa3597fcd4205b27136221229d9efedc9bfa68a83a134cb5e8f51d2b",
27
27
  "dist/core/planner.js": "974c6bac5c533a32a7c45f797ef4d857c644448aa243a929caca5917fb9f3fb8",
@@ -31,7 +31,7 @@
31
31
  "dist/core/provider-starter.js": "69795c1b79360e48664e91db0768f813fe4efc91aede003e1bf032f71731754c",
32
32
  "dist/core/report-pack-schema.js": "fe8ecdc2b8deacd8b2a1e46588ee06744a7b6d8e7342786e320f74cc848665fa",
33
33
  "dist/core/schema.js": "821e79cfc56dbd8c206959c53c0cda6d81507ae78010da1ec614e204c17d96eb",
34
- "dist/core/workflows.js": "e0b9e34fdd9b3f56403e8beb58e2b0bfbc0604beccb3113fc529c460817bc4d4",
34
+ "dist/core/workflows.js": "6573e1a2fd5873947bcba2809b5a4fb93e05db9a97ee5e0c294d67dae5947ff1",
35
35
  "dist/plugins/authoring.js": "ff985a7173c7aeda4a431a8603a4ab8631dc5134486578ede7d1301ebc3e6a19",
36
36
  "dist/plugins/compatibility.js": "a6591cfa4ca5a324fd29856e51319e39be470d6ed835ee268b293727ce11f8b7",
37
37
  "dist/plugins/generation.js": "7b0bae4de23edab4c2596d1f8ed2244d70b30fcae60395994b3964ffc845e9d5",
@@ -43,11 +43,11 @@
43
43
  "dist/version.js": "48af2aa1e425f1e387c082bbf0ad2c0c8d757c468101a614c3864153a94e6889"
44
44
  },
45
45
  "sources": {
46
- "src/core/authoring.ts": "4389a1c8994b1c5da6d470e29806e547d9e5c13ed481b5e34acbe919b6b4b3d8",
46
+ "src/core/authoring.ts": "e01136d3b54410ea0f44302bb43eefe2c2431138bee47600f0c97680785b8cab",
47
47
  "src/core/config.ts": "f282eb246438608e1951556dfeab9372daf65330a15a9d159d36d1f940370344",
48
- "src/core/connections.ts": "dbdaca4193efa203cffa02fd819b95f82bbd604a7678ba7dc6ba065205c49ce7",
48
+ "src/core/connections.ts": "608dd839421b08b612fb1e6f861bd934e2443dce72543b002f23b38b7dafc5b7",
49
49
  "src/core/connector-package.ts": "a36151f6ffb49836547984eae871023ad1ad69a136f33cd6ad608496f84c85f8",
50
- "src/core/contracts.ts": "b7445086085b17d5ae8d5a7f4e2e849f286948e1a2a146ceb79660846e8616e5",
50
+ "src/core/contracts.ts": "f10e2d689709665d7aea27d0c8861ae6d8b7039676d361389bc59a8f00e86aa9",
51
51
  "src/core/deliveries.ts": "6be0a7059c057420ba22a5515703164a892fac9e79a5bdb0e0871ea0b92ddd15",
52
52
  "src/core/delivery-exports.ts": "45a4d13e73a3809340fcd0f018dd06e2e360c67c1a117c672cd48be463c82d58",
53
53
  "src/core/development.ts": "decd47d8f6ca3289fea45331f27b99b0feb7188ae0a9e152c6459cc00658e7bd",
@@ -61,7 +61,7 @@
61
61
  "src/core/installed-plugins.ts": "951d15cdd8ca4a2c7d97fb01a46838822fe1b2fcf083ff8cf4bdca20d0997a2d",
62
62
  "src/core/model-pack-schema.ts": "bc9b30ab257fd25766cba63aa2fb6891c05e4b3eccf56d820caaab2dcc4147b4",
63
63
  "src/core/model-packs.ts": "a110d319f6feeff22a2d0d3a99ce5e49e56817d0cc27ab77d831d04a945ddfd3",
64
- "src/core/operations.ts": "e0a5a0e869e96de4e869f89e0fd794cb9fa537a16fb1f37fd8cf2b7ca777c08d",
64
+ "src/core/operations.ts": "bdb6b3dfaa5acd8367853dbe2d08013f5d68f6443afaacbaec8973f3d96424f0",
65
65
  "src/core/package-references.ts": "5c5cee7e140158928e2e5a37f426520c7c2f042546a37eed4507d79326181f5a",
66
66
  "src/core/packages.ts": "8a9b6612325de68ab2610f540bd39e4f899b06d1cc1fbd54ecfd2abb8b4e9c40",
67
67
  "src/core/planner.ts": "2c6f0d53e21cbb60fc2433e7da1245707133336274c60b9a5678614dabf09bca",
@@ -71,7 +71,7 @@
71
71
  "src/core/provider-starter.ts": "eeb504e347638bb933d204dd4d5a7e3e94ff45db136c5edd3fb457061342e405",
72
72
  "src/core/report-pack-schema.ts": "cf2c68f3725cef847627ce61acd0887f04d0067bc324841d8a552bcb12d274b0",
73
73
  "src/core/schema.ts": "a99a90e51be10ed56713573df6f238e6aca5b897a9679fe7bd2a8c0ad1e7f586",
74
- "src/core/workflows.ts": "f9e8d7f0a3841b5e385a5279e202f5a3334972bb549c8c85f01e7cfbb4e16f25",
74
+ "src/core/workflows.ts": "52f755e429a80e3283be030af58469158fdd19a357981b653e879d97194ec59a",
75
75
  "src/plugins/authoring.ts": "a0d68d406b4569c0b48eec458407d24c3d4eeed5b7918c6c4fcb3828ada93fe5",
76
76
  "src/plugins/compatibility.ts": "b5d357a1dc2a972cf8209311602f9fd816ba0a59cbe747f86f1b5567754073ea",
77
77
  "src/plugins/generation.ts": "a495906af09cf7bf820c7aba390bf742d7e94328ce708a2ff40f613f451fe237",
@@ -34,6 +34,46 @@ export declare function addFlow(root: string, options: {
34
34
  inputDataset?: string;
35
35
  provider?: string;
36
36
  }): Proposal;
37
+ /** Author a connector instance without requiring the editor to edit project YAML. */
38
+ export declare function addConnection(root: string, options: {
39
+ id: string;
40
+ package: string;
41
+ sourceId: string;
42
+ tenantId: string;
43
+ binding?: string;
44
+ settings: Record<string, unknown>;
45
+ }): Proposal;
46
+ /** Create a minimal reviewed-by-human starting point without reading source data. */
47
+ export declare function scaffoldContract(root: string, options: {
48
+ id: string;
49
+ table: string;
50
+ field: string;
51
+ type: "string" | "integer" | "number" | "boolean";
52
+ }): Proposal;
53
+ /** Add a connection-backed ingestion flow with one selected ODCS table. */
54
+ export declare function addConnectionFlow(root: string, options: {
55
+ id: string;
56
+ provider: string;
57
+ connection: string;
58
+ table: string;
59
+ contract: string;
60
+ source: Record<string, unknown>;
61
+ execution?: Record<string, unknown>;
62
+ }): Proposal;
63
+ /** Apply a reviewed source-field selection to one ODCS table. Approval remains separate. */
64
+ export declare function mapContractFields(root: string, options: {
65
+ apiVersion: "ingestron.field-mapping/v1";
66
+ flow: string;
67
+ environment: string;
68
+ contract: string;
69
+ discovery: string;
70
+ stream: string;
71
+ table: string;
72
+ fields: {
73
+ source: string;
74
+ target: string;
75
+ }[];
76
+ }): Proposal;
37
77
  export declare function addTable(root: string, options: {
38
78
  flow: string;
39
79
  id: string;
@@ -244,6 +244,169 @@ export function addFlow(root, options) {
244
244
  files["project.yaml"] = project.toString();
245
245
  return preview(root, files);
246
246
  }
247
+ /** Author a connector instance without requiring the editor to edit project YAML. */
248
+ export function addConnection(root, options) {
249
+ const reader = new Configuration(root);
250
+ const project = yaml(reader.text("project.yaml"), "project.yaml");
251
+ const parsed = projectSchema.parse(reader.load());
252
+ check(!Object.hasOwn(parsed.connections, options.id), "OWNER", `Connection ${options.id} already exists`);
253
+ const pkg = projectPackages(parsed)[options.package];
254
+ check(pkg, "PACKAGE", `Unknown connector package ${options.package}`);
255
+ const locked = resolvePackage(root, configuredPackageReference(pkg));
256
+ const manifest = packageYaml(locked.file);
257
+ check(manifest.apiVersion === "ingestron.connector/v1", "PACKAGE", "Select an installed connector package");
258
+ project.setIn(["connections", options.id], {
259
+ package: options.package,
260
+ sourceId: options.sourceId,
261
+ tenantId: options.tenantId,
262
+ ...(options.binding ? { binding: options.binding } : {}),
263
+ settings: options.settings,
264
+ });
265
+ return preview(root, { "project.yaml": project.toString() });
266
+ }
267
+ /** Create a minimal reviewed-by-human starting point without reading source data. */
268
+ export function scaffoldContract(root, options) {
269
+ const path = `contracts/${options.id}.odcs.yaml`;
270
+ check(!existsSync(fence(root, path)), "OWNER", `Contract ${path} already exists`);
271
+ const physicalTypes = {
272
+ string: "STRING",
273
+ integer: "BIGINT",
274
+ number: "DOUBLE",
275
+ boolean: "BOOLEAN",
276
+ };
277
+ const contract = {
278
+ apiVersion: "v3.1.0",
279
+ kind: "DataContract",
280
+ id: options.id,
281
+ name: options.id.replaceAll("-", " "),
282
+ version: "0.1.0",
283
+ status: "draft",
284
+ description: {
285
+ purpose: "Review this draft against source metadata before approval.",
286
+ },
287
+ schema: [
288
+ {
289
+ name: options.table,
290
+ logicalType: "object",
291
+ physicalType: "table",
292
+ properties: [
293
+ {
294
+ name: options.field,
295
+ logicalType: options.type,
296
+ physicalType: physicalTypes[options.type],
297
+ required: false,
298
+ },
299
+ ],
300
+ },
301
+ ],
302
+ };
303
+ contractColumns(contract, path);
304
+ return preview(root, { [path]: stringify(contract) });
305
+ }
306
+ /** Add a connection-backed ingestion flow with one selected ODCS table. */
307
+ export function addConnectionFlow(root, options) {
308
+ const reader = new Configuration(root);
309
+ const project = yaml(reader.text("project.yaml"), "project.yaml");
310
+ const raw = project.toJS();
311
+ const parsed = projectSchema.parse(reader.load());
312
+ check(!parsed.flows.some((flow) => flow?.id === options.id), "OWNER", `Flow ${options.id} already exists`);
313
+ check(parsed.connections[options.connection], "CONNECTION", "Select a configured connection");
314
+ check(parsed.providers.configurations[options.provider], "PROVIDER", "Select a configured provider");
315
+ check(!resolve(root, options.contract).startsWith(resolve(root, "flows", options.id) + "/"), "PATH", "Contract must be outside the new flow directory");
316
+ const contractFile = fence(root, options.contract);
317
+ check(existsSync(contractFile), "ODCS", "Contract file is missing");
318
+ contractColumns(reader.load(options.contract), options.contract);
319
+ const path = `flows/${options.id}/flow.yaml`;
320
+ check(!existsSync(fence(root, path)), "OWNER", `Flow ${options.id} already exists`);
321
+ const contractRef = relative(dirname(resolve(root, path)), contractFile).replaceAll("\\", "/");
322
+ const flow = {
323
+ apiVersion: "ingestron.flow/v1",
324
+ kind: "ingestion",
325
+ id: options.id,
326
+ provider: options.provider,
327
+ ingestion: {
328
+ connection: options.connection,
329
+ execution: options.execution ?? { mode: "local" },
330
+ },
331
+ tables: {
332
+ [options.table]: {
333
+ source: options.source,
334
+ contract: {
335
+ $resolve: contractRef.startsWith(".")
336
+ ? contractRef
337
+ : `./${contractRef}`,
338
+ },
339
+ },
340
+ },
341
+ };
342
+ project.set("flows", [...raw.flows, { $resolve: `./${path}` }]);
343
+ return preview(root, {
344
+ "project.yaml": project.toString(),
345
+ [path]: stringify(flow),
346
+ });
347
+ }
348
+ /** Apply a reviewed source-field selection to one ODCS table. Approval remains separate. */
349
+ export function mapContractFields(root, options) {
350
+ const reader = new Configuration(root);
351
+ const discovered = JSON.parse(reader.text(options.discovery));
352
+ check(discovered.apiVersion === "ingestron.singer-discovery/v1", "DISCOVERY", "Select saved Singer discovery evidence");
353
+ const stream = discovered.catalog?.streams?.find((entry) => entry?.stream === options.stream ||
354
+ entry?.tap_stream_id === options.stream);
355
+ check(stream, "DISCOVERY", "Selected source dataset is absent from discovery");
356
+ const sourceFields = stream.schema?.properties;
357
+ check(sourceFields &&
358
+ typeof sourceFields === "object" &&
359
+ !Array.isArray(sourceFields), "DISCOVERY", "Discovery has no field schema");
360
+ const doc = yaml(reader.text(options.contract), options.contract);
361
+ const contract = doc.toJS();
362
+ contractColumns(contract, options.contract);
363
+ check(contract.schema[0].name === options.table, "ODCS", "Selected contract table differs from the workflow table");
364
+ const previous = new Map(contract.schema[0].properties.map((property) => [
365
+ property.physicalName ?? property.name,
366
+ property,
367
+ ]));
368
+ const previousTargets = new Map(contract.schema[0].properties.map((property) => [
369
+ property.name,
370
+ property,
371
+ ]));
372
+ const sources = new Set(), targets = new Set();
373
+ const properties = options.fields.map(({ source, target }) => {
374
+ check(Object.hasOwn(sourceFields, source) && !sources.has(source), "MAPPING", `Unknown or duplicate source field ${source}`);
375
+ check(!targets.has(target), "MAPPING", `Duplicate target field ${target}`);
376
+ sources.add(source);
377
+ targets.add(target);
378
+ const old = (previous.get(source) ?? previousTargets.get(target));
379
+ if (old) {
380
+ const next = { ...old, name: target };
381
+ if (source !== target)
382
+ next.physicalName = source;
383
+ else
384
+ delete next.physicalName;
385
+ return next;
386
+ }
387
+ const shape = sourceFields[source], types = Array.isArray(shape?.type) ? shape.type : [shape?.type];
388
+ const type = types.find((value) => value !== "null");
389
+ const physicalType = {
390
+ string: "STRING",
391
+ integer: "BIGINT",
392
+ number: "DOUBLE",
393
+ boolean: "BOOLEAN",
394
+ }[String(type)];
395
+ check(physicalType, "MAPPING", `Source field ${source} needs an explicitly authored ODCS type`);
396
+ return {
397
+ name: target,
398
+ ...(source !== target ? { physicalName: source } : {}),
399
+ logicalType: type,
400
+ physicalType,
401
+ required: !types.includes("null"),
402
+ };
403
+ });
404
+ check(properties.length > 0, "MAPPING", "Select at least one source field");
405
+ doc.setIn(["schema", 0, "properties"], properties);
406
+ doc.set("status", "draft");
407
+ contractColumns(doc.toJS(), options.contract);
408
+ return preview(root, { [options.contract]: doc.toString() });
409
+ }
247
410
  export function addTable(root, options) {
248
411
  const { file, doc } = flowFile(root, options.flow);
249
412
  check(doc.get("kind") === "ingestion", "FLOW", "table add requires an ingestion flow");
@@ -14,6 +14,7 @@ export declare function prepareConnection(root: string, environment: string, flo
14
14
  } | {
15
15
  valid: boolean;
16
16
  specificationSha256: string;
17
+ sourceBoundarySha256: string;
17
18
  connection: string;
18
19
  flow: string;
19
20
  };
@@ -207,6 +207,13 @@ export function prepareConnection(root, environment, flowId, validateOnly = fals
207
207
  ? {
208
208
  valid: true,
209
209
  specificationSha256: input.specificationSha256,
210
+ sourceBoundarySha256: digest(canonical({
211
+ ...specification,
212
+ tables: Object.fromEntries(Object.entries(tables).map(([name, table]) => [
213
+ name,
214
+ table.source,
215
+ ])),
216
+ })),
210
217
  connection: input.connection,
211
218
  flow: flowId,
212
219
  }
@@ -1,6 +1,7 @@
1
1
  export declare function validateContract(contract: Record<string, any>, file?: string): void;
2
2
  export interface Column {
3
3
  name: string;
4
+ target?: string;
4
5
  type: string;
5
6
  required: boolean;
6
7
  key: boolean;
@@ -14,12 +14,17 @@ export function contractColumns(contract, file) {
14
14
  validateContract(contract, file);
15
15
  check(contract.schema?.length === 1 && contract.schema[0].properties?.length, "ODCS", "A dataset contract must contain one table schema with columns", file);
16
16
  const names = new Set();
17
+ const targets = new Set();
17
18
  return contract.schema[0].properties.map((p) => {
18
19
  const name = p.physicalName ?? p.name;
19
20
  check(typeof name === "string" &&
20
21
  /^[A-Za-z_][A-Za-z0-9_]*$/.test(name) &&
21
22
  !names.has(name), "ODCS", `Invalid or duplicate column ${name}`, file);
22
23
  names.add(name);
24
+ check(typeof p.name === "string" &&
25
+ /^[A-Za-z_][A-Za-z0-9_]*$/.test(p.name) &&
26
+ !targets.has(p.name), "ODCS", `Invalid or duplicate target column ${p.name}`, file);
27
+ targets.add(p.name);
23
28
  const type = (p.physicalType ??
24
29
  {
25
30
  string: "STRING",
@@ -37,6 +42,7 @@ export function contractColumns(contract, file) {
37
42
  }
38
43
  return {
39
44
  name,
45
+ ...(p.name !== name ? { target: p.name } : {}),
40
46
  type,
41
47
  required: p.required === true,
42
48
  key: p.primaryKey === true,
@@ -1,5 +1,46 @@
1
1
  import { z } from "zod";
2
2
  export declare const operationSchemas: {
3
+ readonly connection_add: z.ZodObject<{
4
+ id: z.ZodString;
5
+ package: z.ZodString;
6
+ sourceId: z.ZodString;
7
+ tenantId: z.ZodString;
8
+ binding: z.ZodOptional<z.ZodString>;
9
+ settings: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
10
+ }, z.core.$strict>;
11
+ readonly connection_flow_add: z.ZodObject<{
12
+ id: z.ZodString;
13
+ provider: z.ZodString;
14
+ connection: z.ZodString;
15
+ table: z.ZodString;
16
+ contract: z.ZodString;
17
+ source: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
18
+ execution: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
19
+ }, z.core.$strict>;
20
+ readonly contract_scaffold: z.ZodObject<{
21
+ id: z.ZodString;
22
+ table: z.ZodString;
23
+ field: z.ZodString;
24
+ type: z.ZodEnum<{
25
+ boolean: "boolean";
26
+ integer: "integer";
27
+ number: "number";
28
+ string: "string";
29
+ }>;
30
+ }, z.core.$strict>;
31
+ readonly contract_map_fields: z.ZodObject<{
32
+ apiVersion: z.ZodLiteral<"ingestron.field-mapping/v1">;
33
+ flow: z.ZodString;
34
+ environment: z.ZodString;
35
+ contract: z.ZodString;
36
+ discovery: z.ZodString;
37
+ stream: z.ZodString;
38
+ table: z.ZodString;
39
+ fields: z.ZodArray<z.ZodObject<{
40
+ source: z.ZodString;
41
+ target: z.ZodString;
42
+ }, z.core.$strict>>;
43
+ }, z.core.$strict>;
3
44
  readonly provider_scaffold: z.ZodObject<{
4
45
  id: z.ZodString;
5
46
  out: z.ZodString;
@@ -7,7 +7,7 @@ import { deliveryIndexSchema } from "./deliveries.js";
7
7
  import { existsSync, realpathSync } from "node:fs";
8
8
  import { resolve } from "node:path";
9
9
  import { z } from "zod";
10
- import { check, Problem, canonical } from "./errors.js";
10
+ import { check, Problem, canonical, digest } from "./errors.js";
11
11
  import { Configuration } from "./config.js";
12
12
  import { projectSchema, flowSchema, stepSchema, environmentSchema, } from "./schema.js";
13
13
  import { inspectProject, planProject } from "./planner.js";
@@ -24,6 +24,55 @@ import { providerVersions, friendlyReference } from "./package-references.js";
24
24
  import { browseProviders, pluginDetails } from "./installed-plugins.js";
25
25
  export const operationSchemas = {
26
26
  ...executionSchemas,
27
+ connection_add: z
28
+ .object({
29
+ id: workflows.identifier,
30
+ package: workflows.identifier,
31
+ sourceId: workflows.identifier,
32
+ tenantId: workflows.identifier,
33
+ binding: workflows.identifier.optional(),
34
+ settings: z.record(z.string(), z.unknown()).default({}),
35
+ })
36
+ .strict(),
37
+ connection_flow_add: z
38
+ .object({
39
+ id: workflows.identifier,
40
+ provider: workflows.identifier,
41
+ connection: workflows.identifier,
42
+ table: workflows.identifier,
43
+ contract: z.string().min(1),
44
+ source: z.record(z.string(), z.unknown()).default({}),
45
+ execution: z.record(z.string(), z.unknown()).optional(),
46
+ })
47
+ .strict(),
48
+ contract_scaffold: z
49
+ .object({
50
+ id: workflows.identifier,
51
+ table: z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/),
52
+ field: z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/),
53
+ type: z.enum(["string", "integer", "number", "boolean"]),
54
+ })
55
+ .strict(),
56
+ contract_map_fields: z
57
+ .object({
58
+ apiVersion: z.literal("ingestron.field-mapping/v1"),
59
+ flow: workflows.identifier,
60
+ environment: workflows.identifier,
61
+ contract: z.string().min(1),
62
+ discovery: z.string().min(1),
63
+ stream: z.string().min(1),
64
+ table: workflows.identifier,
65
+ fields: z
66
+ .array(z
67
+ .object({
68
+ source: workflows.identifier,
69
+ target: workflows.identifier,
70
+ })
71
+ .strict())
72
+ .min(1)
73
+ .max(500),
74
+ })
75
+ .strict(),
27
76
  provider_scaffold: z
28
77
  .object({ id: workflows.identifier, out: z.string() })
29
78
  .strict(),
@@ -308,6 +357,33 @@ export function execute(context, operation, request = {}) {
308
357
  const write = () => check(context.allowWrite, "PERMISSION", "This operation requires local-write access");
309
358
  let result;
310
359
  switch (operation) {
360
+ case "connection_add":
361
+ result = author.addConnection(root, args);
362
+ break;
363
+ case "connection_flow_add":
364
+ result = author.addConnectionFlow(root, args);
365
+ break;
366
+ case "contract_map_fields": {
367
+ check(args.environment === environment, "ENVIRONMENT", "Mapping draft is for another environment");
368
+ const prepared = prepareConnection(root, environment, args.flow, true);
369
+ const reader = new Configuration(root);
370
+ const discovery = JSON.parse(reader.text(args.discovery));
371
+ const discoveredSpec = discovery.identity?.projectSpecSha256;
372
+ if (discoveredSpec !== prepared.specificationSha256) {
373
+ const saved = JSON.parse(reader.text(`build/generated/flows/${args.flow}/project-connection.lock.json`));
374
+ const { specificationSha256: savedDigest, ...savedSpec } = saved;
375
+ const sourceBoundary = {
376
+ ...savedSpec,
377
+ tables: Object.fromEntries(Object.entries(savedSpec.tables ?? {}).map(([name, table]) => [name, table.source])),
378
+ };
379
+ check(savedDigest === discoveredSpec &&
380
+ savedDigest === digest(canonical(savedSpec)) &&
381
+ digest(canonical(sourceBoundary)) ===
382
+ prepared.sourceBoundarySha256, "DISCOVERY", "Source configuration changed since discovery; rebuild and rediscover before mapping fields");
383
+ }
384
+ result = author.mapContractFields(root, args);
385
+ break;
386
+ }
311
387
  case "runtime_prepare":
312
388
  case "run":
313
389
  case "run_status":
@@ -392,6 +468,9 @@ export function execute(context, operation, request = {}) {
392
468
  case "contract_create":
393
469
  result = workflows.draftContract(root, environment, args);
394
470
  break;
471
+ case "contract_scaffold":
472
+ result = author.scaffoldContract(root, args);
473
+ break;
395
474
  case "flow_create":
396
475
  result = workflows.flowCreate(root, environment, args);
397
476
  break;
@@ -182,12 +182,11 @@ export function pluginConfigure(root, args) {
182
182
  const connector = connectorPackageSchema.parse(sourceManifest);
183
183
  const { project, finish } = packageEditor(root);
184
184
  const name = identifier.parse(args.name ?? connector.id);
185
- const at = reference.lastIndexOf("@");
186
185
  const raw = project.toJS();
187
186
  const friendly = preferredReference(root, raw, reference, connector.id, connector.version);
188
187
  const value = {
189
188
  source: friendly.slice(0, friendly.lastIndexOf("@")),
190
- version: reference.slice(at + 1),
189
+ version: friendly.slice(friendly.lastIndexOf("@") + 1),
191
190
  };
192
191
  const previous = packageValue(raw, name);
193
192
  if (previous) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ingestron/core",
3
- "version": "0.12.4",
3
+ "version": "0.12.5",
4
4
  "type": "module",
5
5
  "description": "Shared Ingestron project compiler, operations and editor schemas",
6
6
  "license": "Apache-2.0",
@@ -45,6 +45,16 @@
45
45
  "type": "git",
46
46
  "url": "git+https://github.com/ingestron/core.git"
47
47
  },
48
+ "devDependencies": {
49
+ "@types/node": "22.20.1",
50
+ "prettier": "3.9.8",
51
+ "tsx": "4.23.13",
52
+ "typescript": "7.0.2"
53
+ },
54
+ "homepage": "https://ingestron.io",
55
+ "bugs": {
56
+ "url": "https://github.com/ingestron/core/issues"
57
+ },
48
58
  "scripts": {
49
59
  "build": "node scripts/build-assets.mjs && tsc && node scripts/compiler-fingerprint.mjs",
50
60
  "test": "tsx --test --test-concurrency=1 test/unit/*.test.ts test/providers/*.test.ts test/integration/*.test.ts",
@@ -54,19 +64,6 @@
54
64
  "release:check": "node scripts/release-check.mjs",
55
65
  "package:check": "tsx scripts/package-check.ts",
56
66
  "validate": "pnpm format:check && pnpm build && pnpm test && pnpm secrets:check && pnpm release:check && pnpm package:check",
57
- "prepack": "npm run build",
58
- "prepublishOnly": "node scripts/release-check.mjs --publish",
59
67
  "licences:inventory": "node scripts/licence-inventory.mjs"
60
- },
61
- "packageManager": "pnpm@10.15.0",
62
- "devDependencies": {
63
- "@types/node": "22.20.1",
64
- "prettier": "3.9.8",
65
- "tsx": "4.23.13",
66
- "typescript": "7.0.2"
67
- },
68
- "homepage": "https://ingestron.io",
69
- "bugs": {
70
- "url": "https://github.com/ingestron/core/issues"
71
68
  }
72
- }
69
+ }