@ingestron/core 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +6 -0
- package/README.md +56 -0
- package/SECURITY.md +44 -0
- package/THIRD_PARTY_NOTICES.md +693 -0
- package/dist/assets/schemas/ODCS-LICENSE +201 -0
- package/dist/assets/schemas/odcs-3.1.0.json +2929 -0
- package/dist/assets/schemas/provenance.json +16 -0
- package/dist/compiler-fingerprint.json +84 -0
- package/dist/core/authoring.d.ts +55 -0
- package/dist/core/authoring.js +315 -0
- package/dist/core/config.d.ts +34 -0
- package/dist/core/config.js +233 -0
- package/dist/core/connections.d.ts +19 -0
- package/dist/core/connections.js +203 -0
- package/dist/core/connector-package.d.ts +37 -0
- package/dist/core/connector-package.js +43 -0
- package/dist/core/contracts.d.ts +8 -0
- package/dist/core/contracts.js +45 -0
- package/dist/core/deliveries.d.ts +16 -0
- package/dist/core/deliveries.js +28 -0
- package/dist/core/delivery-exports.d.ts +20 -0
- package/dist/core/delivery-exports.js +119 -0
- package/dist/core/development.d.ts +50 -0
- package/dist/core/development.js +94 -0
- package/dist/core/ecosystem-authoring.d.ts +13 -0
- package/dist/core/ecosystem-authoring.js +82 -0
- package/dist/core/errors.d.ts +12 -0
- package/dist/core/errors.js +43 -0
- package/dist/core/execution.d.ts +28 -0
- package/dist/core/execution.js +297 -0
- package/dist/core/extension-packs.d.ts +8 -0
- package/dist/core/extension-packs.js +34 -0
- package/dist/core/fingerprint.d.ts +1 -0
- package/dist/core/fingerprint.js +16 -0
- package/dist/core/generate.d.ts +16 -0
- package/dist/core/generate.js +198 -0
- package/dist/core/installation.d.ts +3 -0
- package/dist/core/installation.js +14 -0
- package/dist/core/installed-plugins.d.ts +41 -0
- package/dist/core/installed-plugins.js +92 -0
- package/dist/core/model-pack-schema.d.ts +46 -0
- package/dist/core/model-pack-schema.js +81 -0
- package/dist/core/model-packs.d.ts +4 -0
- package/dist/core/model-packs.js +33 -0
- package/dist/core/operations.d.ts +346 -0
- package/dist/core/operations.js +760 -0
- package/dist/core/package-references.d.ts +36 -0
- package/dist/core/package-references.js +103 -0
- package/dist/core/packages.d.ts +340 -0
- package/dist/core/packages.js +579 -0
- package/dist/core/planner.d.ts +114 -0
- package/dist/core/planner.js +821 -0
- package/dist/core/project-build.d.ts +13 -0
- package/dist/core/project-build.js +266 -0
- package/dist/core/provider-commands.d.ts +83 -0
- package/dist/core/provider-commands.js +252 -0
- package/dist/core/provider-starter.d.ts +5 -0
- package/dist/core/provider-starter.js +155 -0
- package/dist/core/report-pack-schema.d.ts +13 -0
- package/dist/core/report-pack-schema.js +15 -0
- package/dist/core/schema.d.ts +257 -0
- package/dist/core/schema.js +126 -0
- package/dist/core/workflows.d.ts +91 -0
- package/dist/core/workflows.js +438 -0
- package/dist/plugins/authoring.d.ts +88 -0
- package/dist/plugins/authoring.js +34 -0
- package/dist/plugins/compatibility.d.ts +23 -0
- package/dist/plugins/compatibility.js +55 -0
- package/dist/plugins/generation.d.ts +5 -0
- package/dist/plugins/generation.js +105 -0
- package/dist/plugins/provider-renderer.d.ts +7 -0
- package/dist/plugins/provider-renderer.js +37 -0
- package/dist/plugins/provider-worker.d.ts +1 -0
- package/dist/plugins/provider-worker.js +74 -0
- package/dist/sdk/adapter.d.ts +15 -0
- package/dist/sdk/adapter.js +15 -0
- package/dist/sdk/index.d.ts +4 -0
- package/dist/sdk/index.js +3 -0
- package/dist/sdk/schemas.d.ts +34 -0
- package/dist/sdk/schemas.js +74 -0
- package/dist/version.d.ts +5 -0
- package/dist/version.js +7 -0
- package/package.json +72 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { version } from "../version.js";
|
|
2
|
+
import { stringify } from "yaml";
|
|
3
|
+
import { existsSync } from "node:fs";
|
|
4
|
+
import { preview } from "./authoring.js";
|
|
5
|
+
import { fence } from "./packages.js";
|
|
6
|
+
import { check } from "./errors.js";
|
|
7
|
+
/** A complete offline example, deliberately without a platform execution adapter. */
|
|
8
|
+
export function scaffoldProvider(root, args) {
|
|
9
|
+
const { id, out } = args;
|
|
10
|
+
check(/^[a-z][a-z0-9-]*$/.test(id), "PROVIDER", "Provider id must use lowercase letters, digits and hyphens, starting with a letter");
|
|
11
|
+
const module = `const identifier = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
12
|
+
export function validate(plan) {
|
|
13
|
+
if (plan.nodes.length !== 1 || plan.nodes[0].platform !== ${JSON.stringify(id)}) throw new Error('One example activity is required');
|
|
14
|
+
if (!identifier.test(plan.nodes[0].with.target)) throw new Error('Invalid target identifier');
|
|
15
|
+
if (plan.nodes[0].needs.length || plan.delivery?.imports.length) throw new Error('This starter has no input adapter');
|
|
16
|
+
}
|
|
17
|
+
export function render(plan) {
|
|
18
|
+
validate(plan);
|
|
19
|
+
return {'model.sql':{format:'text',value:'CREATE VIEW '+plan.nodes[0].with.target+' AS SELECT 1 AS id;\\n'}};
|
|
20
|
+
}
|
|
21
|
+
`;
|
|
22
|
+
const contract = {
|
|
23
|
+
apiVersion: "v3.1.0",
|
|
24
|
+
kind: "DataContract",
|
|
25
|
+
id: "example",
|
|
26
|
+
name: "example",
|
|
27
|
+
version: "1.0.0",
|
|
28
|
+
status: "draft",
|
|
29
|
+
schema: [
|
|
30
|
+
{
|
|
31
|
+
name: "example",
|
|
32
|
+
logicalType: "object",
|
|
33
|
+
physicalType: "table",
|
|
34
|
+
properties: [
|
|
35
|
+
{
|
|
36
|
+
name: "id",
|
|
37
|
+
logicalType: "integer",
|
|
38
|
+
physicalType: "BIGINT",
|
|
39
|
+
required: true,
|
|
40
|
+
},
|
|
41
|
+
],
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
};
|
|
45
|
+
const files = {
|
|
46
|
+
"plugin/provider.yaml": stringify({
|
|
47
|
+
apiVersion: "ingestron.provider/v1",
|
|
48
|
+
id,
|
|
49
|
+
version: "0.1.0",
|
|
50
|
+
platform: id,
|
|
51
|
+
compatibility: {
|
|
52
|
+
plan: "ingestron.plan/v1",
|
|
53
|
+
minimumCli: "4.2.0",
|
|
54
|
+
requiredFeatures: ["native-project", "delivery-exports"],
|
|
55
|
+
},
|
|
56
|
+
capabilities: { artifactKinds: ["sql-project"], delivery: true },
|
|
57
|
+
configurationSchema: { type: "object", additionalProperties: false },
|
|
58
|
+
activities: { example: "./activity.yaml" },
|
|
59
|
+
renderer: {
|
|
60
|
+
apiVersion: "ingestron.provider-generation/v1",
|
|
61
|
+
module: "./index.mjs",
|
|
62
|
+
},
|
|
63
|
+
}),
|
|
64
|
+
"plugin/activity.yaml": stringify({
|
|
65
|
+
apiVersion: "ingestron.activity/v1",
|
|
66
|
+
id: "example",
|
|
67
|
+
version: "1.0.0",
|
|
68
|
+
description: "Synthetic SQL generator; replace with a tested platform implementation",
|
|
69
|
+
platform: id,
|
|
70
|
+
input: "relation",
|
|
71
|
+
output: "relation",
|
|
72
|
+
effect: "stage",
|
|
73
|
+
optionsSchema: {
|
|
74
|
+
type: "object",
|
|
75
|
+
additionalProperties: false,
|
|
76
|
+
required: ["target"],
|
|
77
|
+
properties: {
|
|
78
|
+
target: { type: "string", pattern: "^[A-Za-z_][A-Za-z0-9_]*$" },
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
generator: { kind: "native-project", artifactKind: "sql-project" },
|
|
82
|
+
}),
|
|
83
|
+
"plugin/index.mjs": module,
|
|
84
|
+
"project.yaml": stringify({
|
|
85
|
+
apiVersion: "ingestron.project/v1",
|
|
86
|
+
id: "example",
|
|
87
|
+
providers: {
|
|
88
|
+
packages: {
|
|
89
|
+
example: { source: "./plugin/provider.yaml", version: "0.1.0" },
|
|
90
|
+
},
|
|
91
|
+
configurations: { example: { package: "example", binding: "example" } },
|
|
92
|
+
},
|
|
93
|
+
defaults: { provider: "example" },
|
|
94
|
+
environments: {
|
|
95
|
+
dev: {
|
|
96
|
+
apiVersion: "ingestron.environment/v1",
|
|
97
|
+
environment: "dev",
|
|
98
|
+
bindings: { example: { kind: id } },
|
|
99
|
+
},
|
|
100
|
+
},
|
|
101
|
+
flows: [
|
|
102
|
+
{
|
|
103
|
+
apiVersion: "ingestron.flow/v1",
|
|
104
|
+
kind: "transformation",
|
|
105
|
+
id: "model",
|
|
106
|
+
steps: [
|
|
107
|
+
{
|
|
108
|
+
id: "example",
|
|
109
|
+
uses: "example",
|
|
110
|
+
with: { target: "example_view" },
|
|
111
|
+
},
|
|
112
|
+
],
|
|
113
|
+
publishes: {
|
|
114
|
+
example: { from: "steps.example.outputs.result", contract },
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
],
|
|
118
|
+
}),
|
|
119
|
+
"package.json": JSON.stringify({
|
|
120
|
+
name: `provider-${id}`,
|
|
121
|
+
version: "0.1.0",
|
|
122
|
+
private: true,
|
|
123
|
+
type: "module",
|
|
124
|
+
scripts: {
|
|
125
|
+
test: "node --test test/*.test.mjs",
|
|
126
|
+
check: "node scripts/check.mjs",
|
|
127
|
+
build: "node scripts/check.mjs build",
|
|
128
|
+
},
|
|
129
|
+
devDependencies: { "@ingestron/core": version },
|
|
130
|
+
engines: { node: ">=22 <23" },
|
|
131
|
+
}, null, 2) + "\n",
|
|
132
|
+
"test/provider.test.mjs": `import {test} from 'node:test';
|
|
133
|
+
import assert from 'node:assert/strict';
|
|
134
|
+
import {render} from '../plugin/index.mjs';
|
|
135
|
+
const plan=()=>({nodes:[{platform:${JSON.stringify(id)},with:{target:'example_view'},needs:[]}]});
|
|
136
|
+
test('deterministic SQL and identifier safety',()=>{assert.deepEqual(render(plan()),render(plan()));const p=plan();p.nodes[0].with.target='view; DROP TABLE data';assert.throws(()=>render(p),/identifier/);});
|
|
137
|
+
test('unsupported inputs fail closed',()=>{const p=plan();p.nodes[0].needs=['other'];assert.throws(()=>render(p),/input adapter/);});
|
|
138
|
+
`,
|
|
139
|
+
"scripts/check.mjs": `import {executeAsync} from '@ingestron/core';
|
|
140
|
+
const build=process.argv[2]==='build';
|
|
141
|
+
const result=await executeAsync({root:process.cwd(),allowWrite:build},build?'build':'provider_check',build?{out:'out'}:{delivery:true});
|
|
142
|
+
console.log(JSON.stringify(result,null,2));
|
|
143
|
+
if(!result.ok)process.exitCode=1;
|
|
144
|
+
`,
|
|
145
|
+
"README.md": `# ${id} provider starter\n\nA synthetic SQL generator demonstrating the Ingestron plugin contract. It has no platform connection or execution adapter.\n\nUse Node 22. Run npm install, npm test, npm run check and npm run build. Review out/model.sql and out/ingestron-project.json. Change the target to an invalid identifier to verify rejection.\n\nReplace the example with your platform implementation and tests before publishing. Declare only implemented capabilities. The core plugin contract is documented at https://github.com/ingestron/core/blob/main/docs/plugins.md.\n`,
|
|
146
|
+
"SECURITY.md": "Do not include secrets or source data in packages or fixtures. Compiler hooks are offline; credentials belong in customer execution context. Report suspected vulnerabilities privately to the repository owner.\n",
|
|
147
|
+
".gitignore": "out/\nnode_modules/\n.ingestron/\n",
|
|
148
|
+
};
|
|
149
|
+
const replacements = Object.fromEntries(Object.entries(files).map(([name, text]) => {
|
|
150
|
+
const path = `${out}/${name}`;
|
|
151
|
+
check(!existsSync(fence(root, path)), "OWNER", `Refusing to replace ${path}`);
|
|
152
|
+
return [path, text];
|
|
153
|
+
}));
|
|
154
|
+
return preview(root, replacements);
|
|
155
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Data-only report pack: native expressions are text, never compiler-side code. */
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
export declare const reportPackSchema: z.ZodObject<{
|
|
4
|
+
apiVersion: z.ZodLiteral<"ingestron.extension-pack/v3">;
|
|
5
|
+
kind: z.ZodLiteral<"report">;
|
|
6
|
+
id: z.ZodString;
|
|
7
|
+
version: z.ZodString;
|
|
8
|
+
description: z.ZodString;
|
|
9
|
+
platform: z.ZodString;
|
|
10
|
+
model: z.ZodString;
|
|
11
|
+
contractVersion: z.ZodLiteral<"1.0.0">;
|
|
12
|
+
report: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
13
|
+
}, z.core.$strict>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/** Data-only report pack: native expressions are text, never compiler-side code. */
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
export const reportPackSchema = z
|
|
4
|
+
.object({
|
|
5
|
+
apiVersion: z.literal("ingestron.extension-pack/v3"),
|
|
6
|
+
kind: z.literal("report"),
|
|
7
|
+
id: z.string().regex(/^[a-z][a-z0-9-]*$/),
|
|
8
|
+
version: z.string().regex(/^\d+\.\d+\.\d+$/),
|
|
9
|
+
description: z.string().min(1),
|
|
10
|
+
platform: z.string().regex(/^[a-z][a-z0-9-]*$/),
|
|
11
|
+
model: z.string().regex(/^[A-Za-z0-9_./-]+@\d+\.\d+\.\d+$/),
|
|
12
|
+
contractVersion: z.literal("1.0.0"),
|
|
13
|
+
report: z.record(z.string(), z.unknown()),
|
|
14
|
+
})
|
|
15
|
+
.strict();
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const providerSchema: z.ZodObject<{
|
|
3
|
+
package: z.ZodString;
|
|
4
|
+
binding: z.ZodString;
|
|
5
|
+
options: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
6
|
+
packs: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
7
|
+
}, z.core.$strict>;
|
|
8
|
+
export declare const environmentSchema: z.ZodObject<{
|
|
9
|
+
apiVersion: z.ZodLiteral<"ingestron.environment/v1">;
|
|
10
|
+
environment: z.ZodOptional<z.ZodString>;
|
|
11
|
+
values: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
12
|
+
bindings: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
13
|
+
}, z.core.$strict>;
|
|
14
|
+
export declare const packageReferenceSchema: z.ZodPreprocess<z.ZodObject<{
|
|
15
|
+
source: z.ZodString;
|
|
16
|
+
version: z.ZodString;
|
|
17
|
+
}, z.core.$strict>, unknown>;
|
|
18
|
+
export declare const projectSchema: z.ZodObject<{
|
|
19
|
+
apiVersion: z.ZodLiteral<"ingestron.project/v1">;
|
|
20
|
+
id: z.ZodString;
|
|
21
|
+
description: z.ZodOptional<z.ZodString>;
|
|
22
|
+
providers: z.ZodObject<{
|
|
23
|
+
packages: z.ZodRecord<z.ZodString, z.ZodPreprocess<z.ZodObject<{
|
|
24
|
+
source: z.ZodString;
|
|
25
|
+
version: z.ZodString;
|
|
26
|
+
}, z.core.$strict>, unknown>>;
|
|
27
|
+
configurations: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
28
|
+
package: z.ZodString;
|
|
29
|
+
binding: z.ZodString;
|
|
30
|
+
options: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
31
|
+
packs: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
32
|
+
}, z.core.$strict>>;
|
|
33
|
+
packs: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodPreprocess<z.ZodObject<{
|
|
34
|
+
source: z.ZodString;
|
|
35
|
+
version: z.ZodString;
|
|
36
|
+
}, z.core.$strict>, unknown>>>;
|
|
37
|
+
}, z.core.$strict>;
|
|
38
|
+
defaults: z.ZodDefault<z.ZodObject<{
|
|
39
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
40
|
+
naming: z.ZodDefault<z.ZodObject<{
|
|
41
|
+
table: z.ZodOptional<z.ZodString>;
|
|
42
|
+
job: z.ZodOptional<z.ZodString>;
|
|
43
|
+
}, z.core.$strict>>;
|
|
44
|
+
with: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
45
|
+
}, z.core.$strict>>;
|
|
46
|
+
environments: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
47
|
+
apiVersion: z.ZodLiteral<"ingestron.environment/v1">;
|
|
48
|
+
environment: z.ZodOptional<z.ZodString>;
|
|
49
|
+
values: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
50
|
+
bindings: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
51
|
+
}, z.core.$strict>>;
|
|
52
|
+
modelPacks: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodPreprocess<z.ZodObject<{
|
|
53
|
+
source: z.ZodString;
|
|
54
|
+
version: z.ZodString;
|
|
55
|
+
}, z.core.$strict>, unknown>>>;
|
|
56
|
+
connections: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
57
|
+
package: z.ZodString;
|
|
58
|
+
connector: z.ZodOptional<z.ZodString>;
|
|
59
|
+
sourceId: z.ZodString;
|
|
60
|
+
tenantId: z.ZodString;
|
|
61
|
+
binding: z.ZodOptional<z.ZodString>;
|
|
62
|
+
settings: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
63
|
+
}, z.core.$strict>>>;
|
|
64
|
+
flows: z.ZodArray<z.ZodUnknown>;
|
|
65
|
+
}, z.core.$strict>;
|
|
66
|
+
export declare const stepSchema: z.ZodObject<{
|
|
67
|
+
id: z.ZodString;
|
|
68
|
+
uses: z.ZodString;
|
|
69
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
70
|
+
with: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
71
|
+
select: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
72
|
+
}, z.core.$strict>;
|
|
73
|
+
export declare const tableSchema: z.ZodObject<{
|
|
74
|
+
source: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
75
|
+
contract: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
76
|
+
ingestion: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
77
|
+
steps: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
78
|
+
with: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
79
|
+
}, z.core.$strict>>>;
|
|
80
|
+
}, z.core.$strict>;
|
|
81
|
+
export declare const flowSchema: z.ZodObject<{
|
|
82
|
+
apiVersion: z.ZodLiteral<"ingestron.flow/v1">;
|
|
83
|
+
kind: z.ZodEnum<{
|
|
84
|
+
ingestion: "ingestion";
|
|
85
|
+
transformation: "transformation";
|
|
86
|
+
}>;
|
|
87
|
+
id: z.ZodString;
|
|
88
|
+
description: z.ZodOptional<z.ZodString>;
|
|
89
|
+
domain: z.ZodOptional<z.ZodString>;
|
|
90
|
+
export: z.ZodOptional<z.ZodString>;
|
|
91
|
+
product: z.ZodOptional<z.ZodString>;
|
|
92
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
93
|
+
defaults: z.ZodDefault<z.ZodObject<{
|
|
94
|
+
source: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
95
|
+
with: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
96
|
+
}, z.core.$strict>>;
|
|
97
|
+
ingestion: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
98
|
+
tables: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
99
|
+
source: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
100
|
+
contract: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
101
|
+
ingestion: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
102
|
+
steps: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
103
|
+
with: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
104
|
+
}, z.core.$strict>>>;
|
|
105
|
+
}, z.core.$strict>>>;
|
|
106
|
+
steps: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
107
|
+
id: z.ZodString;
|
|
108
|
+
uses: z.ZodString;
|
|
109
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
110
|
+
with: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
111
|
+
select: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
112
|
+
}, z.core.$strict>>>;
|
|
113
|
+
requires: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
114
|
+
dataset: z.ZodString;
|
|
115
|
+
handover: z.ZodOptional<z.ZodEnum<{
|
|
116
|
+
files: "files";
|
|
117
|
+
relation: "relation";
|
|
118
|
+
}>>;
|
|
119
|
+
select: z.ZodObject<{
|
|
120
|
+
mode: z.ZodLiteral<"same-run">;
|
|
121
|
+
}, z.core.$strict>;
|
|
122
|
+
}, z.core.$strict>>>;
|
|
123
|
+
publishes: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
124
|
+
from: z.ZodString;
|
|
125
|
+
contract: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
126
|
+
location: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
127
|
+
}, z.core.$strict>>>;
|
|
128
|
+
}, z.core.$strict>;
|
|
129
|
+
export type Project = z.infer<typeof projectSchema>;
|
|
130
|
+
export type Flow = z.infer<typeof flowSchema>;
|
|
131
|
+
export type Step = z.infer<typeof stepSchema>;
|
|
132
|
+
export type Table = z.infer<typeof tableSchema>;
|
|
133
|
+
export type Platform = string;
|
|
134
|
+
export interface Node {
|
|
135
|
+
id: string;
|
|
136
|
+
flow: string;
|
|
137
|
+
table?: string;
|
|
138
|
+
step: string;
|
|
139
|
+
exportGroup?: string;
|
|
140
|
+
resources?: {
|
|
141
|
+
scope: string;
|
|
142
|
+
kind: string;
|
|
143
|
+
name: string;
|
|
144
|
+
}[];
|
|
145
|
+
uses: string;
|
|
146
|
+
platform: Platform;
|
|
147
|
+
runtime?: {
|
|
148
|
+
implementation: "native";
|
|
149
|
+
configuration?: string;
|
|
150
|
+
capabilities?: {
|
|
151
|
+
delivery: boolean;
|
|
152
|
+
artifactKinds: string[];
|
|
153
|
+
handovers: ("files" | "relation")[];
|
|
154
|
+
packContracts: Record<string, {
|
|
155
|
+
version: string;
|
|
156
|
+
activities: string[];
|
|
157
|
+
}>;
|
|
158
|
+
};
|
|
159
|
+
version?: string;
|
|
160
|
+
options?: Record<string, any>;
|
|
161
|
+
outputSchemas?: Record<string, string>;
|
|
162
|
+
outputSchemasRef?: string;
|
|
163
|
+
lifecycleCode?: string;
|
|
164
|
+
lifecycleRef?: string;
|
|
165
|
+
rendererCode?: string;
|
|
166
|
+
rendererRef?: string;
|
|
167
|
+
};
|
|
168
|
+
binding: string;
|
|
169
|
+
with: Record<string, any>;
|
|
170
|
+
needs: string[];
|
|
171
|
+
source?: Record<string, any>;
|
|
172
|
+
contract?: Record<string, any>;
|
|
173
|
+
output: string;
|
|
174
|
+
interface: {
|
|
175
|
+
input: string;
|
|
176
|
+
output: string;
|
|
177
|
+
};
|
|
178
|
+
generation?: {
|
|
179
|
+
kind: string;
|
|
180
|
+
artifactKind?: string;
|
|
181
|
+
entrypoint: string;
|
|
182
|
+
runtime?: string;
|
|
183
|
+
runtimeRef?: string;
|
|
184
|
+
execution?: string;
|
|
185
|
+
persistence?: "capture" | "ephemeral" | "durable";
|
|
186
|
+
code?: string;
|
|
187
|
+
codeRef?: string;
|
|
188
|
+
scope?: "table" | "collection";
|
|
189
|
+
modes?: ("table" | "collection")[];
|
|
190
|
+
groupBy?: string[];
|
|
191
|
+
dependencyOptions?: Record<string, string>;
|
|
192
|
+
requirements?: {
|
|
193
|
+
execution: string;
|
|
194
|
+
features: string[];
|
|
195
|
+
sourceKinds?: string[];
|
|
196
|
+
evidence: "offline";
|
|
197
|
+
};
|
|
198
|
+
};
|
|
199
|
+
implementation?: {
|
|
200
|
+
templates: {
|
|
201
|
+
content: string;
|
|
202
|
+
output: string;
|
|
203
|
+
format: string;
|
|
204
|
+
engine?: "jinja";
|
|
205
|
+
}[];
|
|
206
|
+
dependencies: Record<string, string>;
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
export interface Plan {
|
|
210
|
+
apiVersion: "ingestron.plan/v1";
|
|
211
|
+
delivery?: {
|
|
212
|
+
exportId: string;
|
|
213
|
+
imports: {
|
|
214
|
+
dataset: string;
|
|
215
|
+
producer: string;
|
|
216
|
+
kind: "files" | "relation";
|
|
217
|
+
location: Record<string, unknown>;
|
|
218
|
+
contract: Record<string, unknown>;
|
|
219
|
+
}[];
|
|
220
|
+
};
|
|
221
|
+
compilerVersion: string;
|
|
222
|
+
compilerDigest: string;
|
|
223
|
+
names: {
|
|
224
|
+
jobs: Record<string, string>;
|
|
225
|
+
nodes: Record<string, string>;
|
|
226
|
+
};
|
|
227
|
+
inputDigests: Record<string, string>;
|
|
228
|
+
selection: {
|
|
229
|
+
flow?: string;
|
|
230
|
+
table?: string;
|
|
231
|
+
step?: string;
|
|
232
|
+
delivery?: boolean;
|
|
233
|
+
projectBuild?: boolean;
|
|
234
|
+
provider?: string;
|
|
235
|
+
};
|
|
236
|
+
project: string;
|
|
237
|
+
environment: string;
|
|
238
|
+
bindings: Record<string, any>;
|
|
239
|
+
nodes: Node[];
|
|
240
|
+
activitySources: Record<string, string>;
|
|
241
|
+
flows: Flow[];
|
|
242
|
+
files: Record<string, string>;
|
|
243
|
+
datasets: Record<string, {
|
|
244
|
+
producer: string;
|
|
245
|
+
port?: string;
|
|
246
|
+
contract: Record<string, any>;
|
|
247
|
+
location?: Record<string, any>;
|
|
248
|
+
}>;
|
|
249
|
+
recovery: Record<string, any>[];
|
|
250
|
+
pending: ConfigurationPending[];
|
|
251
|
+
digest: string;
|
|
252
|
+
}
|
|
253
|
+
export interface ConfigurationPending {
|
|
254
|
+
name: string;
|
|
255
|
+
file?: string;
|
|
256
|
+
pointer?: string;
|
|
257
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
const id = z.string().regex(/^[A-Za-z_][A-Za-z0-9_-]*$/);
|
|
3
|
+
const map = z.record(z.string(), z.unknown());
|
|
4
|
+
export const providerSchema = z
|
|
5
|
+
.object({
|
|
6
|
+
package: id,
|
|
7
|
+
binding: id,
|
|
8
|
+
options: map.default({}),
|
|
9
|
+
packs: z.array(id).default([]),
|
|
10
|
+
})
|
|
11
|
+
.strict();
|
|
12
|
+
export const environmentSchema = z
|
|
13
|
+
.object({
|
|
14
|
+
apiVersion: z.literal("ingestron.environment/v1"),
|
|
15
|
+
environment: id.optional(),
|
|
16
|
+
values: map.default({}),
|
|
17
|
+
bindings: z.record(id, map).default({}),
|
|
18
|
+
})
|
|
19
|
+
.strict();
|
|
20
|
+
export const packageReferenceSchema = z.preprocess((value) => {
|
|
21
|
+
if (typeof value !== "string")
|
|
22
|
+
return value;
|
|
23
|
+
const at = value.lastIndexOf("@");
|
|
24
|
+
return at > 0 && at < value.length - 1
|
|
25
|
+
? { source: value.slice(0, at), version: value.slice(at + 1) }
|
|
26
|
+
: value;
|
|
27
|
+
}, z.object({ source: z.string().min(1), version: z.string().min(1) }).strict());
|
|
28
|
+
export const projectSchema = z
|
|
29
|
+
.object({
|
|
30
|
+
apiVersion: z.literal("ingestron.project/v1"),
|
|
31
|
+
id,
|
|
32
|
+
description: z.string().optional(),
|
|
33
|
+
providers: z
|
|
34
|
+
.object({
|
|
35
|
+
packages: z.record(id, packageReferenceSchema),
|
|
36
|
+
configurations: z.record(id, providerSchema),
|
|
37
|
+
packs: z.record(id, packageReferenceSchema).default({}),
|
|
38
|
+
})
|
|
39
|
+
.strict(),
|
|
40
|
+
defaults: z
|
|
41
|
+
.object({
|
|
42
|
+
provider: id.optional(),
|
|
43
|
+
naming: z
|
|
44
|
+
.object({
|
|
45
|
+
table: z.string().optional(),
|
|
46
|
+
job: z.string().optional(),
|
|
47
|
+
})
|
|
48
|
+
.strict()
|
|
49
|
+
.default({}),
|
|
50
|
+
with: map.default({}),
|
|
51
|
+
})
|
|
52
|
+
.strict()
|
|
53
|
+
.default({ naming: {}, with: {} }),
|
|
54
|
+
environments: z.record(id, environmentSchema),
|
|
55
|
+
modelPacks: z.record(id, packageReferenceSchema).default({}),
|
|
56
|
+
connections: z
|
|
57
|
+
.record(id, z
|
|
58
|
+
.object({
|
|
59
|
+
package: id,
|
|
60
|
+
connector: z.string().min(1).optional(),
|
|
61
|
+
sourceId: id,
|
|
62
|
+
tenantId: id,
|
|
63
|
+
binding: id.optional(),
|
|
64
|
+
settings: map.default({}),
|
|
65
|
+
})
|
|
66
|
+
.strict())
|
|
67
|
+
.default({}),
|
|
68
|
+
flows: z.array(z.unknown()),
|
|
69
|
+
})
|
|
70
|
+
.strict();
|
|
71
|
+
export const stepSchema = z
|
|
72
|
+
.object({
|
|
73
|
+
id,
|
|
74
|
+
uses: z.string().min(1),
|
|
75
|
+
provider: id.optional(),
|
|
76
|
+
with: map.default({}),
|
|
77
|
+
select: z.array(id).min(1).optional(),
|
|
78
|
+
})
|
|
79
|
+
.strict();
|
|
80
|
+
export const tableSchema = z
|
|
81
|
+
.object({
|
|
82
|
+
source: map.default({}),
|
|
83
|
+
contract: map,
|
|
84
|
+
ingestion: map.optional(),
|
|
85
|
+
steps: z
|
|
86
|
+
.record(id, z.object({ with: map.default({}) }).strict())
|
|
87
|
+
.default({}),
|
|
88
|
+
})
|
|
89
|
+
.strict();
|
|
90
|
+
export const flowSchema = z
|
|
91
|
+
.object({
|
|
92
|
+
apiVersion: z.literal("ingestron.flow/v1"),
|
|
93
|
+
kind: z.enum(["ingestion", "transformation"]),
|
|
94
|
+
id,
|
|
95
|
+
description: z.string().optional(),
|
|
96
|
+
domain: id.optional(),
|
|
97
|
+
export: id.optional(),
|
|
98
|
+
product: id.optional(),
|
|
99
|
+
provider: id.optional(),
|
|
100
|
+
defaults: z
|
|
101
|
+
.object({ source: map.default({}), with: map.default({}) })
|
|
102
|
+
.strict()
|
|
103
|
+
.default({ source: {}, with: {} }),
|
|
104
|
+
ingestion: map.optional(),
|
|
105
|
+
tables: z.record(id, tableSchema).optional(),
|
|
106
|
+
steps: z.array(stepSchema).min(1).optional(),
|
|
107
|
+
requires: z
|
|
108
|
+
.record(id, z
|
|
109
|
+
.object({
|
|
110
|
+
dataset: z.string(),
|
|
111
|
+
handover: z.enum(["files", "relation"]).optional(),
|
|
112
|
+
select: z
|
|
113
|
+
.object({
|
|
114
|
+
mode: z.literal("same-run"),
|
|
115
|
+
})
|
|
116
|
+
.strict(),
|
|
117
|
+
})
|
|
118
|
+
.strict())
|
|
119
|
+
.default({}),
|
|
120
|
+
publishes: z
|
|
121
|
+
.record(id, z
|
|
122
|
+
.object({ from: z.string(), contract: map, location: map.optional() })
|
|
123
|
+
.strict())
|
|
124
|
+
.default({}),
|
|
125
|
+
})
|
|
126
|
+
.strict();
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export declare const identifier: z.ZodString;
|
|
3
|
+
export declare const sourceSchema: z.ZodObject<{
|
|
4
|
+
apiVersion: z.ZodLiteral<"ingestron.source/v1">;
|
|
5
|
+
id: z.ZodString;
|
|
6
|
+
type: z.ZodString;
|
|
7
|
+
format: z.ZodOptional<z.ZodString>;
|
|
8
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
9
|
+
execution: z.ZodDefault<z.ZodEnum<{
|
|
10
|
+
local: "local";
|
|
11
|
+
manual: "manual";
|
|
12
|
+
}>>;
|
|
13
|
+
binding: z.ZodOptional<z.ZodString>;
|
|
14
|
+
environments: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
15
|
+
binding: z.ZodOptional<z.ZodString>;
|
|
16
|
+
provider: z.ZodOptional<z.ZodString>;
|
|
17
|
+
execution: z.ZodOptional<z.ZodEnum<{
|
|
18
|
+
local: "local";
|
|
19
|
+
manual: "manual";
|
|
20
|
+
}>>;
|
|
21
|
+
resultPath: z.ZodOptional<z.ZodString>;
|
|
22
|
+
}, z.core.$strict>>>;
|
|
23
|
+
}, z.core.$strict>;
|
|
24
|
+
export declare function sourcePath(id: string): string;
|
|
25
|
+
export declare const metadataPath: (id: string, environment: string) => string;
|
|
26
|
+
export declare function sourceShow(root: string, id: string, environment: string): {
|
|
27
|
+
apiVersion: "ingestron.source/v1";
|
|
28
|
+
id: string;
|
|
29
|
+
type: string;
|
|
30
|
+
format?: string | undefined;
|
|
31
|
+
provider?: string | undefined;
|
|
32
|
+
execution: "local" | "manual";
|
|
33
|
+
binding?: string | undefined;
|
|
34
|
+
environments: Record<string, {
|
|
35
|
+
binding?: string | undefined;
|
|
36
|
+
provider?: string | undefined;
|
|
37
|
+
execution?: "local" | "manual" | undefined;
|
|
38
|
+
resultPath?: string | undefined;
|
|
39
|
+
}>;
|
|
40
|
+
discovery: {
|
|
41
|
+
path: string;
|
|
42
|
+
inputDigest: any;
|
|
43
|
+
stale: boolean;
|
|
44
|
+
entities: any;
|
|
45
|
+
imported?: undefined;
|
|
46
|
+
} | {
|
|
47
|
+
inputDigest?: undefined;
|
|
48
|
+
stale?: undefined;
|
|
49
|
+
entities?: undefined;
|
|
50
|
+
path: string;
|
|
51
|
+
imported: boolean;
|
|
52
|
+
};
|
|
53
|
+
selected: {
|
|
54
|
+
resultPath?: string | undefined;
|
|
55
|
+
apiVersion: "ingestron.source/v1";
|
|
56
|
+
id: string;
|
|
57
|
+
type: string;
|
|
58
|
+
format?: string | undefined;
|
|
59
|
+
provider?: string | undefined;
|
|
60
|
+
execution: "local" | "manual";
|
|
61
|
+
binding?: string | undefined;
|
|
62
|
+
environments: Record<string, {
|
|
63
|
+
binding?: string | undefined;
|
|
64
|
+
provider?: string | undefined;
|
|
65
|
+
execution?: "local" | "manual" | undefined;
|
|
66
|
+
resultPath?: string | undefined;
|
|
67
|
+
}>;
|
|
68
|
+
environment: string;
|
|
69
|
+
};
|
|
70
|
+
};
|
|
71
|
+
export declare function sourceList(root: string, environment: string): {
|
|
72
|
+
id: string;
|
|
73
|
+
type: string;
|
|
74
|
+
format: string | undefined;
|
|
75
|
+
execution: "local" | "manual";
|
|
76
|
+
provider: string | undefined;
|
|
77
|
+
}[];
|
|
78
|
+
export declare function sourceAdd(root: string, args: any): import("./authoring.js").Proposal;
|
|
79
|
+
export declare function sourceConfigure(root: string, args: any): import("./authoring.js").Proposal;
|
|
80
|
+
export declare function pluginConfigure(root: string, args: any): import("./authoring.js").Proposal;
|
|
81
|
+
export declare function sourcePrepare(root: string, environment: string, args: any): import("./authoring.js").Proposal;
|
|
82
|
+
export declare function sourceImport(root: string, environment: string, args: any): import("./authoring.js").Proposal;
|
|
83
|
+
export declare function draftContract(root: string, environment: string, args: any): import("./authoring.js").Proposal;
|
|
84
|
+
export declare function contractPath(root: string, id: string): string;
|
|
85
|
+
export declare function contractList(root: string): {
|
|
86
|
+
id: any;
|
|
87
|
+
status: any;
|
|
88
|
+
path: string;
|
|
89
|
+
entities: any;
|
|
90
|
+
}[];
|
|
91
|
+
export declare function flowCreate(root: string, environment: string, args: any): import("./authoring.js").Proposal;
|