@ingestron/core 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +6 -0
- package/README.md +56 -0
- package/SECURITY.md +44 -0
- package/THIRD_PARTY_NOTICES.md +693 -0
- package/dist/assets/schemas/ODCS-LICENSE +201 -0
- package/dist/assets/schemas/odcs-3.1.0.json +2929 -0
- package/dist/assets/schemas/provenance.json +16 -0
- package/dist/compiler-fingerprint.json +84 -0
- package/dist/core/authoring.d.ts +55 -0
- package/dist/core/authoring.js +315 -0
- package/dist/core/config.d.ts +34 -0
- package/dist/core/config.js +233 -0
- package/dist/core/connections.d.ts +19 -0
- package/dist/core/connections.js +203 -0
- package/dist/core/connector-package.d.ts +37 -0
- package/dist/core/connector-package.js +43 -0
- package/dist/core/contracts.d.ts +8 -0
- package/dist/core/contracts.js +45 -0
- package/dist/core/deliveries.d.ts +16 -0
- package/dist/core/deliveries.js +28 -0
- package/dist/core/delivery-exports.d.ts +20 -0
- package/dist/core/delivery-exports.js +119 -0
- package/dist/core/development.d.ts +50 -0
- package/dist/core/development.js +94 -0
- package/dist/core/ecosystem-authoring.d.ts +13 -0
- package/dist/core/ecosystem-authoring.js +82 -0
- package/dist/core/errors.d.ts +12 -0
- package/dist/core/errors.js +43 -0
- package/dist/core/execution.d.ts +28 -0
- package/dist/core/execution.js +297 -0
- package/dist/core/extension-packs.d.ts +8 -0
- package/dist/core/extension-packs.js +34 -0
- package/dist/core/fingerprint.d.ts +1 -0
- package/dist/core/fingerprint.js +16 -0
- package/dist/core/generate.d.ts +16 -0
- package/dist/core/generate.js +198 -0
- package/dist/core/installation.d.ts +3 -0
- package/dist/core/installation.js +14 -0
- package/dist/core/installed-plugins.d.ts +41 -0
- package/dist/core/installed-plugins.js +92 -0
- package/dist/core/model-pack-schema.d.ts +46 -0
- package/dist/core/model-pack-schema.js +81 -0
- package/dist/core/model-packs.d.ts +4 -0
- package/dist/core/model-packs.js +33 -0
- package/dist/core/operations.d.ts +346 -0
- package/dist/core/operations.js +760 -0
- package/dist/core/package-references.d.ts +36 -0
- package/dist/core/package-references.js +103 -0
- package/dist/core/packages.d.ts +340 -0
- package/dist/core/packages.js +579 -0
- package/dist/core/planner.d.ts +114 -0
- package/dist/core/planner.js +821 -0
- package/dist/core/project-build.d.ts +13 -0
- package/dist/core/project-build.js +266 -0
- package/dist/core/provider-commands.d.ts +83 -0
- package/dist/core/provider-commands.js +252 -0
- package/dist/core/provider-starter.d.ts +5 -0
- package/dist/core/provider-starter.js +155 -0
- package/dist/core/report-pack-schema.d.ts +13 -0
- package/dist/core/report-pack-schema.js +15 -0
- package/dist/core/schema.d.ts +257 -0
- package/dist/core/schema.js +126 -0
- package/dist/core/workflows.d.ts +91 -0
- package/dist/core/workflows.js +438 -0
- package/dist/plugins/authoring.d.ts +88 -0
- package/dist/plugins/authoring.js +34 -0
- package/dist/plugins/compatibility.d.ts +23 -0
- package/dist/plugins/compatibility.js +55 -0
- package/dist/plugins/generation.d.ts +5 -0
- package/dist/plugins/generation.js +105 -0
- package/dist/plugins/provider-renderer.d.ts +7 -0
- package/dist/plugins/provider-renderer.js +37 -0
- package/dist/plugins/provider-worker.d.ts +1 -0
- package/dist/plugins/provider-worker.js +74 -0
- package/dist/sdk/adapter.d.ts +15 -0
- package/dist/sdk/adapter.js +15 -0
- package/dist/sdk/index.d.ts +4 -0
- package/dist/sdk/index.js +3 -0
- package/dist/sdk/schemas.d.ts +34 -0
- package/dist/sdk/schemas.js +74 -0
- package/dist/version.d.ts +5 -0
- package/dist/version.js +7 -0
- package/package.json +72 -0
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
import { packageReferenceSchema } from "./schema.js";
|
|
2
|
+
import { connectorPackageSchema } from "./connector-package.js";
|
|
3
|
+
/** User workflows shared by CLI and MCP. These compose the existing offline
|
|
4
|
+
* compiler and isolated plugin APIs. Execution adapters must be separate from
|
|
5
|
+
* this module: importing metadata never grants permission to run SQL or scripts.
|
|
6
|
+
*/
|
|
7
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
8
|
+
import { resolve, relative, dirname } from "node:path";
|
|
9
|
+
import { parseDocument, stringify } from "yaml";
|
|
10
|
+
import { z } from "zod";
|
|
11
|
+
import { Configuration } from "./config.js";
|
|
12
|
+
import { check, digest, canonical } from "./errors.js";
|
|
13
|
+
import { preview, addFlow } from "./authoring.js";
|
|
14
|
+
import { contractDraft } from "./development.js";
|
|
15
|
+
import { validateContract, contractColumns } from "./contracts.js";
|
|
16
|
+
import { providerAuthor, providerReference } from "../plugins/authoring.js";
|
|
17
|
+
import { runProviderCommand } from "./provider-commands.js";
|
|
18
|
+
import { fence, resolvePackage, packageYaml, canonicalPackageReference, } from "./packages.js";
|
|
19
|
+
import { friendlyReference } from "./package-references.js";
|
|
20
|
+
export const identifier = z.string().regex(/^[A-Za-z_][A-Za-z0-9_-]*$/);
|
|
21
|
+
const connection = z
|
|
22
|
+
.object({
|
|
23
|
+
binding: identifier.optional(),
|
|
24
|
+
provider: identifier.optional(),
|
|
25
|
+
execution: z.enum(["manual", "local"]).optional(),
|
|
26
|
+
resultPath: z.string().max(1024).optional(),
|
|
27
|
+
})
|
|
28
|
+
.strict();
|
|
29
|
+
export const sourceSchema = z
|
|
30
|
+
.object({
|
|
31
|
+
apiVersion: z.literal("ingestron.source/v1"),
|
|
32
|
+
id: identifier,
|
|
33
|
+
type: identifier,
|
|
34
|
+
format: identifier.optional(),
|
|
35
|
+
provider: identifier.optional(),
|
|
36
|
+
execution: z.enum(["manual", "local"]).default("manual"),
|
|
37
|
+
binding: identifier.optional(),
|
|
38
|
+
environments: z.record(identifier, connection).default({}),
|
|
39
|
+
})
|
|
40
|
+
.strict();
|
|
41
|
+
export function sourcePath(id) {
|
|
42
|
+
return `sources/${identifier.parse(id)}.yaml`;
|
|
43
|
+
}
|
|
44
|
+
export const metadataPath = (id, environment) => `discovery/${identifier.parse(id)}/${identifier.parse(environment)}/metadata.json`;
|
|
45
|
+
export function sourceShow(root, id, environment) {
|
|
46
|
+
const source = sourceSchema.parse(new Configuration(root).load(sourcePath(id)));
|
|
47
|
+
check(source.id === id, "SOURCE", "Source filename and identifier differ");
|
|
48
|
+
const reader = new Configuration(root);
|
|
49
|
+
check(reader.load().environments[environment], "ENVIRONMENT", `Unknown environment ${environment}`);
|
|
50
|
+
const path = metadataPath(id, environment);
|
|
51
|
+
const snapshot = existsSync(fence(root, path))
|
|
52
|
+
? reader.load(path)
|
|
53
|
+
: undefined;
|
|
54
|
+
const discovery = snapshot
|
|
55
|
+
? {
|
|
56
|
+
path,
|
|
57
|
+
inputDigest: snapshot.inputDigest,
|
|
58
|
+
stale: snapshot.sourceDigest !== digest(reader.text(sourcePath(id))),
|
|
59
|
+
entities: snapshot.contracts?.map((c) => ({
|
|
60
|
+
id: c.id,
|
|
61
|
+
physicalNames: c.schema?.map((s) => s.physicalName ?? s.name),
|
|
62
|
+
})),
|
|
63
|
+
}
|
|
64
|
+
: { path, imported: false };
|
|
65
|
+
return {
|
|
66
|
+
...source,
|
|
67
|
+
discovery,
|
|
68
|
+
selected: { environment, ...source, ...source.environments[environment] },
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
export function sourceList(root, environment) {
|
|
72
|
+
const directory = fence(root, "sources");
|
|
73
|
+
return existsSync(directory)
|
|
74
|
+
? readdirSync(directory)
|
|
75
|
+
.filter((n) => n.endsWith(".yaml"))
|
|
76
|
+
.sort()
|
|
77
|
+
.map((n) => {
|
|
78
|
+
const source = sourceShow(root, n.slice(0, -5), environment);
|
|
79
|
+
return {
|
|
80
|
+
id: source.id,
|
|
81
|
+
type: source.type,
|
|
82
|
+
format: source.format,
|
|
83
|
+
execution: source.selected.execution,
|
|
84
|
+
provider: source.selected.provider,
|
|
85
|
+
};
|
|
86
|
+
})
|
|
87
|
+
: [];
|
|
88
|
+
}
|
|
89
|
+
export function sourceAdd(root, args) {
|
|
90
|
+
const { id, type, format, provider, execution, binding } = args;
|
|
91
|
+
const path = sourcePath(id);
|
|
92
|
+
check(!existsSync(fence(root, path)), "OWNER", `Source ${id} already exists`);
|
|
93
|
+
const source = sourceSchema.parse({
|
|
94
|
+
apiVersion: "ingestron.source/v1",
|
|
95
|
+
id,
|
|
96
|
+
type,
|
|
97
|
+
format,
|
|
98
|
+
provider,
|
|
99
|
+
execution,
|
|
100
|
+
binding,
|
|
101
|
+
});
|
|
102
|
+
return preview(root, { [path]: stringify(source) });
|
|
103
|
+
}
|
|
104
|
+
export function sourceConfigure(root, args) {
|
|
105
|
+
const path = sourcePath(args.id), reader = new Configuration(root);
|
|
106
|
+
const original = sourceSchema.parse(reader.load(path));
|
|
107
|
+
check(!("id" in args.patch) && !("apiVersion" in args.patch), "SOURCE", "Source identity cannot be changed by a patch");
|
|
108
|
+
const next = sourceSchema.parse({ ...original, ...args.patch });
|
|
109
|
+
return preview(root, { [path]: stringify(next) });
|
|
110
|
+
}
|
|
111
|
+
function providerConfiguration(root, selected) {
|
|
112
|
+
const project = new Configuration(root).load();
|
|
113
|
+
const configurations = project.providers.configurations;
|
|
114
|
+
if (!selected)
|
|
115
|
+
selected = project.defaults?.provider;
|
|
116
|
+
if (selected && configurations[selected])
|
|
117
|
+
return selected;
|
|
118
|
+
const matches = Object.entries(configurations).filter(([, value]) => value.package === selected);
|
|
119
|
+
check(matches.length === 1, "PROVIDER", "Select an installed provider configuration by name");
|
|
120
|
+
return matches[0][0];
|
|
121
|
+
}
|
|
122
|
+
/** Register installed provider authoring defaults without overwriting another
|
|
123
|
+
* provider's bindings or values. Cache installation and registration are distinct
|
|
124
|
+
* shared operations; a registration conflict leaves the immutable cache usable. */
|
|
125
|
+
function packageEditor(root) {
|
|
126
|
+
const reader = new Configuration(root);
|
|
127
|
+
const project = parseDocument(reader.text("project.yaml"));
|
|
128
|
+
const reference = project.toJS().providers?.$resolve;
|
|
129
|
+
let providerFile;
|
|
130
|
+
let providerDoc;
|
|
131
|
+
if (reference) {
|
|
132
|
+
check(typeof reference === "string" && !reference.includes("#"), "REFERENCE", "Plugin registration requires a whole providers file, not a fragment");
|
|
133
|
+
providerFile = relative(root, reader.path(reference));
|
|
134
|
+
providerDoc = parseDocument(reader.text(providerFile));
|
|
135
|
+
check(!providerDoc.toJS().packages?.$resolve &&
|
|
136
|
+
!providerDoc.toJS().configurations?.$resolve, "REFERENCE", "Keep packages and configurations in the providers file for automatic registration");
|
|
137
|
+
project.set("providers", providerDoc.contents?.clone());
|
|
138
|
+
}
|
|
139
|
+
const finish = (files) => {
|
|
140
|
+
if (providerFile && providerDoc && files["project.yaml"]) {
|
|
141
|
+
const edited = parseDocument(files["project.yaml"]);
|
|
142
|
+
providerDoc.contents = edited.get("providers", true);
|
|
143
|
+
files[providerFile] = providerDoc.toString();
|
|
144
|
+
edited.set("providers", { $resolve: reference });
|
|
145
|
+
files["project.yaml"] = edited.toString();
|
|
146
|
+
}
|
|
147
|
+
return preview(root, files);
|
|
148
|
+
};
|
|
149
|
+
return { reader, project, finish };
|
|
150
|
+
}
|
|
151
|
+
export function pluginConfigure(root, args) {
|
|
152
|
+
const reference = canonicalPackageReference(args.reference);
|
|
153
|
+
const sourceManifest = packageYaml(resolvePackage(root, reference).file);
|
|
154
|
+
if (sourceManifest.apiVersion === "ingestron.connector/v1") {
|
|
155
|
+
const connector = connectorPackageSchema.parse(sourceManifest);
|
|
156
|
+
const { project, finish } = packageEditor(root);
|
|
157
|
+
const name = identifier.parse(args.name ?? connector.id);
|
|
158
|
+
const at = reference.lastIndexOf("@");
|
|
159
|
+
const friendly = friendlyReference(reference);
|
|
160
|
+
const value = {
|
|
161
|
+
source: friendly.slice(0, friendly.lastIndexOf("@")),
|
|
162
|
+
version: reference.slice(at + 1),
|
|
163
|
+
};
|
|
164
|
+
const previous = project.getIn(["providers", "packages", name]);
|
|
165
|
+
if (previous) {
|
|
166
|
+
const old = packageReferenceSchema.parse(project.toJS().providers.packages[name]);
|
|
167
|
+
check(canonical(old) === canonical(value) ||
|
|
168
|
+
(args.update && old.source === value.source), "CONFLICT", "Connector package key already exists; choose a different name");
|
|
169
|
+
}
|
|
170
|
+
project.setIn(["providers", "packages", name], `${value.source}@${value.version}`);
|
|
171
|
+
return finish({ "project.yaml": project.toString() });
|
|
172
|
+
}
|
|
173
|
+
const { reader, project, finish } = packageEditor(root);
|
|
174
|
+
const raw = project.toJS(), selected = providerReference(root, args.reference);
|
|
175
|
+
const name = identifier.parse(args.name ?? selected.manifest.platform);
|
|
176
|
+
const configurations = raw.providers.configurations;
|
|
177
|
+
if (configurations[name]) {
|
|
178
|
+
const existing = packageReferenceSchema.parse(raw.providers.packages[configurations[name].package]);
|
|
179
|
+
check(args.update ||
|
|
180
|
+
(friendlyReference(existing.source) === selected.source &&
|
|
181
|
+
existing.version === selected.version), "CONFLICT", `Provider configuration ${name} already exists; use provider_migrate or a different name`);
|
|
182
|
+
if (args.update) {
|
|
183
|
+
check(friendlyReference(existing.source) === selected.source, "CONFLICT", "Use plugin migrate to change provider identity");
|
|
184
|
+
project.setIn(["providers", "packages", configurations[name].package], `${selected.source}@${selected.version}`);
|
|
185
|
+
return finish({ "project.yaml": project.toString() });
|
|
186
|
+
}
|
|
187
|
+
if (existing.source !== selected.source) {
|
|
188
|
+
project.setIn(["providers", "packages", configurations[name].package], `${selected.source}@${selected.version}`);
|
|
189
|
+
return finish({ "project.yaml": project.toString() });
|
|
190
|
+
}
|
|
191
|
+
return finish({});
|
|
192
|
+
}
|
|
193
|
+
check(!raw.providers.packages[name], "CONFLICT", `Package key ${name} already exists`);
|
|
194
|
+
const environmentNames = Object.keys(raw.environments);
|
|
195
|
+
const files = providerAuthor(root, args.reference, {
|
|
196
|
+
operation: "initialise",
|
|
197
|
+
options: {
|
|
198
|
+
id: raw.id,
|
|
199
|
+
provider: args.reference,
|
|
200
|
+
environments: environmentNames,
|
|
201
|
+
},
|
|
202
|
+
}, (n) => ["project.yaml", "README.md", ".gitignore"].includes(n) ||
|
|
203
|
+
/^environments\/[\w-]+\.yaml$/.test(n));
|
|
204
|
+
const skeleton = parseDocument(files["project.yaml"]).toJS();
|
|
205
|
+
const defaults = skeleton.providers.configurations[skeleton.defaults.provider];
|
|
206
|
+
check(defaults, "PROVIDER", "Provider did not supply its default configuration");
|
|
207
|
+
project.setIn(["providers", "packages", name], `${selected.source}@${selected.version}`);
|
|
208
|
+
const bindingName = `${name}_${defaults.binding}`;
|
|
209
|
+
project.setIn(["providers", "configurations", name], {
|
|
210
|
+
...defaults,
|
|
211
|
+
package: name,
|
|
212
|
+
binding: bindingName,
|
|
213
|
+
});
|
|
214
|
+
if (!raw.defaults?.provider)
|
|
215
|
+
project.setIn(["defaults", "provider"], name);
|
|
216
|
+
const replacements = {};
|
|
217
|
+
for (const env of environmentNames) {
|
|
218
|
+
const reference = raw.environments[env];
|
|
219
|
+
const envPath = reference.$resolve
|
|
220
|
+
? relative(root, reader.path(reference.$resolve))
|
|
221
|
+
: undefined;
|
|
222
|
+
const current = envPath
|
|
223
|
+
? parseDocument(reader.text(envPath))
|
|
224
|
+
: parseDocument(stringify(reference));
|
|
225
|
+
const generated = parseDocument(files[`environments/${env}.yaml`]).toJS();
|
|
226
|
+
for (const group of ["values", "bindings"])
|
|
227
|
+
for (const [originalKey, value] of Object.entries(generated[group] ?? {})) {
|
|
228
|
+
const key = group === "bindings" && originalKey === defaults.binding
|
|
229
|
+
? bindingName
|
|
230
|
+
: originalKey;
|
|
231
|
+
if (current.hasIn([group, key]))
|
|
232
|
+
check(canonical(current.toJS()[group][key]) === canonical(value), "CONFLICT", `Environment ${env} ${group}.${key} conflicts with provider defaults; configure explicitly before registration`);
|
|
233
|
+
else
|
|
234
|
+
current.setIn([group, key], value);
|
|
235
|
+
}
|
|
236
|
+
if (envPath)
|
|
237
|
+
replacements[envPath] = current.toString();
|
|
238
|
+
else
|
|
239
|
+
project.setIn(["environments", env], current.toJS());
|
|
240
|
+
}
|
|
241
|
+
replacements["project.yaml"] = project.toString();
|
|
242
|
+
return finish(replacements);
|
|
243
|
+
}
|
|
244
|
+
function artifactFiles(envelope) {
|
|
245
|
+
check(envelope.apiVersion === "ingestron.provider-command-result/v1" &&
|
|
246
|
+
envelope.execution === "offline-json", "PROVIDER", "Expected an offline provider result");
|
|
247
|
+
const artifacts = envelope.result?.artifacts;
|
|
248
|
+
check(envelope.result?.apiVersion === "ingestron.artifact-proposal/v1" &&
|
|
249
|
+
artifacts &&
|
|
250
|
+
typeof artifacts === "object" &&
|
|
251
|
+
!Array.isArray(artifacts), "PROVIDER", "Provider did not return file artifacts");
|
|
252
|
+
check(Object.keys(artifacts).length > 0 && Object.keys(artifacts).length <= 200, "LIMIT", "Expected 1–200 artifacts");
|
|
253
|
+
for (const [name, text] of Object.entries(artifacts))
|
|
254
|
+
check(/^[A-Za-z0-9_-][A-Za-z0-9_.-]*\.(json|yaml|yml|sql|py|mjs|txt|md)$/.test(name) &&
|
|
255
|
+
typeof text === "string" &&
|
|
256
|
+
text.length <= 2000000, "PROVIDER", "Invalid artifact filename or content");
|
|
257
|
+
return artifacts;
|
|
258
|
+
}
|
|
259
|
+
export function sourcePrepare(root, environment, args) {
|
|
260
|
+
const source = sourceShow(root, args.id, environment).selected;
|
|
261
|
+
check(source.execution === "local" || source.execution === "manual", "NOT_IMPLEMENTED", "Source preparation supports manual and local execution modes only.");
|
|
262
|
+
const input = new Configuration(root).load(args.input);
|
|
263
|
+
check(!input.sourceId || input.sourceId === source.id, "SOURCE", "Reader input belongs to a different source");
|
|
264
|
+
check(!input.sourceKind || input.sourceKind === source.type, "SOURCE", "Reader input has a different source type");
|
|
265
|
+
check(!input.format || input.format === source.format, "SOURCE", "Reader input has a different format");
|
|
266
|
+
const result = runProviderCommand(root, environment, {
|
|
267
|
+
configuration: providerConfiguration(root, source.provider),
|
|
268
|
+
command: "discover source prepare",
|
|
269
|
+
input: {
|
|
270
|
+
...input,
|
|
271
|
+
sourceId: source.id,
|
|
272
|
+
sourceKind: source.type,
|
|
273
|
+
...(source.format ? { format: source.format } : {}),
|
|
274
|
+
},
|
|
275
|
+
});
|
|
276
|
+
const artifacts = artifactFiles(result), files = {};
|
|
277
|
+
const out = args.out ?? `discovery/${source.id}/reader`;
|
|
278
|
+
for (const [name, text] of Object.entries(artifacts)) {
|
|
279
|
+
const path = `${out}/${name}`;
|
|
280
|
+
check(!existsSync(fence(root, path)), "OWNER", `Existing reader file ${path}; choose another output directory`);
|
|
281
|
+
files[path] = text;
|
|
282
|
+
}
|
|
283
|
+
return preview(root, files);
|
|
284
|
+
}
|
|
285
|
+
function metadataContracts(root, environment, path, provider) {
|
|
286
|
+
const reader = new Configuration(root), metadata = reader.load(path);
|
|
287
|
+
if (metadata.kind === "DataContract") {
|
|
288
|
+
validateContract(metadata);
|
|
289
|
+
return [metadata];
|
|
290
|
+
}
|
|
291
|
+
if (metadata.columns) {
|
|
292
|
+
const proposal = contractDraft(root, path, "imported", "contracts/imported.odcs.yaml");
|
|
293
|
+
return [parseDocument(proposal.changes[0].after).toJS()];
|
|
294
|
+
}
|
|
295
|
+
const result = runProviderCommand(root, environment, {
|
|
296
|
+
configuration: providerConfiguration(root, provider),
|
|
297
|
+
command: "discover contracts",
|
|
298
|
+
input: metadata,
|
|
299
|
+
});
|
|
300
|
+
const contracts = Object.entries(artifactFiles(result))
|
|
301
|
+
.filter(([n]) => n.endsWith(".odcs.json"))
|
|
302
|
+
.map(([, text]) => JSON.parse(text));
|
|
303
|
+
check(contracts.length > 0, "METADATA", "No contracts found in metadata");
|
|
304
|
+
contracts.forEach((c) => validateContract(c));
|
|
305
|
+
return contracts;
|
|
306
|
+
}
|
|
307
|
+
export function sourceImport(root, environment, args) {
|
|
308
|
+
const source = sourceShow(root, args.id, environment).selected, reader = new Configuration(root);
|
|
309
|
+
const input = reader.load(args.metadata);
|
|
310
|
+
check(!input.sourceId || input.sourceId === source.id, "SOURCE", "Metadata belongs to a different source");
|
|
311
|
+
check(!input.sourceKind || input.sourceKind === source.type, "SOURCE", "Metadata has a different source type");
|
|
312
|
+
check(!input.format || input.format === source.format, "SOURCE", "Metadata has a different format");
|
|
313
|
+
const contracts = metadataContracts(root, environment, args.metadata, source.provider);
|
|
314
|
+
const path = metadataPath(source.id, environment);
|
|
315
|
+
// Replacement is a normal digest-bound authoring proposal, so --dry-run and
|
|
316
|
+
// MCP review/apply protect imported evidence just like configuration edits.
|
|
317
|
+
return preview(root, {
|
|
318
|
+
[path]: JSON.stringify({
|
|
319
|
+
apiVersion: "ingestron.discovery/v1",
|
|
320
|
+
sourceId: source.id,
|
|
321
|
+
environment,
|
|
322
|
+
sourceDigest: digest(reader.text(sourcePath(source.id))),
|
|
323
|
+
inputDigest: digest(reader.text(args.metadata)),
|
|
324
|
+
contracts,
|
|
325
|
+
evidence: "Imported metadata; source access and completeness are not verified by core",
|
|
326
|
+
}, null, 2) + "\n",
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
export function draftContract(root, environment, args) {
|
|
330
|
+
identifier.parse(args.id);
|
|
331
|
+
check(!!args.source !== !!args.metadata, "INPUT", "Choose exactly one of source or metadata");
|
|
332
|
+
let contracts, evidence;
|
|
333
|
+
if (args.source) {
|
|
334
|
+
const source = sourceShow(root, args.source, environment);
|
|
335
|
+
const reader = new Configuration(root), snapshot = reader.load(metadataPath(source.id, environment));
|
|
336
|
+
check(snapshot.apiVersion === "ingestron.discovery/v1" &&
|
|
337
|
+
snapshot.sourceId === source.id &&
|
|
338
|
+
snapshot.environment === environment, "METADATA", "Import discovery metadata for this source and selected environment first");
|
|
339
|
+
check(snapshot.sourceDigest === digest(reader.text(sourcePath(source.id))), "STALE", "Source configuration changed; import fresh metadata before drafting");
|
|
340
|
+
contracts = snapshot.contracts;
|
|
341
|
+
evidence = {
|
|
342
|
+
sourceId: source.id,
|
|
343
|
+
inputDigest: snapshot.inputDigest,
|
|
344
|
+
environment,
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
else {
|
|
348
|
+
contracts = metadataContracts(root, environment, args.metadata, args.provider);
|
|
349
|
+
evidence = {
|
|
350
|
+
inputDigest: digest(new Configuration(root).text(args.metadata)),
|
|
351
|
+
environment,
|
|
352
|
+
};
|
|
353
|
+
}
|
|
354
|
+
const matches = args.entity
|
|
355
|
+
? contracts.filter((c) => c.id === args.entity ||
|
|
356
|
+
c.id.endsWith(`__${args.entity}`) ||
|
|
357
|
+
c.schema?.some((s) => s.name === args.entity || s.physicalName === args.entity))
|
|
358
|
+
: contracts;
|
|
359
|
+
check(matches.length === 1, "SELECT", "Select exactly one discovered entity with entity (contract ID or physical name)");
|
|
360
|
+
const contract = {
|
|
361
|
+
...matches[0],
|
|
362
|
+
id: args.id,
|
|
363
|
+
name: args.id,
|
|
364
|
+
status: "draft",
|
|
365
|
+
};
|
|
366
|
+
validateContract(contract);
|
|
367
|
+
const path = args.out ?? `contracts/${args.id}.odcs.yaml`;
|
|
368
|
+
const provenance = `contracts/${args.id}.provenance.json`;
|
|
369
|
+
for (const file of [path, provenance])
|
|
370
|
+
check(!existsSync(fence(root, file)), "OWNER", `Contract output ${file} already exists; use a new ID or review the existing file`);
|
|
371
|
+
return preview(root, {
|
|
372
|
+
[path]: stringify(contract),
|
|
373
|
+
[provenance]: JSON.stringify({
|
|
374
|
+
...evidence,
|
|
375
|
+
contract: path,
|
|
376
|
+
status: "draft",
|
|
377
|
+
review: "Review schema, keys, types and business requirements before building",
|
|
378
|
+
}, null, 2) + "\n",
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
export function contractPath(root, id) {
|
|
382
|
+
if (/\.(json|ya?ml)$/.test(id)) {
|
|
383
|
+
fence(root, id);
|
|
384
|
+
return id;
|
|
385
|
+
}
|
|
386
|
+
return `contracts/${identifier.parse(id)}.odcs.yaml`;
|
|
387
|
+
}
|
|
388
|
+
export function contractList(root) {
|
|
389
|
+
const dir = fence(root, "contracts");
|
|
390
|
+
return existsSync(dir)
|
|
391
|
+
? readdirSync(dir)
|
|
392
|
+
.filter((n) => /\.odcs\.(json|ya?ml)$/.test(n))
|
|
393
|
+
.sort()
|
|
394
|
+
.map((n) => {
|
|
395
|
+
const path = `contracts/${n}`, contract = new Configuration(root).load(path);
|
|
396
|
+
validateContract(contract, path);
|
|
397
|
+
return {
|
|
398
|
+
id: contract.id,
|
|
399
|
+
status: contract.status,
|
|
400
|
+
path,
|
|
401
|
+
entities: contract.schema?.map((s) => s.physicalName ?? s.name) ?? [],
|
|
402
|
+
};
|
|
403
|
+
})
|
|
404
|
+
: [];
|
|
405
|
+
}
|
|
406
|
+
export function flowCreate(root, environment, args) {
|
|
407
|
+
const { contract: selectedContract, source: selectedSource, ...options } = args;
|
|
408
|
+
let sourceId = selectedSource;
|
|
409
|
+
if (!sourceId &&
|
|
410
|
+
selectedContract &&
|
|
411
|
+
/^[A-Za-z_][\w-]*$/.test(selectedContract)) {
|
|
412
|
+
const provenance = fence(root, `contracts/${selectedContract}.provenance.json`);
|
|
413
|
+
if (existsSync(provenance))
|
|
414
|
+
sourceId = JSON.parse(readFileSync(provenance, "utf8")).sourceId;
|
|
415
|
+
}
|
|
416
|
+
if (sourceId) {
|
|
417
|
+
const source = sourceShow(root, sourceId, environment).selected;
|
|
418
|
+
options.sourceKind ??= source.type;
|
|
419
|
+
options.format ??= source.format;
|
|
420
|
+
options.sourceBinding ??= source.binding;
|
|
421
|
+
options.provider ??= source.provider;
|
|
422
|
+
}
|
|
423
|
+
if (options.provider)
|
|
424
|
+
options.provider = providerConfiguration(root, options.provider);
|
|
425
|
+
const proposal = addFlow(root, options);
|
|
426
|
+
if (!selectedContract)
|
|
427
|
+
return proposal;
|
|
428
|
+
check(options.kind === "ingestion", "INPUT", "Use flow output add to bind a transformation's producing port and output contract");
|
|
429
|
+
const path = contractPath(root, selectedContract), contract = new Configuration(root).load(path);
|
|
430
|
+
contractColumns(contract, path);
|
|
431
|
+
const tableId = identifier.parse(contract.id);
|
|
432
|
+
const replacements = Object.fromEntries(proposal.changes.map((c) => [c.path, c.after]));
|
|
433
|
+
const file = `flows/${options.id}/flow.yaml`, doc = parseDocument(replacements[file]);
|
|
434
|
+
const relativeContract = relative(dirname(resolve(root, file)), resolve(root, path)).replaceAll("\\", "/");
|
|
435
|
+
doc.setIn(["tables", tableId], { contract: { $resolve: relativeContract } });
|
|
436
|
+
replacements[file] = doc.toString();
|
|
437
|
+
return preview(root, replacements);
|
|
438
|
+
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
export declare function providerReference(root: string, reference: string): {
|
|
2
|
+
file: string;
|
|
3
|
+
root: string;
|
|
4
|
+
entry: {
|
|
5
|
+
repository: string;
|
|
6
|
+
path: string;
|
|
7
|
+
commit: string;
|
|
8
|
+
files: Record<string, string>;
|
|
9
|
+
};
|
|
10
|
+
manifest: {
|
|
11
|
+
apiVersion: "ingestron.provider/v1";
|
|
12
|
+
id: string;
|
|
13
|
+
version: string;
|
|
14
|
+
platform: string;
|
|
15
|
+
activities: Record<string, string>;
|
|
16
|
+
capabilities?: {
|
|
17
|
+
artifactKinds: string[];
|
|
18
|
+
delivery: boolean;
|
|
19
|
+
handovers: ("files" | "relation")[];
|
|
20
|
+
packContracts: Record<string, {
|
|
21
|
+
version: string;
|
|
22
|
+
activities: string[];
|
|
23
|
+
}>;
|
|
24
|
+
} | undefined;
|
|
25
|
+
projectAssembly?: {
|
|
26
|
+
apiVersion: "ingestron.project-assembly/v1";
|
|
27
|
+
command: string;
|
|
28
|
+
} | undefined;
|
|
29
|
+
connectorRuntimes?: Record<string, {
|
|
30
|
+
selectionSchema: Record<string, unknown>;
|
|
31
|
+
executionSchema: Record<string, unknown>;
|
|
32
|
+
prepareCommand: string;
|
|
33
|
+
contractsCommand?: string | undefined;
|
|
34
|
+
executionPlatforms: string[];
|
|
35
|
+
}> | undefined;
|
|
36
|
+
configurationSchema?: string | Record<string, unknown> | undefined;
|
|
37
|
+
standards?: {
|
|
38
|
+
id: string;
|
|
39
|
+
description: string;
|
|
40
|
+
sourceKinds?: string[] | undefined;
|
|
41
|
+
dataFlow?: "forbidden" | "required" | undefined;
|
|
42
|
+
}[] | undefined;
|
|
43
|
+
compatibility?: {
|
|
44
|
+
plan: "ingestron.plan/v1";
|
|
45
|
+
requiredFeatures: string[];
|
|
46
|
+
minimumCli: string;
|
|
47
|
+
} | undefined;
|
|
48
|
+
outputSchemas?: Record<string, string> | undefined;
|
|
49
|
+
lifecycle?: {
|
|
50
|
+
apiVersion: "ingestron.provider-lifecycle/v1";
|
|
51
|
+
module: string;
|
|
52
|
+
} | undefined;
|
|
53
|
+
planner?: {
|
|
54
|
+
apiVersion: "ingestron.provider-planning/v1";
|
|
55
|
+
module: string;
|
|
56
|
+
} | undefined;
|
|
57
|
+
commands?: {
|
|
58
|
+
apiVersion: "ingestron.provider-commands/v1";
|
|
59
|
+
namespace?: string | undefined;
|
|
60
|
+
execution: "offline-json";
|
|
61
|
+
module: string;
|
|
62
|
+
definitions: {
|
|
63
|
+
name: string;
|
|
64
|
+
description: string;
|
|
65
|
+
reportPack?: boolean | undefined;
|
|
66
|
+
examples?: string[] | undefined;
|
|
67
|
+
resultSchema?: Record<string, unknown> | undefined;
|
|
68
|
+
inputSchema: Record<string, unknown>;
|
|
69
|
+
}[];
|
|
70
|
+
} | undefined;
|
|
71
|
+
execution?: {
|
|
72
|
+
apiVersion: "ingestron.execution/v1";
|
|
73
|
+
transport: string;
|
|
74
|
+
entryPoint: string;
|
|
75
|
+
actions: ("approve" | "discover" | "prepare" | "review" | "run")[];
|
|
76
|
+
status: "receipt";
|
|
77
|
+
retry: "same-run-id";
|
|
78
|
+
cancellation: "interrupt";
|
|
79
|
+
} | undefined;
|
|
80
|
+
renderer?: {
|
|
81
|
+
apiVersion: "ingestron.provider-generation/v1";
|
|
82
|
+
module: string;
|
|
83
|
+
} | undefined;
|
|
84
|
+
};
|
|
85
|
+
source: string;
|
|
86
|
+
version: string;
|
|
87
|
+
};
|
|
88
|
+
export declare function providerAuthor(root: string, reference: string, input: Record<string, unknown>, allowed: (name: string) => boolean): Record<string, string>;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { friendlyReference } from "../core/package-references.js";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { resolve, dirname, relative } from "node:path";
|
|
4
|
+
import { canonicalPackageReference, resolvePackage, providerPackageSchema, packageYaml, fence, } from "../core/packages.js";
|
|
5
|
+
import { check, isMap } from "../core/errors.js";
|
|
6
|
+
import { executeProvider } from "./provider-renderer.js";
|
|
7
|
+
import { checkProviderCompatibility } from "./compatibility.js";
|
|
8
|
+
export function providerReference(root, reference) {
|
|
9
|
+
const canonical = canonicalPackageReference(reference), at = canonical.lastIndexOf("@");
|
|
10
|
+
const pkg = resolvePackage(root, canonical), manifest = providerPackageSchema.parse(packageYaml(pkg.file));
|
|
11
|
+
checkProviderCompatibility(manifest);
|
|
12
|
+
return {
|
|
13
|
+
...pkg,
|
|
14
|
+
manifest,
|
|
15
|
+
source: friendlyReference(canonical.slice(0, at)),
|
|
16
|
+
version: canonical.slice(at + 1),
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export function providerAuthor(root, reference, input, allowed) {
|
|
20
|
+
const selected = providerReference(root, reference);
|
|
21
|
+
check(selected.manifest.lifecycle, "PROVIDER", "This provider does not declare lifecycle authoring; select a migrated version");
|
|
22
|
+
const file = fence(selected.root, relative(selected.root, resolve(dirname(selected.file), selected.manifest.lifecycle.module)));
|
|
23
|
+
const result = executeProvider(readFileSync(file, "utf8"), {
|
|
24
|
+
apiVersion: "ingestron.provider-authoring/v1",
|
|
25
|
+
...input,
|
|
26
|
+
provider: { source: selected.source, version: selected.version },
|
|
27
|
+
}, false, "author");
|
|
28
|
+
check(isMap(result.files) && Object.keys(result).length === 1, "PROVIDER", "Invalid authoring response");
|
|
29
|
+
for (const [name, value] of Object.entries(result.files)) {
|
|
30
|
+
fence(root, name);
|
|
31
|
+
check(allowed(name) && typeof value === "string" && value.length <= 1000000, "PROVIDER", `Invalid authored file ${name}`);
|
|
32
|
+
}
|
|
33
|
+
return result.files;
|
|
34
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/** All provider entry points use one legacy ABI check. App semver resets must
|
|
2
|
+
* not make authoring work while planning or command dispatch rejects the package. */
|
|
3
|
+
export declare const hostFeatures: string[];
|
|
4
|
+
export declare function checkPluginCompatibility(minimum?: string, requiredFeatures?: string[]): void;
|
|
5
|
+
/** Capability opt-ins must also declare the host protocol features they need. */
|
|
6
|
+
export declare function checkProviderCompatibility(manifest: {
|
|
7
|
+
compatibility?: {
|
|
8
|
+
minimumCli: string;
|
|
9
|
+
requiredFeatures?: string[];
|
|
10
|
+
};
|
|
11
|
+
projectAssembly?: unknown;
|
|
12
|
+
execution?: unknown;
|
|
13
|
+
connectorRuntimes?: Record<string, unknown>;
|
|
14
|
+
capabilities?: {
|
|
15
|
+
artifactKinds: string[];
|
|
16
|
+
delivery: boolean;
|
|
17
|
+
handovers: string[];
|
|
18
|
+
packContracts: Record<string, {
|
|
19
|
+
version: string;
|
|
20
|
+
activities: string[];
|
|
21
|
+
}>;
|
|
22
|
+
};
|
|
23
|
+
}): void;
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { pluginCompatibilityVersion } from "../version.js";
|
|
2
|
+
import { check } from "../core/errors.js";
|
|
3
|
+
/** All provider entry points use one legacy ABI check. App semver resets must
|
|
4
|
+
* not make authoring work while planning or command dispatch rejects the package. */
|
|
5
|
+
export const hostFeatures = [
|
|
6
|
+
"project-assembly",
|
|
7
|
+
"provider-execution",
|
|
8
|
+
"connector-runtime-capabilities",
|
|
9
|
+
"connector-runtime-assets",
|
|
10
|
+
"project-connections",
|
|
11
|
+
"odcs-connections",
|
|
12
|
+
"report-packs",
|
|
13
|
+
"native-project",
|
|
14
|
+
"extension-packs",
|
|
15
|
+
"delivery-exports",
|
|
16
|
+
"relation-handovers",
|
|
17
|
+
"export-groups",
|
|
18
|
+
"model-inputs",
|
|
19
|
+
"ingestion-presets",
|
|
20
|
+
];
|
|
21
|
+
export function checkPluginCompatibility(minimum, requiredFeatures = []) {
|
|
22
|
+
for (const feature of requiredFeatures)
|
|
23
|
+
check(hostFeatures.includes(feature), "COMPATIBILITY", `Unsupported provider host feature ${feature}`);
|
|
24
|
+
if (!minimum)
|
|
25
|
+
return;
|
|
26
|
+
const current = pluginCompatibilityVersion.split(".").map(Number);
|
|
27
|
+
const required = minimum.split(".").map(Number);
|
|
28
|
+
check((current
|
|
29
|
+
.map((value, index) => value - required[index])
|
|
30
|
+
.find((value) => value !== 0) ?? 0) >= 0, "COMPATIBILITY", `Provider requires plugin compatibility ${minimum}; this host supports ${pluginCompatibilityVersion}`);
|
|
31
|
+
}
|
|
32
|
+
/** Capability opt-ins must also declare the host protocol features they need. */
|
|
33
|
+
export function checkProviderCompatibility(manifest) {
|
|
34
|
+
const required = manifest.compatibility?.requiredFeatures ?? [];
|
|
35
|
+
check(!manifest.execution || required.includes("provider-execution"), "COMPATIBILITY", "Execution providers must declare requiredFeatures provider-execution");
|
|
36
|
+
checkPluginCompatibility(manifest.compatibility?.minimumCli, required);
|
|
37
|
+
if (manifest.connectorRuntimes &&
|
|
38
|
+
Object.keys(manifest.connectorRuntimes).length)
|
|
39
|
+
check(required.includes("connector-runtime-capabilities"), "COMPATIBILITY", "Provider must declare requiredFeatures: connector-runtime-capabilities");
|
|
40
|
+
if (manifest.projectAssembly)
|
|
41
|
+
check(required.includes("project-assembly"), "COMPATIBILITY", "Provider must declare requiredFeatures: project-assembly");
|
|
42
|
+
const c = manifest.capabilities;
|
|
43
|
+
if (!c)
|
|
44
|
+
return;
|
|
45
|
+
const needed = [
|
|
46
|
+
...(c.artifactKinds.some((k) => !["native-notebook", "native-pipeline"].includes(k))
|
|
47
|
+
? ["native-project"]
|
|
48
|
+
: []),
|
|
49
|
+
...(c.delivery ? ["delivery-exports"] : []),
|
|
50
|
+
...(c.handovers.includes("relation") ? ["relation-handovers"] : []),
|
|
51
|
+
...(Object.keys(c.packContracts).length ? ["extension-packs"] : []),
|
|
52
|
+
];
|
|
53
|
+
for (const feature of needed)
|
|
54
|
+
check(required.includes(feature), "COMPATIBILITY", `Provider must declare requiredFeatures: ${feature}`);
|
|
55
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { Plan } from "../core/schema.js";
|
|
2
|
+
export declare function validateProviderPlan(plan: Plan): void;
|
|
3
|
+
export declare function generateProvider(root: string, plan: Plan): Record<string, string>;
|
|
4
|
+
/** The same provider schema gate applies after native rendering and project assembly. */
|
|
5
|
+
export declare function validateProviderFiles(code: string, schemas: Record<string, string>, files: Record<string, string>): void;
|