@ingestron/core 0.12.0

package/dist/core/delivery-exports.js

@@ -0,0 +1,119 @@
+import { canonical, check, digest } from "./errors.js";
+/** Partition compilation only. This never schedules jobs or asserts data exists. */
+export function deliveryExports(plan) {
+    const groups = new Map();
+    const owner = new Map();
+    for (const node of plan.nodes) {
+        check(plan.selection.projectBuild || node.runtime?.capabilities?.delivery, "CAPABILITY", `Provider configuration ${node.runtime?.configuration} must opt in to delivery exports`);
+        check(node.runtime?.configuration, "PROVIDER", "Native node needs a provider configuration");
+        const id = plan.selection.projectBuild
+            ? node.runtime.configuration
+            : `${node.exportGroup ?? node.flow}/${node.runtime.configuration}`;
+        if (!groups.has(id))
+            groups.set(id, []);
+        groups.get(id).push(node);
+        owner.set(node.id, id);
+    }
+    const claims = new Map();
+    for (const [group, nodes] of groups)
+        for (const node of nodes)
+            for (const resource of node.resources ?? []) {
+                const identity = canonical([
+                    node.platform,
+                    resource.scope,
+                    resource.kind,
+                    resource.name,
+                ]);
+                check(!claims.has(identity) || claims.get(identity) === group, "OWNER", `Native ${resource.kind} ${resource.name} is claimed by exports ${claims.get(identity)} and ${group}; use one export group`);
+                claims.set(identity, group);
+            }
+    const exports = [...groups].map(([id, nodes]) => {
+        const ids = new Set(nodes.map((n) => n.id));
+        const imports = [];
+        for (const node of nodes)
+            for (const dependency of node.needs) {
+                if (ids.has(dependency))
+                    continue;
+                const flow = plan.flows.find((f) => f.id === node.flow);
+                const requirements = Object.values(flow.requires).filter((r) => plan.datasets[r.dataset]?.producer === dependency);
+                check(requirements.length, "HANDOVER", `Export ${id} requires an explicit handover for ${dependency}`);
+                for (const required of requirements) {
+                    check(required.handover, "HANDOVER", `Export ${id} requires an explicit handover for ${dependency}`);
+                    const dataset = plan.datasets[required.dataset];
+                    if (!imports.some((i) => i.dataset === required.dataset))
+                        imports.push({
+                            dataset: required.dataset,
+                            producer: owner.get(dependency),
+                            kind: required.handover,
+                            location: dataset.location,
+                            contract: dataset.contract,
+                        });
+                }
+            }
+        imports.sort((a, b) => a.dataset < b.dataset ? -1 : a.dataset > b.dataset ? 1 : 0);
+        const refs = new Set(nodes
+            .flatMap((n) => [
+            n.runtime?.rendererRef,
+            n.runtime?.lifecycleRef,
+            n.runtime?.outputSchemasRef,
+            n.generation?.runtimeRef,
+            n.generation?.codeRef,
+        ])
+            .filter(Boolean));
+        const bindings = new Set(nodes.flatMap((n) => [n.binding, n.source?.binding]).filter(Boolean));
+        for (const input of imports)
+            bindings.add(input.location.binding);
+        for (const dataset of Object.values(plan.datasets))
+            if (ids.has(dataset.producer) &&
+                typeof dataset.location?.binding === "string")
+                bindings.add(dataset.location.binding);
+        const { digest: _digest, ...body } = plan;
+        const partBody = {
+            ...body,
+            selection: {},
+            nodes: nodes.map((n) => ({
+                ...n,
+                needs: n.needs.filter((d) => ids.has(d)),
+            })),
+            flows: plan.flows.filter((f) => nodes.some((n) => n.flow === f.id)),
+            datasets: Object.fromEntries(Object.entries(plan.datasets).filter(([key, d]) => ids.has(d.producer) || imports.some((i) => i.dataset === key))),
+            bindings: Object.fromEntries(Object.entries(plan.bindings).filter(([key]) => bindings.has(key))),
+            activitySources: Object.fromEntries(Object.entries(plan.activitySources).filter(([key]) => refs.has(key))),
+            names: {
+                nodes: Object.fromEntries(Object.entries(plan.names.nodes).filter(([key]) => ids.has(key))),
+                jobs: Object.fromEntries(Object.entries(plan.names.jobs).filter(([key]) => nodes.some((n) => n.flow === key))),
+            },
+            recovery: plan.recovery.filter((r) => nodes.some((n) => n.flow === r.flow)),
+            delivery: { exportId: id, imports },
+        };
+        const part = { ...partBody, digest: digest(canonical(partBody)) };
+        return {
+            id,
+            directory: `exports/${id}`,
+            needs: [...new Set(imports.map((i) => i.producer))].sort(),
+            imports,
+            resources: [
+                ...new Map(nodes.flatMap((n) => n.resources ?? []).map((r) => [canonical(r), r])).values(),
+            ],
+            plan: part,
+        };
+    });
+    const ordered = [], active = new Set(), done = new Set();
+    const visit = (id) => {
+        check(!active.has(id), "CYCLE", `Delivery export dependency cycle at ${id}`);
+        if (done.has(id))
+            return;
+        const entry = exports.find((e) => e.id === id);
+        check(entry, "HANDOVER", `Missing delivery export ${id}`);
+        active.add(id);
+        entry.needs.forEach(visit);
+        active.delete(id);
+        done.add(id);
+        ordered.push(entry);
+    };
+    exports
+        .map((e) => e.id)
+        .sort()
+        .forEach(visit);
+    return ordered;
+}

package/dist/core/development.d.ts

@@ -0,0 +1,50 @@
+export declare function activityInspect(root: string, path: string): {
+    manifest: {
+        apiVersion: "ingestron.activity/v1";
+        id: string;
+        version: string;
+        description: string;
+        platform: string;
+        input: "committed" | "files" | "relation" | "source";
+        output: "committed" | "files" | "relation";
+        effect: "commit" | "control" | "read" | "stage";
+        capabilities: string[];
+        optionsSchema: Record<string, unknown>;
+        generator?: {
+            kind: "native-notebook" | "native-pipeline" | "native-project";
+            artifactKind?: string | undefined;
+            execution?: string | undefined;
+            persistence?: "capture" | "durable" | "ephemeral" | undefined;
+            runtime?: string | undefined;
+            code?: string | undefined;
+            scope: "collection" | "table";
+            modes: ("collection" | "table")[];
+            groupBy: string[];
+            dependencyOptions: Record<string, string>;
+            entrypoint: string;
+        } | undefined;
+        requirements?: {
+            execution: string;
+            features: string[];
+            sourceKinds?: string[] | undefined;
+            evidence: "offline";
+        } | undefined;
+        templates: {
+            source: string;
+            output: string;
+            format: "json" | "python" | "sql" | "text" | "yaml";
+        }[];
+        dependencies: Record<string, string>;
+    };
+    generationSource: string | undefined;
+    templates: {
+        source: string;
+        output: string;
+        format: "json" | "python" | "sql" | "text" | "yaml";
+        content: string;
+    }[];
+    sourceDigests: Record<string, string>;
+    evidence: string;
+};
+export declare function contractDraft(root: string, path: string, id: string, out: string): import("./authoring.js").Proposal;
+export declare function exportProviderArtifacts(root: string, path: string, directory: string): import("./authoring.js").Proposal;

package/dist/core/development.js

@@ -0,0 +1,94 @@
+import { Configuration } from "./config.js";
+import { externalActivitySchema, packageYaml } from "./packages.js";
+import { contractColumns, validateContract } from "./contracts.js";
+import { preview } from "./authoring.js";
+import { check, isMap } from "./errors.js";
+import { existsSync } from "node:fs";
+import { resolve } from "node:path";
+import { fence } from "./packages.js";
+import { stringify } from "yaml";
+export function activityInspect(root, path) {
+    const reader = new Configuration(root), file = reader.path(path);
+    const manifest = externalActivitySchema.parse(packageYaml(file));
+    reader.text(file);
+    const generationSource = manifest.generator?.code
+        ? reader.text(manifest.generator.code, file)
+        : undefined;
+    const templates = manifest.templates.map((t) => ({
+        ...t,
+        content: reader.text(t.source, file),
+    }));
+    return {
+        manifest,
+        generationSource,
+        templates,
+        sourceDigests: reader.files,
+        evidence: "Manifest and source inventory only; no activity code executed. Composition is checked by the owning provider.",
+    };
+}
+export function contractDraft(root, path, id, out) {
+    const reader = new Configuration(root), metadata = reader.load(path);
+    check(Array.isArray(metadata.columns) && metadata.columns.length, "METADATA", "Imported metadata needs a non-empty columns array");
+    const types = {
+        string: "string",
+        integer: "integer",
+        number: "number",
+        boolean: "boolean",
+        date: "date",
+        timestamp: "timestamp",
+    };
+    const contract = {
+        apiVersion: "v3.1.0",
+        kind: "DataContract",
+        id,
+        name: id,
+        version: "1.0.0",
+        status: "draft",
+        schema: [
+            {
+                name: id,
+                physicalName: metadata.table ?? id,
+                logicalType: "object",
+                physicalType: "table",
+                properties: metadata.columns.map((c) => {
+                    check(types[c.logicalType], "METADATA", `Review logicalType for ${c.name}`);
+                    return {
+                        name: c.name,
+                        logicalType: c.logicalType,
+                        ...(c.physicalType ? { physicalType: c.physicalType } : {}),
+                        required: c.required ?? false,
+                        ...(c.primaryKey ? { primaryKey: true } : {}),
+                    };
+                }),
+            },
+        ],
+    };
+    contractColumns(contract);
+    return preview(root, { [out]: stringify(contract) });
+}
+export function exportProviderArtifacts(root, path, directory) {
+    const reader = new Configuration(root), envelope = JSON.parse(reader.text(path));
+    check(envelope.apiVersion === "ingestron.provider-command-result/v1" &&
+        envelope.execution === "offline-json", "PROVIDER", "Expected a saved offline provider command result");
+    const proposal = envelope.result;
+    check(proposal?.apiVersion === "ingestron.artifact-proposal/v1" &&
+        isMap(proposal.artifacts), "PROVIDER", "Provider did not return exportable artifacts");
+    const entries = Object.entries(proposal.artifacts);
+    check(entries.length > 0 && entries.length <= 200, "LIMIT", "Export requires 1–200 artifacts");
+    const files = {};
+    for (const [name, value] of entries) {
+        check(/^(?:[A-Za-z0-9_-][A-Za-z0-9_.-]*\/)*[A-Za-z0-9_-][A-Za-z0-9_.-]*\.(json|yaml|yml|sql|py|mjs|txt|md|pbip|pbir|pbism|bim|svg)$/.test(name) &&
+            typeof value === "string" &&
+            value.length <= 2000000, "PROVIDER", "Invalid artifact name or content");
+        const target = directory + "/" + name;
+        fence(root, target);
+        check(!existsSync(resolve(root, target)), "OWNER", `Existing artifact ${target}; choose a new export directory`);
+        if (/\.(json|pbip|pbir|pbism|bim)$/.test(name)) {
+            const doc = JSON.parse(value);
+            if (name.endsWith(".odcs.json"))
+                validateContract(doc);
+        }
+        files[target] = value;
+    }
+    return preview(root, files);
+}

package/dist/core/ecosystem-authoring.d.ts

@@ -0,0 +1,13 @@
+export declare function configurePack(root: string, args: {
+    reference: string;
+    name: string;
+    provider?: string;
+}): import("./authoring.js").Proposal;
+export declare function configureFlow(root: string, args: {
+    id: string;
+    group?: string;
+    input?: string;
+    dataset?: string;
+    handover?: "files" | "relation";
+    table?: string;
+}): import("./authoring.js").Proposal;

package/dist/core/ecosystem-authoring.js

@@ -0,0 +1,82 @@
+import { validateModelPack } from "./model-pack-schema.js";
+import { parseDocument } from "yaml";
+import { Configuration } from "./config.js";
+import { preview } from "./authoring.js";
+import { check } from "./errors.js";
+import { projectSchema } from "./schema.js";
+import { extensionPackSchema, packageYaml, resolvePackage, providerPackageSchema, } from "./packages.js";
+import { extensionActivities } from "./extension-packs.js";
+import { checkProviderCompatibility } from "../plugins/compatibility.js";
+export function configurePack(root, args) {
+    const reader = new Configuration(root), doc = parseDocument(reader.text("project.yaml"));
+    const source = resolvePackage(root, args.reference);
+    const manifest = packageYaml(source.file);
+    if (manifest.apiVersion === "ingestron.extension-pack/v2") {
+        validateModelPack(manifest);
+        check(!args.provider, "MODEL_PACK", "Model packs are independent of provider configurations; omit provider");
+        const project = projectSchema.parse(reader.load());
+        check(!project.modelPacks[args.name], "MODEL_PACK", "Model pack alias already exists");
+        const split = args.reference.lastIndexOf("@");
+        check(split > 0, "MODEL_PACK", "Select an exact installed model pack");
+        doc.setIn(["modelPacks", args.name], {
+            source: args.reference.slice(0, split),
+            version: args.reference.slice(split + 1).replace(/^v(?=\d)/, ""),
+        });
+        return preview(root, { "project.yaml": doc.toString() });
+    }
+    check(manifest.apiVersion !== "ingestron.extension-pack/v3", "REPORT_PACK", "Report packs are selected by exact reference in report build input, not attached as activities");
+    extensionPackSchema.parse(manifest);
+    check(args.provider, "PACK", "Activity preset packs require a provider configuration");
+    const project = projectSchema.parse(reader.load());
+    const configuration = project.providers.configurations[args.provider];
+    check(configuration, "PROVIDER", "Unknown provider configuration");
+    check(!project.providers.packs[args.name], "PACK", "Pack name already exists");
+    const pkg = project.providers.packages[configuration.package];
+    const file = pkg.source.startsWith(".")
+        ? reader.path(pkg.source)
+        : resolvePackage(root, `${pkg.source}@${pkg.version}`).file;
+    const provider = providerPackageSchema.parse(packageYaml(file));
+    checkProviderCompatibility(provider);
+    const split = args.reference.lastIndexOf("@");
+    check(split > 0, "PACK", "Use an exact installed pack reference");
+    const selection = {
+        source: args.reference.slice(0, split),
+        version: args.reference.slice(split + 1).replace(/^v(?=\d)/, ""),
+    };
+    project.providers.packs[args.name] = selection;
+    extensionActivities(reader, project, [args.name], provider);
+    doc.setIn(["providers", "packs", args.name], selection);
+    doc.setIn(["providers", "configurations", args.provider, "packs"], [...configuration.packs, args.name]);
+    return preview(root, { "project.yaml": doc.toString() });
+}
+export function configureFlow(root, args) {
+    const reader = new Configuration(root), project = parseDocument(reader.text("project.yaml"));
+    const entries = project.toJS().flows ?? [];
+    for (let i = 0; i < entries.length; i++) {
+        const entry = entries[i], file = entry.$resolve ? reader.path(entry.$resolve) : "project.yaml";
+        const doc = entry.$resolve ? parseDocument(reader.text(file)) : project;
+        const prefix = entry.$resolve ? [] : ["flows", i];
+        if (doc.getIn([...prefix, "id"]) !== args.id)
+            continue;
+        if (args.group)
+            doc.setIn([...prefix, "export"], args.group);
+        else {
+            check(args.input && args.dataset && args.handover, "HANDOVER", "Supply input, dataset and handover");
+            check(!doc.hasIn([...prefix, "requires", args.input]), "HANDOVER", "Input already exists");
+            doc.setIn([...prefix, "requires", args.input], {
+                dataset: args.dataset,
+                select: { mode: "same-run" },
+                handover: args.handover,
+            });
+            if (args.table) {
+                check(args.handover === "files" &&
+                    doc.hasIn([...prefix, "tables", args.table]), "HANDOVER", "Select an existing ingestion table for a file input");
+                doc.setIn([...prefix, "tables", args.table, "source"], {
+                    from: `requires.${args.input}`,
+                });
+            }
+        }
+        return preview(root, { [file]: doc.toString() });
+    }
+    throw new Error(`Unknown flow ${args.id}`);
+}

package/dist/core/errors.d.ts

@@ -0,0 +1,12 @@
+export declare class Problem extends Error {
+    code: string;
+    file?: string | undefined;
+    pointer?: string | undefined;
+    hint?: string | undefined;
+    constructor(code: string, message: string, file?: string | undefined, pointer?: string | undefined, hint?: string | undefined);
+}
+export declare function check(condition: unknown, code: string, message: string, file?: string): asserts condition;
+export declare const digest: (value: string | Buffer) => string;
+export declare const canonical: (value: any) => string;
+export declare const isMap: (v: unknown) => v is Record<string, any>;
+export declare function merge(a: any, b: any): any;

package/dist/core/errors.js

@@ -0,0 +1,43 @@
+import { createHash } from "node:crypto";
+export class Problem extends Error {
+    code;
+    file;
+    pointer;
+    hint;
+    constructor(code, message, file, pointer, hint) {
+        super(message);
+        this.code = code;
+        this.file = file;
+        this.pointer = pointer;
+        this.hint = hint;
+        this.name = "Problem";
+    }
+}
+export function check(condition, code, message, file) {
+    if (!condition)
+        throw new Problem(code, message, file);
+}
+export const digest = (value) => createHash("sha256").update(value).digest("hex");
+export const canonical = (value) => JSON.stringify(sort(value));
+function sort(value) {
+    return Array.isArray(value)
+        ? value.map(sort)
+        : value && typeof value === "object"
+            ? Object.fromEntries(Object.keys(value)
+                .sort()
+                .map((k) => [k, sort(value[k])]))
+            : value;
+}
+export const isMap = (v) => !!v && typeof v === "object" && !Array.isArray(v);
+export function merge(a, b) {
+    if (!isMap(a) || !isMap(b))
+        return structuredClone(b);
+    const result = structuredClone(a);
+    for (const [key, value] of Object.entries(b)) {
+        check(!["__proto__", "prototype", "constructor"].includes(key), "KEY", "Unsafe mapping key");
+        result[key] = Object.hasOwn(result, key)
+            ? merge(result[key], value)
+            : structuredClone(value);
+    }
+    return result;
+}

package/dist/core/execution.d.ts

@@ -0,0 +1,28 @@
+import { z } from "zod";
+import type { Context } from "./operations.js";
+export declare const executionSchemas: {
+    runtime_prepare: z.ZodObject<{
+        from: z.ZodOptional<z.ZodString>;
+        provider: z.ZodOptional<z.ZodString>;
+        flow: z.ZodOptional<z.ZodString>;
+        python: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    run: z.ZodObject<{
+        from: z.ZodOptional<z.ZodString>;
+        provider: z.ZodOptional<z.ZodString>;
+        flow: z.ZodOptional<z.ZodString>;
+        action: z.ZodDefault<z.ZodEnum<{
+            approve: "approve";
+            discover: "discover";
+            review: "review";
+            run: "run";
+        }>>;
+        runId: z.ZodOptional<z.ZodString>;
+        retry: z.ZodOptional<z.ZodString>;
+        envFile: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>;
+    run_status: z.ZodObject<{
+        id: z.ZodString;
+    }, z.core.$strict>;
+};
+export declare function performExecution(context: Context, operation: keyof typeof executionSchemas, args: any): Promise<any>;