@ingestron/core 0.12.0

package/dist/plugins/generation.js

@@ -0,0 +1,105 @@
+import { deliveryExports } from "../core/delivery-exports.js";
+import { Ajv } from "ajv";
+import { executeProvider } from "./provider-renderer.js";
+import { contractColumns } from "../core/contracts.js";
+import { check } from "../core/errors.js";
+import { Configuration } from "../core/config.js";
+import { stringify } from "yaml";
+function guestPlan(plan) {
+    const needed = new Set(plan.nodes
+        .flatMap((n) => [n.generation?.codeRef, n.generation?.runtimeRef])
+        .filter(Boolean));
+    return {
+        ...plan,
+        activitySources: Object.fromEntries(Object.entries(plan.activitySources).filter(([ref]) => needed.has(ref))),
+    };
+}
+export function validateProviderPlan(plan) {
+    if (plan.selection.delivery) {
+        for (const entry of deliveryExports(plan))
+            validateProviderPlan(entry.plan);
+        return;
+    }
+    const renderer = plan.nodes[0]?.runtime?.rendererRef;
+    check(renderer && plan.nodes.every((n) => n.runtime?.rendererRef === renderer), "PROVIDER", "An export must select one provider renderer");
+    executeProvider(plan.activitySources[renderer], {
+        ...guestPlan(plan),
+        nodes: plan.nodes.map((n) => ({
+            ...n,
+            columns: n.contract ? contractColumns(n.contract) : [],
+            sql: "",
+        })),
+    }, true);
+}
+export function generateProvider(root, plan) {
+    if (plan.selection.delivery) {
+        const files = {};
+        let count = 0, bytes = 0;
+        const add = (name, content) => {
+            count++;
+            bytes += Buffer.byteLength(content);
+            check(count <= 10000 && bytes <= 10 * 1024 * 1024, "LIMIT", "Delivery exceeds 10000 files or 10 MiB");
+            files[name] = content;
+        };
+        const exports = deliveryExports(plan);
+        for (const entry of exports)
+            for (const [name, content] of Object.entries(generateProvider(root, entry.plan)))
+                add(`${entry.directory}/${name}`, content);
+        add("ingestron-delivery.json", JSON.stringify({
+            apiVersion: "ingestron.delivery/v1",
+            project: plan.project,
+            environment: plan.environment,
+            planDigest: plan.digest,
+            evidence: "offline",
+            execution: "not-run",
+            exports: exports.map(({ plan: part, ...entry }) => ({
+                ...entry,
+                planDigest: part.digest,
+                nodes: part.nodes.map((n) => n.id),
+            })),
+        }, null, 2) + "\n");
+        return files;
+    }
+    validateProviderPlan(plan);
+    const reader = new Configuration(root);
+    const request = {
+        ...guestPlan(plan),
+        nodes: plan.nodes.map((n) => ({
+            ...n,
+            columns: n.contract ? contractColumns(n.contract) : [],
+            sql: n.with.sqlFile ? reader.text(n.with.sqlFile) : undefined,
+            sqlFile: n.with.sqlFile,
+        })),
+    };
+    const assets = executeProvider(plan.activitySources[plan.nodes[0].runtime.rendererRef], request);
+    const files = Object.fromEntries(Object.entries(assets).map(([name, a]) => [
+        name,
+        a.format === "json"
+            ? JSON.stringify(a.value, null, 2) + "\n"
+            : a.format === "yaml"
+                ? stringify(a.value)
+                : a.value,
+    ]));
+    const hooks = [
+        ...new Set(plan.nodes.map((n) => n.runtime?.lifecycleRef).filter(Boolean)),
+    ];
+    for (const ref of hooks) {
+        const node = plan.nodes.find((n) => n.runtime?.lifecycleRef === ref);
+        const schemas = node.runtime?.outputSchemasRef
+            ? JSON.parse(plan.activitySources[node.runtime.outputSchemasRef])
+            : {};
+        validateProviderFiles(plan.activitySources[ref], schemas, files);
+    }
+    return files;
+}
+/** The same provider schema gate applies after native rendering and project assembly. */
+export function validateProviderFiles(code, schemas, files) {
+    const response = executeProvider(code, { apiVersion: "ingestron.provider-output/v1", files }, false, "validateOutput");
+    check(Array.isArray(response.documents) && response.documents.length <= 100, "PROVIDER", "Invalid output validation response");
+    const ajv = new Ajv({ allErrors: true, strict: false });
+    for (const doc of response.documents) {
+        check(typeof schemas[doc.schema] === "string", "PROVIDER", "Undeclared output schema");
+        const validate = ajv.compile(JSON.parse(schemas[doc.schema]));
+        check(validate(doc.value), "PROVIDER", `Provider output schema: ${ajv.errorsText(validate.errors)}`);
+    }
+}

package/dist/plugins/provider-renderer.d.ts

@@ -0,0 +1,7 @@
+export type ProviderAssets = Record<string, {
+    format: "json" | "yaml" | "text";
+    value: any;
+}>;
+/** Only JSON crosses the process/VM boundary. The guest cannot access files or networks. */
+export declare function executeProvider(code: string, input: unknown, validateOnly?: boolean, operation?: "render" | "expand" | "command" | "author" | "model" | "validateOutput"): ProviderAssets;
+export declare function expandProvider(code: string, input: unknown): any;

package/dist/plugins/provider-renderer.js

@@ -0,0 +1,37 @@
+import { compiledPath } from "../core/installation.js";
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, writeFileSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { resolve } from "node:path";
+import { check, Problem } from "../core/errors.js";
+/** Only JSON crosses the process/VM boundary. The guest cannot access files or networks. */
+export function executeProvider(code, input, validateOnly = false, operation = "render") {
+    const request = JSON.stringify({ code, input, validateOnly, operation });
+    check(request.length <= 10_000_000, "PROVIDER", "Provider request exceeds 10 MB");
+    let response;
+    // A private request file avoids large synchronous stdin pipe stalls. The VM
+    // receives parsed JSON only; it never receives host filesystem access.
+    const temporary = mkdtempSync(resolve(tmpdir(), "ingestron-worker-"));
+    const requestPath = resolve(temporary, "request.json");
+    try {
+        writeFileSync(requestPath, request, { mode: 0o600, flag: "wx" });
+        response = JSON.parse(execFileSync(process.execPath, [compiledPath("plugins/provider-worker.js"), requestPath], {
+            encoding: "utf8",
+            env: {},
+            timeout: 30000,
+            maxBuffer: 10_000_000,
+            stdio: ["ignore", "pipe", "pipe"],
+        }));
+    }
+    catch {
+        throw new Problem("PROVIDER", "Provider exceeded its process limits or could not run");
+    }
+    finally {
+        rmSync(temporary, { recursive: true, force: true });
+    }
+    check(response.ok, "PROVIDER", response.error ?? "Provider renderer failed");
+    return response.assets;
+}
+export function expandProvider(code, input) {
+    return executeProvider(code, input, false, "expand");
+}

package/dist/plugins/provider-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/plugins/provider-worker.js

@@ -0,0 +1,74 @@
+/** Isolated provider generation: one ES module, JSON in/out, no host APIs. */
+import { getQuickJS } from "quickjs-emscripten";
+import { readFileSync } from "node:fs";
+import { isMap, check } from "../core/errors.js";
+const request = JSON.parse(readFileSync(process.argv[2] ?? 0, "utf8"));
+try {
+    check(typeof request.code === "string" && request.code.length <= 2_000_000, "PROVIDER", "Provider source exceeds 2 MB");
+    const engine = await getQuickJS(), runtime = engine.newRuntime();
+    runtime.setMemoryLimit(64 * 1024 * 1024);
+    runtime.setMaxStackSize(512 * 1024);
+    const deadline = Date.now() + 2000;
+    runtime.setInterruptHandler(() => Date.now() > deadline);
+    runtime.setModuleLoader((name) => {
+        if (name !== "provider-entry")
+            throw new Error("Provider imports are forbidden");
+        return request.code;
+    });
+    const context = runtime.newContext();
+    const evaluate = (source, module = false) => {
+        const result = context.evalCode(source, module ? "provider.mjs" : "request.js", { type: module ? "module" : "global" });
+        if (result.error) {
+            const message = context.dump(result.error);
+            result.error.dispose();
+            throw new Error(message.message ?? String(message));
+        }
+        const value = context.dump(result.value);
+        result.value.dispose();
+        return value;
+    };
+    try {
+        evaluate("globalThis.Date = undefined; globalThis.performance = undefined; Math.random = () => { throw new Error('Providers must be deterministic'); };");
+        evaluate("import * as provider from 'provider-entry'; globalThis.__provider = provider;", true);
+        const encoded = JSON.stringify(JSON.stringify(request.input));
+        const result = evaluate(["command", "author", "model", "validateOutput"].includes(request.operation)
+            ? `JSON.stringify((() => { const value = __provider[${JSON.stringify(request.operation)}](JSON.parse(${encoded})); if (value && typeof value.then === 'function') throw new Error('Provider command must be synchronous'); return value; })())`
+            : request.operation === "expand"
+                ? `JSON.stringify((() => { const value = __provider.expand(JSON.parse(${encoded})); if (value && typeof value.then === 'function') throw new Error('Provider expand must be synchronous'); return value; })())`
+                : `JSON.stringify((() => { const input = JSON.parse(${encoded}); const validation = __provider.validate(input); if (validation && typeof validation.then === 'function') throw new Error('Provider validate must be synchronous'); const result = ${request.validateOnly ? "{}" : "__provider.render(input)"}; if (result && typeof result.then === 'function') throw new Error('Provider render must be synchronous'); return result; })())`);
+        check(typeof result === "string" && result.length <= 8_000_000, "PROVIDER", "Provider output exceeds 8 MB or is not synchronous JSON");
+        const assets = JSON.parse(result);
+        check(isMap(assets) && Object.keys(assets).length <= 1000, "PROVIDER", "Provider must return at most 1000 assets");
+        if (request.operation === "expand") {
+            check(Array.isArray(assets.steps) &&
+                assets.steps.length <= 1000 &&
+                isMap(assets.recovery) &&
+                Object.keys(assets).every((k) => ["steps", "recovery"].includes(k)), "PROVIDER", "Invalid provider planning response");
+        }
+        for (const [path, asset] of request.operation !== "render"
+            ? []
+            : Object.entries(assets)) {
+            check(path.length <= 512 &&
+                !path.startsWith("/") &&
+                !path.includes("\\") &&
+                !path.includes("\0") &&
+                path.split("/").every((p) => p && p !== "." && p !== ".."), "PROVIDER", "Unsafe provider output path");
+            check(isMap(asset) &&
+                ["json", "yaml", "text"].includes(asset.format) &&
+                Object.hasOwn(asset, "value") &&
+                Object.keys(asset).every((k) => ["format", "value"].includes(k)), "PROVIDER", "Invalid provider asset envelope");
+            check(asset.format !== "text" || typeof asset.value === "string", "PROVIDER", "Text asset must contain a string");
+        }
+        process.stdout.write(JSON.stringify({ ok: true, assets }));
+    }
+    finally {
+        context.dispose();
+        runtime.dispose();
+    }
+}
+catch (error) {
+    process.stdout.write(JSON.stringify({
+        ok: false,
+        error: error instanceof Error ? error.message : String(error),
+    }));
+}

package/dist/sdk/adapter.d.ts

@@ -0,0 +1,15 @@
+/** First-party adapter helpers; version-lock adapters to this package. */
+export { Configuration } from "../core/config.js";
+export { fence, packageYaml, resolvePackage } from "../core/packages.js";
+export { check, Problem } from "../core/errors.js";
+export { friendlyReference } from "../core/package-references.js";
+export { installPackage, installedProviders } from "../core/packages.js";
+export { digest, canonical } from "../core/errors.js";
+export { planProject, inspectProject } from "../core/planner.js";
+export { renderProject, writeOutput, validateOutput, } from "../core/generate.js";
+export { checkPluginCompatibility } from "../plugins/compatibility.js";
+export { canonicalPackageReference, packageLock, extensionPackSchema, } from "../core/packages.js";
+export { taggedVersions, providerVersions, } from "../core/package-references.js";
+export { initialise, addFlow, addTable, apply } from "../core/authoring.js";
+export { contractColumns } from "../core/contracts.js";
+export { browseProviders, pluginKindLabel } from "../core/installed-plugins.js";

package/dist/sdk/adapter.js

@@ -0,0 +1,15 @@
+/** First-party adapter helpers; version-lock adapters to this package. */
+export { Configuration } from "../core/config.js";
+export { fence, packageYaml, resolvePackage } from "../core/packages.js";
+export { check, Problem } from "../core/errors.js";
+export { friendlyReference } from "../core/package-references.js";
+export { installPackage, installedProviders } from "../core/packages.js";
+export { digest, canonical } from "../core/errors.js";
+export { planProject, inspectProject } from "../core/planner.js";
+export { renderProject, writeOutput, validateOutput, } from "../core/generate.js";
+export { checkPluginCompatibility } from "../plugins/compatibility.js";
+export { canonicalPackageReference, packageLock, extensionPackSchema, } from "../core/packages.js";
+export { taggedVersions, providerVersions, } from "../core/package-references.js";
+export { initialise, addFlow, addTable, apply } from "../core/authoring.js";
+export { contractColumns } from "../core/contracts.js";
+export { browseProviders, pluginKindLabel } from "../core/installed-plugins.js";

package/dist/sdk/index.d.ts

@@ -0,0 +1,4 @@
+/** Node runtime API. Import /schemas for editor-only validation without runtime loading. */
+export { execute, executeAsync, operationSchemas } from "../core/operations.js";
+export type { Context, Result, OperationName } from "../core/operations.js";
+export { version } from "../version.js";

package/dist/sdk/index.js

@@ -0,0 +1,3 @@
+/** Node runtime API. Import /schemas for editor-only validation without runtime loading. */
+export { execute, executeAsync, operationSchemas } from "../core/operations.js";
+export { version } from "../version.js";

package/dist/sdk/schemas.d.ts

@@ -0,0 +1,34 @@
+import { projectSchema, flowSchema, environmentSchema, stepSchema } from "../core/schema.js";
+export { projectSchema, flowSchema, environmentSchema, stepSchema };
+export declare const documentSchemas: {
+    readonly project: typeof projectSchema;
+    readonly flow: typeof flowSchema;
+    readonly environment: typeof environmentSchema;
+    readonly step: typeof stepSchema;
+};
+export type DocumentKind = keyof typeof documentSchemas;
+export interface DocumentDiagnostic {
+    code: string;
+    message: string;
+    file: string;
+    pointer: string;
+    /** Zero-based UTF-16 offsets into the supplied content; end is exclusive. */
+    range: {
+        start: number;
+        end: number;
+    };
+    /** One-based line and column for the start offset. */
+    line: number;
+    column: number;
+}
+export interface DocumentValidation {
+    apiVersion: "ingestron.document-validation/v1";
+    valid: boolean;
+    diagnostics: DocumentDiagnostic[];
+}
+/** Validate the caller's unsaved YAML/JSON buffer, never the on-disk file. */
+export declare function validateDocument(input: {
+    kind: DocumentKind;
+    content: string;
+    uri: string;
+}): DocumentValidation;

package/dist/sdk/schemas.js

@@ -0,0 +1,74 @@
+/** Editor-safe exports: no filesystem access, runtime loading or plugin execution. */
+import { LineCounter, parseDocument, isMap, isSeq } from "yaml";
+import { projectSchema, flowSchema, environmentSchema, stepSchema, } from "../core/schema.js";
+export { projectSchema, flowSchema, environmentSchema, stepSchema };
+export const documentSchemas = {
+    project: projectSchema,
+    flow: flowSchema,
+    environment: environmentSchema,
+    step: stepSchema,
+};
+/** Validate the caller's unsaved YAML/JSON buffer, never the on-disk file. */
+export function validateDocument(input) {
+    const lines = new LineCounter();
+    const doc = parseDocument(input.content, { lineCounter: lines });
+    const diagnostics = [];
+    const add = (code, message, path, start, end) => {
+        const position = lines.linePos(start);
+        diagnostics.push({
+            code,
+            message,
+            file: input.uri,
+            pointer: path.length
+                ? "/" +
+                    path
+                        .map((p) => String(p).replaceAll("~", "~0").replaceAll("/", "~1"))
+                        .join("/")
+                : "",
+            range: { start, end },
+            line: position.line,
+            column: position.col,
+        });
+    };
+    for (const error of doc.errors)
+        add("YAML", error.message, [], error.pos[0], error.pos[1]);
+    if (diagnostics.length)
+        return {
+            apiVersion: "ingestron.document-validation/v1",
+            valid: false,
+            diagnostics,
+        };
+    try {
+        const schema = Object.hasOwn(documentSchemas, input.kind)
+            ? documentSchemas[input.kind]
+            : undefined;
+        if (!schema) {
+            add("SCHEMA", "Unsupported document kind", [], 0, 0);
+        }
+        else {
+            const result = schema.safeParse(doc.toJS({ maxAliasCount: 100 }));
+            if (!result.success)
+                for (const issue of result.error.issues) {
+                    let node = doc.contents;
+                    for (const key of issue.path) {
+                        const child = isMap(node) || isSeq(node) ? node.get(key, true) : undefined;
+                        if (!child)
+                            break;
+                        node = child;
+                    }
+                    const range = node && typeof node === "object" && "range" in node
+                        ? node.range
+                        : null;
+                    add("SCHEMA", issue.message, issue.path, range?.[0] ?? 0, range?.[1] ?? 0);
+                }
+        }
+    }
+    catch (error) {
+        add("YAML", error instanceof Error ? error.message : String(error), [], 0, 0);
+    }
+    return {
+        apiVersion: "ingestron.document-validation/v1",
+        valid: diagnostics.length === 0,
+        diagnostics,
+    };
+}

package/dist/version.d.ts

@@ -0,0 +1,5 @@
+export declare const version: string;
+/** Legacy minimumCli values describe the existing plugin ABI, not the reset app
+ * release number. Keep this compatibility level until manifests gain an explicit
+ * protocol range; never raise it without installed-provider acceptance tests. */
+export declare const pluginCompatibilityVersion = "4.2.0";

package/dist/version.js

@@ -0,0 +1,7 @@
+import { installationRoot } from "./core/installation.js";
+import { readFileSync } from "node:fs";
+export const version = JSON.parse(readFileSync(installationRoot + "/package.json", "utf8")).version;
+/** Legacy minimumCli values describe the existing plugin ABI, not the reset app
+ * release number. Keep this compatibility level until manifests gain an explicit
+ * protocol range; never raise it without installed-provider acceptance tests. */
+export const pluginCompatibilityVersion = "4.2.0";

package/package.json

@@ -0,0 +1,72 @@
+{
+  "name": "@ingestron/core",
+  "version": "0.12.0",
+  "type": "module",
+  "description": "Shared Ingestron project compiler, operations and editor schemas",
+  "license": "Apache-2.0",
+  "engines": {
+    "node": ">=22 <23"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "SECURITY.md",
+    "LICENSE",
+    "NOTICE",
+    "THIRD_PARTY_NOTICES.md"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/sdk/index.d.ts",
+      "import": "./dist/sdk/index.js"
+    },
+    "./schemas": {
+      "types": "./dist/sdk/schemas.d.ts",
+      "import": "./dist/sdk/schemas.js"
+    },
+    "./adapter": {
+      "types": "./dist/sdk/adapter.d.ts",
+      "import": "./dist/sdk/adapter.js"
+    }
+  },
+  "dependencies": {
+    "@lezer/python": "1.1.19",
+    "ajv": "8.20.0",
+    "ajv-formats": "3.0.1",
+    "quickjs-emscripten": "0.32.0",
+    "yaml": "2.9.0",
+    "zod": "4.5.4"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ingestron/core.git"
+  },
+  "scripts": {
+    "build": "node scripts/build-assets.mjs && tsc && node scripts/compiler-fingerprint.mjs",
+    "test": "tsx --test --test-concurrency=1 test/unit/*.test.ts test/providers/*.test.ts test/integration/*.test.ts",
+    "format": "prettier --write .",
+    "format:check": "prettier --check .",
+    "secrets:check": "node scripts/secrets.mjs",
+    "release:check": "node scripts/release-check.mjs",
+    "package:check": "tsx scripts/package-check.ts",
+    "validate": "pnpm format:check && pnpm build && pnpm test && pnpm secrets:check && pnpm release:check && pnpm package:check",
+    "prepack": "npm run build",
+    "prepublishOnly": "node scripts/release-check.mjs --publish",
+    "licences:inventory": "node scripts/licence-inventory.mjs"
+  },
+  "packageManager": "pnpm@10.15.0",
+  "devDependencies": {
+    "@types/node": "22.20.1",
+    "prettier": "3.9.6",
+    "tsx": "4.23.13",
+    "typescript": "7.0.2"
+  },
+  "homepage": "https://ingestron.io",
+  "bugs": {
+    "url": "https://github.com/ingestron/core/issues"
+  }
+}