ingestron 0.17.8 → 1.1.0

package/build/cli/cli/operation-worker.d.ts

@@ -1 +0,0 @@
-export {};

package/build/cli/cli/operation-worker.js

@@ -1,5 +0,0 @@
-/** Keep package-network work off the presentation thread so progress reflects
- * actual pending work. This worker uses the same JSON operations as MCP/Web. */
-import { parentPort, workerData } from "node:worker_threads";
-import { executePluginOperation } from "./plugin-resolution.js";
-parentPort.postMessage(await executePluginOperation(workerData.context, workerData.operation, workerData.args));

package/build/cli/cli/output.d.ts

@@ -1,3 +0,0 @@
-import type { Result } from "@ingestron/core";
-export declare function exitCode(result: Result): number;
-export declare function terminal(result: Result, verbose?: boolean, args?: any): string;

package/build/cli/cli/output.js

@@ -1,246 +0,0 @@
-/** Presentation is deliberately separate from operation results. JSON is the
- * lossless API; terminal output is bounded, plain text and actionable. Never
- * serialise arbitrary plugin data or resolved bindings in the default summary. */
-import { friendlyReference } from "@ingestron/core/adapter";
-import { stringify } from "yaml";
-import { officialPlugins } from "./plugin-resolution.js";
-export function exitCode(result) {
-    if (result.ok)
-        return 0;
-    const codes = result.diagnostics.map((d) => d.code);
-    if (codes.includes("CANCELLED"))
-        return 130;
-    if (codes.includes("NOT_IMPLEMENTED"))
-        return 4;
-    if (codes.some((c) => ["STALE", "OWNER", "CONFLICT"].includes(c)))
-        return 5;
-    if (codes.some((c) => [
-        "INPUT",
-        "BINDING",
-        "CAPABILITY",
-        "PACKAGE",
-        "OPTION",
-        "COMMAND",
-        "SCHEMA",
-    ].includes(c)))
-        return 3;
-    return 2;
-}
-const clean = (value) => String(value).replace(/[\u0000-\u001f\u007f-\u009f]/g, " ");
-// Data product ownership from ODCS contracts; absent with older core versions.
-function dataProductLines(products, quality) {
-    if (!Array.isArray(products) || !products.length)
-        return "";
-    const summary = quality?.summary;
-    const shown = products.slice(0, 20).map((p) => {
-        const owners = Array.isArray(p.owners) ? p.owners.map(clean) : [];
-        return `\n  ${clean(p.contract)}  ${clean(p.status ?? "no status")}  ${owners.length ? `owner ${owners.join(", ")}` : "no owner"}`;
-    });
-    const recorded = products.reduce((n, p) => n + Math.max(0, (p.qualityRules ?? 0) - (p.enforcedQualityRules ?? 0)), 0);
-    return ("\nData products:" +
-        shown.join("") +
-        (products.length > 20 ? `\n  … ${products.length - 20} more` : "") +
-        (summary && summary.rules
-            ? `\nQuality rules: ${summary.rules} (${summary.atLoad} at load, ${summary.afterLoad} after load, ${summary.unsupported} not enforced by the target${summary.documentation ? `, ${summary.documentation} documentation only` : ""}).`
-            : !summary && recorded
-                ? `\n${recorded} quality rule(s) are recorded in contracts but not yet enforced.`
-                : ""));
-}
-// Quality results recorded by connector runtimes before commit; counts only.
-function runQualityLine(flows) {
-    if (!Array.isArray(flows))
-        return "";
-    const rules = flows.flatMap((f) => Array.isArray(f?.quality) ? f.quality : []);
-    if (!rules.length)
-        return "";
-    const warned = rules.filter((r) => r.passed === false);
-    return `\nQuality rules: ${rules.length} checked before commit${warned.length
-        ? `; ${warned.length} warning(s): ${warned
-            .slice(0, 5)
-            .map((r) => clean(r.id))
-            .join(", ")}${warned.length > 5 ? ", …" : ""}`
-        : ", all passed"}.`;
-}
-// Source routes from core (PB-064); absent with older core versions.
-const costLabel = {
-    none: "no extra cost",
-    included: "included in platform pricing",
-    "separately-billed": "billed separately",
-};
-function routeLabel(route) {
-    const r = route?.reference;
-    const via = route?.route === "native"
-        ? `native on ${clean(route.configuration)} via ${(route.standards ?? []).map(clean).join(", ")}`
-        : route?.route === "bridge"
-            ? `bridge: lands on ${clean(route.via)}, ingests on ${clean(route.configuration)} via ${(route.standards ?? []).map(clean).join(" → ")}`
-            : `portable ${clean(route?.package ?? "connector")} on ${clean(route?.configuration)}`;
-    if (!r)
-        return `${via} [no reference record]`;
-    const stale = Date.now() - new Date(`${r.verified}T00:00:00Z`).getTime() >
-        180 * 86_400_000;
-    return `${via} [${clean(r.maturity)}, ${costLabel[r.cost?.model] ?? clean(r.cost?.model)}${stale ? `, record checked ${clean(r.verified)}` : ""}]`;
-}
-function sourceLines(sources) {
-    if (!Array.isArray(sources) || !sources.length)
-        return "";
-    return ("\nSources:" +
-        sources
-            .slice(0, 20)
-            .map((s) => `\n  ${clean(s.flow)}: ${clean(s.connection)} (${clean(s.kind)}) ${routeLabel(s.selected)}` +
-            (s.alternative
-                ? `\n    Also available: ${routeLabel(s.alternative)}`
-                : ""))
-            .join("") +
-        (sources.length > 20 ? `\n  … ${sources.length - 20} more` : "") +
-        "\nUse --json for every route with documentation, licence, cost and access.");
-}
-function pluginLabel(reference) {
-    for (const [name, entry] of Object.entries(officialPlugins)) {
-        const prefixes = [
-            `${entry.repository}/${entry.path}@`,
-            ...(name === "local" ? [`${entry.repository}@`] : []),
-        ];
-        for (const prefix of prefixes)
-            if (reference.startsWith(prefix)) {
-                const version = reference.slice(prefix.length);
-                if (/^\d+\.\d+\.\d+$/.test(version))
-                    return `${name}@${version}`;
-            }
-    }
-    return friendlyReference(reference);
-}
-function lines(title, items, empty) {
-    if (!items.length)
-        return empty;
-    return `${title} (${items.length})\n${items
-        .slice(0, 20)
-        .map((i) => `  ${clean(i)}`)
-        .join("\n")}${items.length > 20 ? "\n  More results available with --json or --verbose." : ""}`;
-}
-export function terminal(result, verbose = false, args = {}) {
-    if (!result.ok)
-        return result.diagnostics
-            .map((d) => `Error: ${clean(d.message)}${verbose ? ` [${d.code}]` : ""}${d.file ? `\n  File: ${clean(d.file)}${clean(d.pointer ?? "")}` : ""}${d.hint ? `\n  Next: ${clean(d.hint)}` : ""}`)
-            .join("\n");
-    const value = result.result;
-    if (verbose)
-        return `${result.operation.replaceAll("_", " ")}\n${typeof value === "string" ? value : stringify(value).trimEnd()}`;
-    if (value?.apiVersion === "ingestron.change/v1")
-        return (lines("Proposed file changes", value.changes.map((c) => `${c.before === null ? "Create" : "Update"} ${c.path}`), "No file changes needed.") + "\nPreview only; no files changed.");
-    if (value?.applied)
-        return lines("Saved files", value.files, "No file changes needed.");
-    switch (result.operation) {
-        case "plugin_browse":
-            return ((value.length
-                ? "Installed plugins\n\n" +
-                    value
-                        .map((p) => `  ${clean(p.id).padEnd(13)} ${clean(p.name)} [${clean(p.kindLabel)} · ${clean(p.availability)}]\n    ${clean(p.description)}\n    ${clean(p.url)}`)
-                        .join("\n\n")
-                : "No installed plugins match this search.") +
-                "\n\nInstall a provider: ingestron provider install local\nInstall a connector: ingestron connector install github");
-        case "plugin_versions":
-            return (lines(`Versions for ${value.provider}`, value.versions.map((v) => `${v.reference}${v.ambiguous ? "  (conflicting tags; select a commit)" : ""}`), "No stable version tags found.") +
-                "\n\n" +
-                value.evidence +
-                "\nInstall: ingestron plugin install <owner/repository>@<version>");
-        case "packages_install":
-            return `${value.cached ? "Using cached" : "Installed"} ${args.kind ?? "plugin"} ${clean(pluginLabel(args.reference ?? value.reference ?? "package"))}.\nUse --verbose for the exact reference and licence information.`;
-        case "packages_list":
-            return lines("Installed plugins", Object.keys(value.packages).map(pluginLabel), "No plugins installed.\nNext: ingestron provider install local");
-        case "source_list":
-            return lines("Sources", value.map((s) => `${s.id}  ${s.type}${s.format ? "/" + s.format : ""}  ${s.execution}`), "No sources configured.\nNext: ingestron source add <name> --type <type>");
-        case "source_show":
-            return `Source ${clean(value.id)}\n  Type: ${clean(value.type)}${value.format ? ` (${clean(value.format)})` : ""}\n  Execution: ${clean(value.selected.execution)}\n  Environment: ${clean(value.selected.environment)}\n  Provider: ${clean(value.selected.provider ?? "project default")}\nUse --verbose to inspect connection references.`;
-        case "contract_list":
-            return lines("Contracts", value.map((c) => `${c.id}  ${c.status}  ${c.path}`), "No contracts drafted.\nNext: ingestron contract draft <name> --metadata <file>");
-        case "contract_show":
-            return (`Contract ${clean(value.id)} (${clean(value.status)})\n` +
-                lines("Entities", (value.schema ?? []).map((s) => `${s.physicalName ?? s.name}: ${(s.properties ?? []).length} fields`), "No entities.") +
-                "\nUse --verbose to review the complete contract.");
-        case "contract_check":
-            return `Contract valid: ${clean(value.path)}${value.status === "draft" ? "\nStatus: draft; review business requirements before use." : ""}\n${clean(value.evidence)}`;
-        case "contract_validate":
-            return `Contract valid for native table authoring (${value.columns.length} columns).`;
-        case "inspect":
-            return lines("Flows", value.flows.map((f) => `${f.id}  ${f.kind}  ${f.tables.length} tables, ${f.outputs.length} outputs`), "No flows configured.\nNext: ingestron flow add <name> --contract <name> --standard <standard>");
-        case "flow_show":
-            return `Flow ${clean(value.id)}\n  Kind: ${clean(value.kind)}\n  Tables: ${Object.keys(value.tables ?? {})
-                .map(clean)
-                .join(", ") || "none"}\n  Outputs: ${Object.keys(value.publishes ?? {})
-                .map(clean)
-                .join(", ") || "none"}\nUse --verbose to inspect configuration.`;
-        case "validate":
-            return value.mode === "draft"
-                ? `Draft configuration checked (${value.flows} flows).\n${value.pending.length} unresolved inputs. Strict build readiness has not been checked.`
-                : `Project checks passed${value.nodes === undefined ? "" : ` (${value.nodes} planned nodes)`}.\nValidation is local; no platform connection was tested.` +
-                    dataProductLines(value.dataProducts, value.quality) +
-                    sourceLines(value.sources);
-        case "doctor":
-            return `Project ${clean(value.project)} (${clean(value.environment)})\n${value.pending.length} unresolved inputs.\n${clean(value.platformAccess)}\nNext: ${clean(value.next)}`;
-        case "runtime_prepare":
-            return (`Runtime ${clean(value.status)} for ${value.flows.map(clean).join(", ")}.` +
-                (value.result?.flows ?? [])
-                    .map((f) => `\n  ${clean(f.flow)}: Python ${clean(f.pythonVersion ?? "3.12")} (${f.reused ? "reused" : "prepared"})`)
-                    .join("") +
-                `\nUse ingestron run status ${clean(value.id)} --verbose for environment paths and details.`);
-        case "discover":
-            return (`Discovered ${clean(value.flow)} (${clean(value.route)}):` +
-                (value.tables ?? [])
-                    .map((t) => `\n  ${clean(t.table)}: ${Number(t.fields)} fields → ${clean(t.file)}` +
-                    (t.keys?.length
-                        ? ` (source keys: ${t.keys.map(clean).join(", ")})`
-                        : "") +
-                    (t.skipped ?? [])
-                        .slice(0, 10)
-                        .map((s) => `\n    skipped ${clean(s.name)}: ${clean(s.reason)}`)
-                        .join(""))
-                    .join("") +
-                (value.files ?? []).map((f) => `\n  wrote ${clean(f)}`).join("") +
-                (value.review ?? [])
-                    .slice(0, 10)
-                    .map((r) => `\n  review: ${clean(r)}`)
-                    .join("") +
-                `\nNext: ${clean(value.next)}`);
-        case "run":
-        case "run_status":
-            return `Run ${clean(value.id)}: ${clean(value.status)} (${clean(value.action)}).\nTarget: ${clean(value.configuration)}; flows: ${value.flows.map(clean).join(", ")}.${runQualityLine(value.result?.flows)}\nUse ingestron run status ${clean(value.id)} to inspect the receipt.`;
-        case "build":
-            return `Built ${value.packages?.length ?? 1} project package(s) into ${clean(value.directory)}.\n${clean(value.evidence)}`;
-        case "generate":
-            return "Generated native assets.\nUse check --output <directory> to verify local output.";
-        case "validate_output":
-            return "Generated output checks passed.\nOwnership and syntax checked locally; no platform connection was tested.";
-        case "environments":
-            return lines("Environments", value.available.map((e) => `${e}${e === value.selected ? " (selected)" : ""}`), "No environments configured.");
-        case "tables":
-            return lines("Flow tables", value.map((f) => `${f.flow}: ${f.tables.join(", ") || "none"}`), "No tables configured.");
-        case "catalogue":
-            return lines(args.kind ?? "Capabilities", value.map((v) => `${v.id}${v.version ? "@" + v.version : ""}${v.description ? "  " + v.description : ""}`), `No ${args.kind ?? "capabilities"} available.\nNext: ingestron plugin list`);
-        case "provider_commands":
-            return lines("Plugin operations", value.commands.map((c) => `${c.name}  ${c.description}`), "No plugin operations available.");
-        case "plugin_command":
-        case "provider_command":
-            return "Plugin operation completed locally.\nUse --json, --verbose or --out to inspect its result.";
-        case "plan":
-            return `Local build plan ready (${value.nodes.length} nodes).\nUse --json or --verbose to inspect the plan.`;
-        case "diff":
-            return `Local differences: ${value.added.length} added, ${value.changed.length} changed, ${value.removed.length} removed.`;
-        case "config_explain":
-            return `Configuration: ${clean(value.path)} (${clean(value.environment)})\nUse --verbose to inspect declared and resolved values.`;
-        case "resolve":
-            return (`Project ${clean(value.project.id)} — resolved build configuration\n` +
-                `Providers: ${Object.keys(value.project.providers?.configurations ?? {})
-                    .map(clean)
-                    .join(", ") || "none"}\n` +
-                `Connections: ${Object.keys(value.project.connections ?? {})
-                    .map(clean)
-                    .join(", ") || "none"}\n` +
-                `Flows: ${value.flows.length}; unresolved inputs: ${value.pending.length}.\n` +
-                "Authored files: ingestron source list; ingestron contract list.\nUse --verbose for resolved configuration and input-file digests.");
-        case "read":
-            return value.content;
-        case "schema":
-            return "Schema available. Use --json for tools or --verbose to inspect it.";
-    }
-    return `${result.operation.replaceAll("_", " ")} completed.\nUse --json for structured data or --verbose for details.`;
-}

package/build/cli/cli/plugin-browser.d.ts

@@ -1,14 +0,0 @@
-interface Browser {
-    select(message: string, options: {
-        value: string;
-        label: string;
-        hint?: string;
-    }[]): Promise<string>;
-    search(): Promise<string>;
-    details(text: string): void;
-}
-/** Read-only navigation of installed plugin metadata. */
-export declare function browsePlugins(ui: Browser, query?: string, initialKind?: string, filters?: {
-    engine?: string;
-}, root?: string): Promise<void>;
-export {};

package/build/cli/cli/plugin-browser.js

@@ -1,74 +0,0 @@
-import { browseProviders, pluginKindLabel } from "@ingestron/core/adapter";
-/** Read-only navigation of installed plugin metadata. */
-export async function browsePlugins(ui, query = "", initialKind, filters = {}, root = process.cwd()) {
-    browseProviders(root, query, initialKind, filters); // Validate filters even when catalogue is empty.
-    let kind = initialKind;
-    let listing = !!query || !!kind;
-    let page = 0;
-    for (;;) {
-        if (!listing) {
-            const choice = await ui.select("Browse plugins", [
-                { value: "all", label: "Browse all plugins" },
-                { value: "search", label: "Search plugins" },
-                ...["provider", "model-pack", "activity-pack", "report-pack"].map((value) => ({ value, label: pluginKindLabel(value) })),
-                {
-                    value: "connector",
-                    label: "Connectors",
-                    hint: "Installed connector packages",
-                },
-                { value: "exit", label: "Finish browsing" },
-            ]);
-            if (choice === "exit")
-                return;
-            kind = ["all", "search"].includes(choice) ? undefined : choice;
-            query = choice === "search" ? await ui.search() : "";
-            listing = true;
-            page = 0;
-        }
-        const plugins = browseProviders(root, query, kind, filters);
-        const visible = plugins.slice(page * 25, (page + 1) * 25);
-        const selected = await ui.select(plugins.length
-            ? `Select a plugin (${plugins.length} results, page ${page + 1})`
-            : "No plugins match; change category or search", [
-            ...visible.map((p) => ({
-                value: p.reference,
-                label: `${p.name} [${p.kindLabel}]`,
-                hint: `${p.availability} · ${p.description}`,
-            })),
-            ...(page > 0 ? [{ value: ":previous", label: "Previous page" }] : []),
-            ...((page + 1) * 25 < plugins.length
-                ? [{ value: ":next", label: "Next page" }]
-                : []),
-            { value: ":search", label: "Search this category" },
-            { value: ":back", label: "Change category / browse all" },
-            { value: ":exit", label: "Finish browsing" },
-        ]);
-        if (selected === ":next") {
-            page++;
-            continue;
-        }
-        if (selected === ":previous") {
-            page--;
-            continue;
-        }
-        if (selected === ":exit")
-            return;
-        if (selected === ":back") {
-            listing = false;
-            continue;
-        }
-        if (selected === ":search") {
-            query = await ui.search();
-            page = 0;
-            continue;
-        }
-        const plugin = plugins.find((p) => p.reference === selected);
-        ui.details(`${plugin.name} (${plugin.id})\n${plugin.kindLabel} · ${plugin.availability}\n\n${plugin.description}\n${plugin.evidence}\n\nLicence: ${plugin.licence}\nDocumentation: ${plugin.documentation}\nSource and package documentation: ${plugin.url}\n\nExact-version licences/notices and provenance are saved to .ingestron/plugin-info/ during installation. Engine compatibility is package-specific; installed metadata is not runtime verification.`);
-        const action = await ui.select("Next step", [
-            { value: "back", label: "Back to results" },
-            { value: "exit", label: "Finish browsing" },
-        ]);
-        if (action === "exit")
-            return;
-    }
-}

package/build/cli/cli/plugin-resolution.d.ts

@@ -1,126 +0,0 @@
-/** Official-name convenience at the adapter boundary; core only receives exact references. */
-import { type Context, type OperationName, type Result } from "@ingestron/core";
-export declare const officialPlugins: {
-    readonly adf: {
-        readonly kind: "provider";
-        readonly repository: "ingestron/provider-adf";
-        readonly path: "plugin/provider.yaml";
-        readonly tagPrefix: "";
-    };
-    readonly "azure-blob": {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/azure-blob/connector.yaml";
-        readonly tagPrefix: "azure-blob-";
-    };
-    readonly databricks: {
-        readonly kind: "provider";
-        readonly repository: "ingestron/provider-databricks";
-        readonly path: "plugin/provider.yaml";
-        readonly tagPrefix: "";
-    };
-    readonly files: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/files/connector.yaml";
-        readonly tagPrefix: "files-";
-    };
-    readonly gcs: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/gcs/connector.yaml";
-        readonly tagPrefix: "gcs-";
-    };
-    readonly github: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/github/connector.yaml";
-        readonly tagPrefix: "github-";
-    };
-    readonly hubspot: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/hubspot/connector.yaml";
-        readonly tagPrefix: "hubspot-";
-    };
-    readonly jira: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/jira/connector.yaml";
-        readonly tagPrefix: "jira-";
-    };
-    readonly local: {
-        readonly kind: "provider";
-        readonly repository: "ingestron/provider-local";
-        readonly path: "plugin/provider.yaml";
-        readonly tagPrefix: "";
-    };
-    readonly mysql: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/mysql/connector.yaml";
-        readonly tagPrefix: "mysql-";
-    };
-    readonly onedrive: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/onedrive/connector.yaml";
-        readonly tagPrefix: "onedrive-";
-    };
-    readonly oracle: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/oracle/connector.yaml";
-        readonly tagPrefix: "oracle-";
-    };
-    readonly postgresql: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/postgresql/connector.yaml";
-        readonly tagPrefix: "postgresql-";
-    };
-    readonly s3: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/s3/connector.yaml";
-        readonly tagPrefix: "s3-";
-    };
-    readonly salesforce: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/salesforce/connector.yaml";
-        readonly tagPrefix: "salesforce-";
-    };
-    readonly sftp: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/sftp/connector.yaml";
-        readonly tagPrefix: "sftp-";
-    };
-    readonly sharepoint: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/sharepoint/connector.yaml";
-        readonly tagPrefix: "sharepoint-";
-    };
-    readonly "sql-server": {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/sql-server/connector.yaml";
-        readonly tagPrefix: "sql-server-";
-    };
-    readonly stripe: {
-        readonly kind: "connector";
-        readonly repository: "ingestron/connectors";
-        readonly path: "connectors/stripe/connector.yaml";
-        readonly tagPrefix: "stripe-";
-    };
-};
-export declare function officialName(reference: string): {
-    name: keyof typeof officialPlugins;
-    version: string;
-} | undefined;
-export declare function catalogueReleases(value: any, name: keyof typeof officialPlugins, host?: string): string[];
-export declare function fetchCatalogue(): Promise<unknown>;
-export declare function resolvePluginArgs(context: Context, operation: OperationName, args: any, catalogue?: typeof fetchCatalogue): Promise<any>;
-export declare function executePluginOperation(context: Context, operation: OperationName, args?: any): Promise<Result>;