@ingestron/core 0.12.0

package/dist/core/package-references.d.ts

@@ -0,0 +1,36 @@
+export declare function providerRepository(name: string): string;
+/** Preserve custom manifest paths and immutable selectors when formatting references. */
+export declare function friendlyReference(reference: string): string;
+export declare function packageGitEnvironment(): {
+    PATH: string | undefined;
+    HOME: string | undefined;
+    GH_CONFIG_DIR: string | undefined;
+    GH_TOKEN: string | undefined;
+    GITHUB_TOKEN: string | undefined;
+    GIT_CONFIG_GLOBAL: string;
+    GIT_CONFIG_NOSYSTEM: string;
+    GIT_TERMINAL_PROMPT: string;
+};
+export declare function gitTags(repository: string, fromGit?: string): string;
+export declare function taggedVersions(text: string, prefix?: string): {
+    version: string;
+    tag: string;
+    commit: string;
+    ambiguous: boolean;
+}[];
+export declare function providerVersions(name: string, fromGit?: string, tagPrefix?: string): {
+    provider: string;
+    repository: string;
+    source: string;
+    versions: {
+        version: string;
+        tag: string;
+        commit: string;
+        ambiguous: boolean;
+        repository: string;
+        reference: string;
+        selectable: boolean;
+        compatibility: string;
+    }[];
+    evidence: string;
+};

package/dist/core/package-references.js

@@ -0,0 +1,103 @@
+/** Generic package references and version discovery; no bundled provider registry. */
+import { realpathSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { check, Problem } from "./errors.js";
+export function providerRepository(name) {
+    check(/^[A-Za-z0-9][A-Za-z0-9_.-]*\/[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(name) &&
+        !name.includes(".."), "PACKAGE", "Use an explicit GitHub owner/repository");
+    return name;
+}
+/** Preserve custom manifest paths and immutable selectors when formatting references. */
+export function friendlyReference(reference) {
+    return reference
+        .replace(/\/plugin\/provider\.yaml(?=@|$)/, "")
+        .replace(/@v(?=\d+\.\d+\.\d+$)/, "@");
+}
+export function packageGitEnvironment() {
+    return {
+        PATH: process.env.PATH,
+        HOME: process.env.HOME,
+        GH_CONFIG_DIR: process.env.GH_CONFIG_DIR,
+        GH_TOKEN: process.env.GH_TOKEN,
+        GITHUB_TOKEN: process.env.GITHUB_TOKEN,
+        GIT_CONFIG_GLOBAL: "/dev/null",
+        GIT_CONFIG_NOSYSTEM: "1",
+        GIT_TERMINAL_PROMPT: "0",
+    };
+}
+export function gitTags(repository, fromGit) {
+    providerRepository(repository);
+    try {
+        return execFileSync("git", [
+            "-c",
+            "core.hooksPath=/dev/null",
+            "-c",
+            "credential.helper=!gh auth git-credential",
+            "ls-remote",
+            "--tags",
+            "--",
+            fromGit
+                ? realpathSync(fromGit)
+                : `https://github.com/${repository}.git`,
+        ], {
+            encoding: "utf8",
+            timeout: 30000,
+            maxBuffer: 1024 * 1024,
+            env: packageGitEnvironment(),
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+    }
+    catch {
+        // Git stderr may include authentication diagnostics. Do not echo it into
+        // terminal/JSON responses; callers get a stable, actionable diagnostic.
+        throw new Problem("PACKAGE", `Cannot read plugin versions from ${repository}`, undefined, undefined, "Check repository access and GitHub authentication, then retry.");
+    }
+}
+export function taggedVersions(text, prefix = "") {
+    check(/^[a-z0-9-]*$/.test(prefix), "PACKAGE", "Invalid package tag prefix");
+    const refs = new Map();
+    const lines = text.trim() ? text.trim().split("\n") : [];
+    check(lines.length <= 10000, "LIMIT", "Provider tag listing exceeds 10,000 refs");
+    for (const line of lines) {
+        if (prefix && !line.includes("refs/tags/" + prefix))
+            continue;
+        const match = /^([a-f0-9]{40})\s+refs\/tags\/(v?(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*))(\^\{\})?$/.exec(prefix ? line.replace("refs/tags/" + prefix, "refs/tags/") : line);
+        if (match)
+            refs.set(match[2] + (match[3] ?? ""), match[1]);
+    }
+    const versions = new Map();
+    for (const [tag, object] of refs) {
+        if (tag.endsWith("^{}"))
+            continue;
+        const version = tag.replace(/^v/, ""), commit = refs.get(tag + "^{}") ?? object;
+        const prior = versions.get(version);
+        const ambiguous = !!prior && (prior.ambiguous || prior.commit !== commit);
+        versions.set(version, {
+            version,
+            tag: prior && !tag.startsWith("v") ? prior.tag : prefix + tag,
+            commit,
+            ambiguous,
+        });
+    }
+    return [...versions.values()].sort((a, b) => {
+        const left = a.version.split(".").map(Number), right = b.version.split(".").map(Number);
+        return right[0] - left[0] || right[1] - left[1] || right[2] - left[2];
+    });
+}
+export function providerVersions(name, fromGit, tagPrefix = "") {
+    const repository = providerRepository(name);
+    const versions = taggedVersions(gitTags(repository, fromGit), tagPrefix).map((v) => ({
+        ...v,
+        repository,
+        reference: `${repository}@${v.version}`,
+        selectable: !v.ambiguous,
+        compatibility: "unchecked",
+    }));
+    return {
+        provider: repository,
+        repository,
+        source: fromGit ? "local-git" : "github",
+        versions,
+        evidence: "Git tags only; manifest and host compatibility are checked during installation",
+    };
+}

package/dist/core/packages.d.ts

@@ -0,0 +1,340 @@
+import { z } from "zod";
+export declare const externalActivitySchema: z.ZodObject<{
+    apiVersion: z.ZodLiteral<"ingestron.activity/v1">;
+    id: z.ZodString;
+    version: z.ZodString;
+    description: z.ZodString;
+    platform: z.ZodString;
+    input: z.ZodEnum<{
+        committed: "committed";
+        files: "files";
+        relation: "relation";
+        source: "source";
+    }>;
+    output: z.ZodEnum<{
+        committed: "committed";
+        files: "files";
+        relation: "relation";
+    }>;
+    effect: z.ZodEnum<{
+        commit: "commit";
+        control: "control";
+        read: "read";
+        stage: "stage";
+    }>;
+    capabilities: z.ZodDefault<z.ZodArray<z.ZodString>>;
+    optionsSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    generator: z.ZodOptional<z.ZodObject<{
+        kind: z.ZodEnum<{
+            "native-notebook": "native-notebook";
+            "native-pipeline": "native-pipeline";
+            "native-project": "native-project";
+        }>;
+        artifactKind: z.ZodOptional<z.ZodString>;
+        execution: z.ZodOptional<z.ZodString>;
+        persistence: z.ZodOptional<z.ZodEnum<{
+            capture: "capture";
+            durable: "durable";
+            ephemeral: "ephemeral";
+        }>>;
+        runtime: z.ZodOptional<z.ZodString>;
+        code: z.ZodOptional<z.ZodString>;
+        scope: z.ZodDefault<z.ZodEnum<{
+            collection: "collection";
+            table: "table";
+        }>>;
+        modes: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+            collection: "collection";
+            table: "table";
+        }>>>;
+        groupBy: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        dependencyOptions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
+        entrypoint: z.ZodDefault<z.ZodString>;
+    }, z.core.$strict>>;
+    requirements: z.ZodOptional<z.ZodObject<{
+        execution: z.ZodString;
+        features: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        sourceKinds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        evidence: z.ZodLiteral<"offline">;
+    }, z.core.$strict>>;
+    templates: z.ZodDefault<z.ZodArray<z.ZodObject<{
+        source: z.ZodString;
+        output: z.ZodString;
+        format: z.ZodEnum<{
+            json: "json";
+            python: "python";
+            sql: "sql";
+            text: "text";
+            yaml: "yaml";
+        }>;
+    }, z.core.$strict>>>;
+    dependencies: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
+}, z.core.$strict>;
+export type ExternalActivity = z.infer<typeof externalActivitySchema>;
+export declare const providerPackageSchema: z.ZodObject<{
+    apiVersion: z.ZodLiteral<"ingestron.provider/v1">;
+    id: z.ZodString;
+    version: z.ZodString;
+    platform: z.ZodString;
+    activities: z.ZodRecord<z.ZodString, z.ZodString>;
+    capabilities: z.ZodOptional<z.ZodObject<{
+        artifactKinds: z.ZodArray<z.ZodString>;
+        delivery: z.ZodDefault<z.ZodBoolean>;
+        handovers: z.ZodDefault<z.ZodArray<z.ZodEnum<{
+            files: "files";
+            relation: "relation";
+        }>>>;
+        packContracts: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+            version: z.ZodString;
+            activities: z.ZodArray<z.ZodString>;
+        }, z.core.$strict>>>;
+    }, z.core.$strict>>;
+    projectAssembly: z.ZodOptional<z.ZodObject<{
+        apiVersion: z.ZodLiteral<"ingestron.project-assembly/v1">;
+        command: z.ZodString;
+    }, z.core.$strict>>;
+    connectorRuntimes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
+        selectionSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        executionSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        prepareCommand: z.ZodString;
+        contractsCommand: z.ZodOptional<z.ZodString>;
+        executionPlatforms: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>>;
+    configurationSchema: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodRecord<z.ZodString, z.ZodUnknown>]>>;
+    standards: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        description: z.ZodString;
+        sourceKinds: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        dataFlow: z.ZodOptional<z.ZodEnum<{
+            forbidden: "forbidden";
+            required: "required";
+        }>>;
+    }, z.core.$strict>>>;
+    compatibility: z.ZodOptional<z.ZodObject<{
+        plan: z.ZodLiteral<"ingestron.plan/v1">;
+        requiredFeatures: z.ZodDefault<z.ZodArray<z.ZodString>>;
+        minimumCli: z.ZodString;
+    }, z.core.$strict>>;
+    outputSchemas: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    lifecycle: z.ZodOptional<z.ZodObject<{
+        apiVersion: z.ZodLiteral<"ingestron.provider-lifecycle/v1">;
+        module: z.ZodString;
+    }, z.core.$strict>>;
+    planner: z.ZodOptional<z.ZodObject<{
+        apiVersion: z.ZodLiteral<"ingestron.provider-planning/v1">;
+        module: z.ZodString;
+    }, z.core.$strict>>;
+    commands: z.ZodOptional<z.ZodObject<{
+        apiVersion: z.ZodLiteral<"ingestron.provider-commands/v1">;
+        namespace: z.ZodOptional<z.ZodString>;
+        execution: z.ZodLiteral<"offline-json">;
+        module: z.ZodString;
+        definitions: z.ZodArray<z.ZodObject<{
+            name: z.ZodString;
+            description: z.ZodString;
+            reportPack: z.ZodOptional<z.ZodBoolean>;
+            examples: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            resultSchema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            inputSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
+    execution: z.ZodOptional<z.ZodObject<{
+        apiVersion: z.ZodLiteral<"ingestron.execution/v1">;
+        transport: z.ZodString;
+        entryPoint: z.ZodString;
+        actions: z.ZodArray<z.ZodEnum<{
+            approve: "approve";
+            discover: "discover";
+            prepare: "prepare";
+            review: "review";
+            run: "run";
+        }>>;
+        status: z.ZodLiteral<"receipt">;
+        retry: z.ZodLiteral<"same-run-id">;
+        cancellation: z.ZodLiteral<"interrupt">;
+    }, z.core.$strict>>;
+    renderer: z.ZodOptional<z.ZodObject<{
+        apiVersion: z.ZodLiteral<"ingestron.provider-generation/v1">;
+        module: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+/** Data-only presets: no modules, nested dependencies or renderer replacement. */
+export declare const extensionPackSchema: z.ZodObject<{
+    apiVersion: z.ZodLiteral<"ingestron.extension-pack/v1">;
+    id: z.ZodString;
+    version: z.ZodString;
+    description: z.ZodString;
+    extends: z.ZodObject<{
+        provider: z.ZodString;
+        version: z.ZodString;
+        platform: z.ZodString;
+        contract: z.ZodString;
+        contractVersion: z.ZodString;
+    }, z.core.$strict>;
+    activities: z.ZodRecord<z.ZodString, z.ZodObject<{
+        uses: z.ZodString;
+        with: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export declare function fence(root: string, name: string): string;
+export declare function packageYaml(file: string): any;
+/** Stable convenience names; locks always retain the explicit repository and path. */
+export declare function canonicalPackageReference(reference: string): string;
+export declare function packageLock(root: string): {
+    apiVersion: "ingestron.packages-lock/v1";
+    packages: Record<string, {
+        repository: string;
+        path: string;
+        commit: string;
+        files: Record<string, string>;
+    }>;
+};
+export declare function resolvePackage(root: string, reference: string): {
+    file: string;
+    root: string;
+    entry: {
+        repository: string;
+        path: string;
+        commit: string;
+        files: Record<string, string>;
+    };
+};
+/** Derived information lives outside the integrity-checked source cache. */
+export declare function savePluginInformation(root: string, reference: string): string;
+export declare function installPackage(root: string, reference: string, options?: {
+    fromGit?: string;
+    update?: boolean;
+    frozen?: boolean;
+    tagPrefix?: string;
+}): {
+    repository: string;
+    path: string;
+    commit: string;
+    files: Record<string, string>;
+    reference: string;
+    cached: boolean;
+    informationFile: string;
+};
+export declare function installedProviders(root: string): {
+    apiVersion: "ingestron.provider/v1";
+    id: string;
+    version: string;
+    platform: string;
+    activities: Record<string, string>;
+    capabilities?: {
+        artifactKinds: string[];
+        delivery: boolean;
+        handovers: ("files" | "relation")[];
+        packContracts: Record<string, {
+            version: string;
+            activities: string[];
+        }>;
+    } | undefined;
+    projectAssembly?: {
+        apiVersion: "ingestron.project-assembly/v1";
+        command: string;
+    } | undefined;
+    connectorRuntimes?: Record<string, {
+        selectionSchema: Record<string, unknown>;
+        executionSchema: Record<string, unknown>;
+        prepareCommand: string;
+        contractsCommand?: string | undefined;
+        executionPlatforms: string[];
+    }> | undefined;
+    configurationSchema?: string | Record<string, unknown> | undefined;
+    standards?: {
+        id: string;
+        description: string;
+        sourceKinds?: string[] | undefined;
+        dataFlow?: "forbidden" | "required" | undefined;
+    }[] | undefined;
+    compatibility?: {
+        plan: "ingestron.plan/v1";
+        requiredFeatures: string[];
+        minimumCli: string;
+    } | undefined;
+    outputSchemas?: Record<string, string> | undefined;
+    lifecycle?: {
+        apiVersion: "ingestron.provider-lifecycle/v1";
+        module: string;
+    } | undefined;
+    planner?: {
+        apiVersion: "ingestron.provider-planning/v1";
+        module: string;
+    } | undefined;
+    commands?: {
+        apiVersion: "ingestron.provider-commands/v1";
+        namespace?: string | undefined;
+        execution: "offline-json";
+        module: string;
+        definitions: {
+            name: string;
+            description: string;
+            reportPack?: boolean | undefined;
+            examples?: string[] | undefined;
+            resultSchema?: Record<string, unknown> | undefined;
+            inputSchema: Record<string, unknown>;
+        }[];
+    } | undefined;
+    execution?: {
+        apiVersion: "ingestron.execution/v1";
+        transport: string;
+        entryPoint: string;
+        actions: ("approve" | "discover" | "prepare" | "review" | "run")[];
+        status: "receipt";
+        retry: "same-run-id";
+        cancellation: "interrupt";
+    } | undefined;
+    renderer?: {
+        apiVersion: "ingestron.provider-generation/v1";
+        module: string;
+    } | undefined;
+    reference: string;
+    commit: string;
+    evidence: string;
+}[];
+export declare function installedActivities(root: string): {
+    id: string;
+    provider: string;
+    version: string;
+    description: string;
+    platforms: string[];
+    evidence: string;
+    implementations: {
+        apiVersion: "ingestron.activity/v1";
+        id: string;
+        version: string;
+        description: string;
+        platform: string;
+        input: "committed" | "files" | "relation" | "source";
+        output: "committed" | "files" | "relation";
+        effect: "commit" | "control" | "read" | "stage";
+        capabilities: string[];
+        optionsSchema: Record<string, unknown>;
+        generator?: {
+            kind: "native-notebook" | "native-pipeline" | "native-project";
+            artifactKind?: string | undefined;
+            execution?: string | undefined;
+            persistence?: "capture" | "durable" | "ephemeral" | undefined;
+            runtime?: string | undefined;
+            code?: string | undefined;
+            scope: "collection" | "table";
+            modes: ("collection" | "table")[];
+            groupBy: string[];
+            dependencyOptions: Record<string, string>;
+            entrypoint: string;
+        } | undefined;
+        requirements?: {
+            execution: string;
+            features: string[];
+            sourceKinds?: string[] | undefined;
+            evidence: "offline";
+        } | undefined;
+        templates: {
+            source: string;
+            output: string;
+            format: "json" | "python" | "sql" | "text" | "yaml";
+        }[];
+        dependencies: Record<string, string>;
+    }[];
+}[];