@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.0

package/dist/catalog/cli.d.ts

@@ -0,0 +1,46 @@
+import { type ObjectStore } from "@capybara-hq/plugin-sdk";
+import type { KernelConfig } from "../config/schema.js";
+import { type ExtensionCli } from "../extensions/cli.js";
+import type { KernelLogger } from "../log.js";
+import { CatalogService } from "./catalog.service.js";
+import { PluginImport } from "./plugin-import.js";
+/** {@link openCatalogCli}'s result: the extension CLI context plus the catalog services. */
+export interface CatalogCli extends ExtensionCli {
+    catalog: CatalogService;
+    /** Plugin import (prepare / confirm). The CLI acts without an account (actor `null`). */
+    imports: PluginImport;
+}
+/** What the CLI can do with the deployment's object store, and why not when it cannot. */
+export type CliObjectStore = {
+    store: ObjectStore;
+} | {
+    store: undefined;
+    reason: string;
+};
+/**
+ * Sets up the deployment's object-store extension inside the CLI process. The CLI hosts no
+ * extensions, so this runs only the provider's `setup` with a context that offers `config` and
+ * `provide` (any other ctx access fails). It uses a provider only when replicas would load it:
+ * - a config-declared one: its permissions are approved and its deployment config is valid;
+ * - else an install record desired enabled (e.g. the bootstrap plugin object-store-s3) whose
+ *   reviewed manifest provides `object-store@1` and whose name no config-declared extension takes:
+ *   a package with migrations has had them run for its sha256 (`migrated_sha`; the CLI never
+ *   migrates), its package's permissions are covered by the approved snapshot (or `autoApprove`),
+ *   the package syncs into `syncDir` (checked against its sha256), its module matches the reviewed
+ *   manifest, and its deployment config is valid.
+ * Config-declared providers are tried first, then install records by name.
+ */
+export declare function loadCliObjectStore(cli: Pick<ExtensionCli, "inspected" | "deploymentConfigs" | "deploymentConfigOf" | "approvals" | "config" | "configIssues" | "installs" | "packageSet" | "readPackage">, log: Pick<KernelLogger, "warn">, syncDir: string): Promise<CliObjectStore>;
+/**
+ * The CLI's plugin catalog context (`capybara plugins *`): {@link openExtensionCli} plus
+ * {@link PluginImport} and {@link CatalogService}. Imports run with actor `null` (the operator of the
+ * deployment, like `capybara extensions install`: no enterprise-admin re-check, audits without an
+ * actor, permission snapshots written with source `cli`). Files of the plugin (skills, connector
+ * templates, icons) go to the deployment's object store, set up in-process from the declared or
+ * installed object-store extension (see {@link loadCliObjectStore}; an installed package is synced
+ * into a temporary directory removed on `close`); connector templates of connectors the
+ * plugin does not bring are checked statically only (the CLI loads no connector).
+ */
+export declare function openCatalogCli(config: KernelConfig, log: KernelLogger, opts?: {
+    resolveFrom?: string;
+}): Promise<CatalogCli>;

package/dist/catalog/cli.js

@@ -0,0 +1,211 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { OBJECT_STORE_EXTENSION_POINT, } from "@capybara-hq/plugin-sdk";
+import { ApiError } from "../errors.js";
+import { planApprovals, snapshotCovers } from "../extensions/approvals.js";
+import { openExtensionCli } from "../extensions/cli.js";
+import { assertSecretsReachable } from "../extensions/host.js";
+import { importExtensionModule, matchesSection } from "../extensions/instance.js";
+import { packageRootOf, syncPackage } from "../extensions/sync.js";
+import { packagePath } from "../packages/tgz.js";
+import { StorageService } from "../storage/storage.service.js";
+import { CatalogService } from "./catalog.service.js";
+import { PluginImport } from "./plugin-import.js";
+/**
+ * Sets up the deployment's object-store extension inside the CLI process. The CLI hosts no
+ * extensions, so this runs only the provider's `setup` with a context that offers `config` and
+ * `provide` (any other ctx access fails). It uses a provider only when replicas would load it:
+ * - a config-declared one: its permissions are approved and its deployment config is valid;
+ * - else an install record desired enabled (e.g. the bootstrap plugin object-store-s3) whose
+ *   reviewed manifest provides `object-store@1` and whose name no config-declared extension takes:
+ *   a package with migrations has had them run for its sha256 (`migrated_sha`; the CLI never
+ *   migrates), its package's permissions are covered by the approved snapshot (or `autoApprove`),
+ *   the package syncs into `syncDir` (checked against its sha256), its module matches the reviewed
+ *   manifest, and its deployment config is valid.
+ * Config-declared providers are tried first, then install records by name.
+ */
+export async function loadCliObjectStore(cli, log, syncDir) {
+    const providers = cli.inspected.filter((p) => p.manifest.provides.includes(OBJECT_STORE_EXTENSION_POINT));
+    const declared = new Set(cli.inspected.map((p) => p.manifest.name));
+    const installed = (await cli.installs.desired()).filter((d) => !declared.has(d.name) &&
+        (d.section.manifest?.provides ?? []).includes(OBJECT_STORE_EXTENSION_POINT));
+    if (providers.length === 0 && installed.length === 0) {
+        return { store: undefined, reason: "no object-store extension is declared or installed" };
+    }
+    const autoApprove = cli.config.extensionPermissions.autoApprove;
+    const sets = new Map(cli.inspected.map((p) => [p.manifest.name, p.permissions]));
+    const snapshots = await cli.approvals.all();
+    const decisions = planApprovals(cli.inspected.map((p) => p.manifest), sets, snapshots, { autoApprove });
+    const reasons = [];
+    /** Runs `setup` with the restricted context; the store it provided, or undefined (reason noted). */
+    const setUp = async (manifest, config) => {
+        const name = manifest.name;
+        let store;
+        const ctx = new Proxy({
+            config,
+            provide(point, impl) {
+                if (point === OBJECT_STORE_EXTENSION_POINT)
+                    store = impl;
+            },
+        }, {
+            get(target, prop, receiver) {
+                if (prop in target)
+                    return Reflect.get(target, prop, receiver);
+                throw new Error(`the CLI does not support ctx.${String(prop)}`);
+            },
+        });
+        try {
+            // biome-ignore lint/suspicious/noExplicitAny: a restricted context stands in for the full one
+            await manifest.setup(ctx);
+        }
+        catch (err) {
+            log.warn("the CLI cannot set up the object store extension", { plugin: name, err });
+            reasons.push(`${name} needs more than ctx.config and ctx.provide`);
+            return undefined;
+        }
+        if (!store)
+            reasons.push(`${name} provided no object store`);
+        return store;
+    };
+    for (const provider of providers) {
+        const name = provider.manifest.name;
+        const decision = decisions.find((d) => d.plugin === name);
+        if (decision?.action === "pending") {
+            reasons.push(`${name} awaits permission approval`);
+            continue;
+        }
+        const hasConfig = provider.manifest.config?.deployment !== undefined;
+        if (cli.configIssues.has(name) || (hasConfig && !cli.deploymentConfigs.has(name))) {
+            reasons.push(`${name} is not configured (pending deployment config)`);
+            continue;
+        }
+        const store = await setUp(provider.manifest, cli.deploymentConfigs.get(name) ?? {});
+        if (store)
+            return { store };
+    }
+    for (const d of installed) {
+        // Replicas set it up only after its migrations ran (ensureMigrated records migrated_sha); the
+        // CLI never migrates, so a package whose migrations have not run for this sha is skipped
+        if (d.section.migrations !== undefined && d.migratedSha !== d.sha256) {
+            reasons.push(`${d.name} awaits its migrations (a replica runs them when it loads it)`);
+            continue;
+        }
+        let set;
+        try {
+            set = await cli.packageSet(d.sha256);
+        }
+        catch (err) {
+            log.warn("the CLI cannot read an installed object store package", { plugin: d.name, err });
+            reasons.push(`${d.name}: its package cannot be read`);
+            continue;
+        }
+        if (!autoApprove && !snapshotCovers(snapshots.get(d.name), set)) {
+            reasons.push(`${d.name} awaits permission approval`);
+            continue;
+        }
+        let manifest;
+        try {
+            const url = await syncPackage(syncDir, d.name, d.sha256, { get: (sha) => cli.readPackage(sha) }, { maxBytes: cli.config.packages.maxBytes });
+            manifest = await importExtensionModule(url, d.name, packageRootOf(url, packagePath(d.section.entry)));
+            assertSecretsReachable(manifest);
+        }
+        catch (err) {
+            log.warn("the CLI cannot load an installed object store extension", { plugin: d.name, err });
+            reasons.push(`${d.name} cannot be loaded`);
+            continue;
+        }
+        if (!matchesSection(manifest, d.name, d.section.manifest)) {
+            reasons.push(`${d.name}: its module does not match the reviewed manifest`);
+            continue;
+        }
+        const config = await cli.deploymentConfigOf(d.name, manifest.config?.deployment);
+        if (!config.ok) {
+            reasons.push(`${d.name} is not configured (pending deployment config)`);
+            continue;
+        }
+        const store = await setUp(manifest, config.value);
+        if (store)
+            return { store };
+    }
+    return { store: undefined, reason: reasons.join("; ") };
+}
+/** `storage.unavailable` with the CLI's constraint spelled out (the generic text points to a plugin install). */
+class CliStorage extends StorageService {
+    reason;
+    constructor(deps, reason) {
+        super(deps);
+        this.reason = reason;
+    }
+    assertAvailable() {
+        if (this.available)
+            return;
+        throw new ApiError("storage.unavailable", `Object storage is not available to the CLI (${this.reason ?? "unknown"}). The CLI can use only a config-declared or installed and enabled object-store extension that is approved, configured and whose setup needs only ctx.config and ctx.provide; import this plugin in the admin console instead`);
+    }
+}
+/**
+ * The CLI's plugin catalog context (`capybara plugins *`): {@link openExtensionCli} plus
+ * {@link PluginImport} and {@link CatalogService}. Imports run with actor `null` (the operator of the
+ * deployment, like `capybara extensions install`: no enterprise-admin re-check, audits without an
+ * actor, permission snapshots written with source `cli`). Files of the plugin (skills, connector
+ * templates, icons) go to the deployment's object store, set up in-process from the declared or
+ * installed object-store extension (see {@link loadCliObjectStore}; an installed package is synced
+ * into a temporary directory removed on `close`); connector templates of connectors the
+ * plugin does not bring are checked statically only (the CLI loads no connector).
+ */
+export async function openCatalogCli(config, log, opts = {}) {
+    const cli = await openExtensionCli(config, log, opts);
+    let syncDir;
+    const removeSyncDir = async () => {
+        if (syncDir !== undefined)
+            await rm(syncDir, { recursive: true, force: true }).catch(() => { });
+    };
+    try {
+        syncDir = await mkdtemp(join(tmpdir(), "capybara-cli-"));
+        const { store, ...rest } = await loadCliObjectStore(cli, log, syncDir);
+        const storage = new CliStorage({
+            provider: () => store,
+            plugins: async () => cli.inspected.map((p) => p.manifest.name),
+        }, "reason" in rest ? rest.reason : undefined);
+        const declared = new Map(cli.inspected.map((p) => [p.manifest.name, p.source]));
+        const imports = new PluginImport({
+            db: cli.db,
+            config: () => config,
+            storage,
+            installs: cli.installs,
+            ops: cli.ops,
+            blobs: { get: (sha256) => cli.readPackage(sha256) },
+            declared: () => declared,
+            // The CLI loads no connectors: templates get static checks only (the org install re-checks)
+            findConnector: () => undefined,
+            connectorsLoaded: false,
+            log,
+        });
+        const catalog = new CatalogService({
+            db: cli.db,
+            config: () => config,
+            installs: cli.installs,
+            ops: cli.ops,
+            pluginImport: imports,
+            // Unreachable from the CLI: only marketplace imports (CatalogService.importFromMarketplace)
+            // download archives, and the CLI offers file, npm and direct git imports only.
+            fetchArchive: () => Promise.reject(new ApiError("internal", "marketplace imports are not available in the CLI")),
+            // Only marketplace imports send a marketplace token; a direct git import has none
+            marketplaceToken: async () => undefined,
+        });
+        return {
+            ...cli,
+            catalog,
+            imports,
+            async close() {
+                await cli.close();
+                await removeSyncDir();
+            },
+        };
+    }
+    catch (e) {
+        await cli.close();
+        await removeSyncDir();
+        throw e;
+    }
+}

package/dist/catalog/git-source.d.ts

@@ -0,0 +1,127 @@
+import { type NetPolicy } from "../net/public-address.js";
+import { type MarketplaceNetCode } from "./marketplace-fetch.js";
+/**
+ * Reading plugins from git (spec §5.1, §5.2; Decision 2): isomorphic-git (pure JS, any smart-HTTP
+ * host) over the kernel's public-address network layer. The repo is hostile input:
+ * - https smart HTTP only (`GET info/refs` + `POST git-upload-pack`), each request through
+ *   `net/safeFetch` (public addresses, pinned connection, no redirects); the token goes only to
+ *   `tokenOrigin`;
+ * - one commit, depth 1, no tags, no submodules (never fetched); the pack is at most `maxBytes`;
+ * - before isomorphic-git indexes the pack (which inflates every object), the pack is scanned with
+ *   bounded inflation: at most {@link GIT_LIMITS}`.maxObjects` objects and 4 × `maxBytes` of declared
+ *   content in all, so a deflate bomb is refused before it is expanded;
+ * - files are read from the object store (`readTree` / `readBlob`), never checked out; symlinks and
+ *   submodule entries under `path` are refused; at most 5000 files and 4 × `maxBytes`; LFS pointer
+ *   files are ordinary files;
+ * - the object store is a temp dir removed after each operation; 120 s per operation.
+ */
+export declare const GIT_LIMITS: {
+    readonly maxFiles: 5000;
+    /** Objects in one pack (a depth-1 commit: the commit, its trees and blobs). */
+    readonly maxObjects: 20000;
+    /** Unpacked bytes (pack content and files read) at most this × `maxBytes`. */
+    readonly unpackedFactor: 4;
+    readonly timeoutMs: 120000;
+};
+/** `./a/b/` -> `a/b`; the root is `.`. */
+export declare const normalGitPath: (path: string | undefined) => string;
+/**
+ * The identity of a git location, `<host>/<repo>:<path>` (host lower case, the repo without a
+ * leading `/` or a trailing `.git`, `path` as {@link normalGitPath}).
+ */
+export declare function gitLocationIdentity(l: {
+    host: string;
+    repo: string;
+    path: string;
+}): string;
+/** {@link gitLocationIdentity} of a repo URL and a path in it. */
+export declare function gitUrlIdentity(url: string, path: string | undefined): string;
+export type GitLocation = {
+    url: string;
+    ref?: string;
+    commit?: string;
+    path?: string;
+};
+export type GitOptions = {
+    net: NetPolicy;
+    token?: string;
+    /** The token is sent only to this origin (the marketplace URL's). */
+    tokenOrigin?: string;
+    signal: AbortSignal;
+    /** `packages.maxBytes`: the largest pack. */
+    maxBytes: number;
+    /** Where the temp object store is created (default the OS temp dir); tests only. */
+    tempRoot?: string;
+};
+export type GitSourceErrorCode = "marketplace.git_failed" | "marketplace.git_too_large" | "marketplace.git_unsupported_entry";
+/**
+ * What a git failure names, for API error details (localized by the web): `host` the host the
+ * network rules refused, `path` the refused tree entry (symlink, submodule, invalid name), `hint`
+ * `git_suffix` when a redirect came from a repo URL not ending in `.git` (GitLab, Gitea redirect it).
+ */
+export interface GitErrorDetail {
+    host?: string;
+    path?: string;
+    hint?: "git_suffix";
+}
+/** A git read failed. English message without the token; `netCode` when the network rules refused. */
+export declare class GitSourceError extends Error {
+    readonly code: GitSourceErrorCode;
+    readonly netCode: MarketplaceNetCode | undefined;
+    readonly detail: GitErrorDetail;
+    constructor(code: GitSourceErrorCode, message: string, options?: {
+        cause?: unknown;
+        netCode?: MarketplaceNetCode;
+        detail?: GitErrorDetail;
+    });
+}
+/**
+ * The details of a git failure's API error: the host (the one the network rules refused, else the
+ * repo's), the repo path, the ref or commit asked for, and the refused entry. Lets the web name them
+ * in localized copy; the English message stays for logs and API users.
+ */
+export declare function gitErrorDetails(e: GitSourceError, loc: Pick<GitLocation, "url" | "ref" | "commit">): Record<string, string>;
+/** `owner/repo` -> `https://github.com/owner/repo.git`; anything else is returned unchanged. */
+export declare function expandGitUrl(input: string): string;
+/**
+ * The commit to read (40 lower-case hex): `commit` when given (it wins over `ref`), otherwise the
+ * head of `ref` (a branch, a tag or a full `refs/…` name) or of the remote HEAD.
+ */
+export declare function resolveGitCommit(loc: GitLocation, o: GitOptions): Promise<string>;
+/**
+ * Walks a pack (v2/v3) with bounded inflation before anything else expands it: object count and the
+ * declared content sizes (objects, and delta results) must fit; an object that inflates beyond its
+ * declared size is refused.
+ */
+export declare function scanPack(pack: Uint8Array, maxUnpacked: number): void;
+/**
+ * The files under `path` at `commit` (paths relative to `path`, `/`-separated, sorted). Throws
+ * {@link GitSourceError}: `marketplace.git_failed` (network, server, missing commit or path),
+ * `marketplace.git_too_large` (limits), `marketplace.git_unsupported_entry` (symlink, submodule).
+ */
+export declare function readGitTree(loc: GitLocation & {
+    commit: string;
+}, o: ReadGitTreeOptions): Promise<Map<string, Uint8Array>>;
+/** Options of {@link readGitTree}. */
+export type ReadGitTreeOptions = GitOptions & {
+    /**
+     * Which entries under `path` to read (`path` relative to it): a tree for which it returns false
+     * is not descended, a blob is not read. Entries left out are not checked either (a symlink
+     * elsewhere in a marketplace repo does not matter to its manifest). Default: everything.
+     */
+    include?: (path: string, type: "tree" | "blob") => boolean;
+};
+/**
+ * {@link readGitTree}, plus the git tree oids of directories at the same commit: `trees` gets the
+ * files read and returns directory paths relative to the repository root (`.` is the root); each
+ * one that exists and is a directory gets its tree oid (equal oids mean equal contents, so a
+ * plugin directory that did not change keeps its oid across commits).
+ */
+export declare function readGitTreeAndOids(loc: GitLocation & {
+    commit: string;
+}, o: ReadGitTreeOptions & {
+    trees?: (files: Map<string, Uint8Array>) => string[];
+}): Promise<{
+    files: Map<string, Uint8Array>;
+    trees: Map<string, string>;
+}>;