@capybara-hq/kernel 0.1.0-rc.2 → 0.1.0-rc.21

package/dist/extensions/blobs.js

@@ -9,10 +9,13 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
 };
 import { createHash } from "node:crypto";
 import { Inject, Injectable } from "@nestjs/common";
+import { ClientBuilds } from "../client/client-builds.service.js";
+import { CLIENT_DIR } from "../client/client-files.js";
 import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
 import { Transactional } from "../infra/database/transactional.decorator.js";
+import { unpackPackageTgz } from "../packages/tgz.js";
 import { StorageService } from "../storage/storage.service.js";
 /**
  * Content-addressed extension package store (C13): one copy per sha256 of the whole tgz.
@@ -21,6 +24,13 @@ import { StorageService } from "../storage/storage.service.js";
  *   extension, F-Q): PG `extension_blobs` (`blob_store = pg`, also limited by `packages.maxBytes`).
  * Each package's location is recorded in `extension_packages.blob_store`; reads follow the record and
  * check the sha256.
+ *
+ * A package with an `extension.client` section also stores its client build (`ClientBuilds`,
+ * client extensions spec §9.1): the client files are read again from the bytes being stored,
+ * checked against the section and written to object storage before the package row's transaction,
+ * and the build row is inserted in that transaction, all under the build's lock. Without object
+ * storage (`ClientBuilds`' own, which the CLI points at its store) such a package is refused
+ * (`storage.unavailable`), so a stored build always has its files.
  */
 /** 取到的包与 sha256 不符（存储里的字节被篡改或损坏）。 */
 export class ExtensionBlobIntegrityError extends Error {
@@ -39,6 +49,7 @@ let ExtensionBlobs = class ExtensionBlobs {
     txHost;
     storage;
     config;
+    clientBuilds;
     /**
      * CLI only: packages the CLI process imported itself and reads back before the store, since such
      * a package may be recorded in an object store the CLI cannot read. Each entry's sha256 was
@@ -48,11 +59,12 @@ let ExtensionBlobs = class ExtensionBlobs {
     local = new Map();
     constructor(db, txHost, 
     /** Without an object-store extension (the CLI) packages go to PG. */
-    storage, config) {
+    storage, config, clientBuilds) {
         this.db = db;
         this.txHost = txHost;
         this.storage = storage;
         this.config = config;
+        this.clientBuilds = clientBuilds;
     }
     /** `packages.maxBytes` (read at every call). */
     maxBytes() {
@@ -85,16 +97,38 @@ let ExtensionBlobs = class ExtensionBlobs {
         if (createHash("sha256").update(bytes).digest("hex") !== p.sha256) {
             throw new ExtensionBlobIntegrityError(p.sha256);
         }
+        const client = p.section.client;
+        if (client === undefined) {
+            await this.store(p, bytes, src, undefined);
+            return;
+        }
+        // client code is delivered from object storage: refused before anything is written without it
+        this.clientBuilds.assertAvailable();
+        const files = await this.clientFiles(bytes);
+        // one store or delete of this package's client build at a time (ClientBuilds header)
+        await this.clientBuilds.locked(p.sha256, () => this.store(p, bytes, src, { section: client, files }));
+    }
+    /** {@link put} after its checks; `client` for a package with client code (under its lock). */
+    async store(p, bytes, src, client) {
+        const plugin = p.section.manifest.name;
         const existing = await this.db
             .selectFrom("extension_packages")
             .select("blob_store")
             .where("sha256", "=", p.sha256)
             .executeTakeFirst();
         if (existing) {
-            // 记录已在：字节若丢了（对象被删、库被清）就按原存放位置补上
+            // The row is there: bytes that went missing (an object deleted, a database cleared) are
+            // restored where the row says, and so are the client build's files and row
             await this.repair(p.sha256, existing.blob_store, bytes);
+            if (client !== undefined) {
+                await this.clientBuilds.extract(p.sha256, plugin, client.files, client.section);
+            }
             return;
         }
+        // checked and written before the transaction (no object storage call inside one)
+        const build = client !== undefined
+            ? await this.clientBuilds.writeFiles(p.sha256, plugin, client.files, client.section)
+            : undefined;
         const store = src.store ?? (this.storage.available ? "object" : "pg");
         if (store === "object") {
             if (!this.storage.available)
@@ -106,9 +140,14 @@ let ExtensionBlobs = class ExtensionBlobs {
         }
         // the row stores the plugin's extension section as reviewed (without the name / version copy)
         const { name: _name, version: _version, ...section } = p.section;
-        await this.record(p, section, bytes, src, store);
+        await this.record(p, section, bytes, src, store, build);
+    }
+    /** The files under `dist/client/` of a package's bytes (sha256 already checked by the caller). */
+    async clientFiles(bytes) {
+        const files = await unpackPackageTgz(bytes, { maxBytes: this.maxBytes() });
+        return new Map(files.filter((f) => f.path.startsWith(CLIENT_DIR)).map((f) => [f.path, f.data]));
     }
-    async record(p, section, bytes, src, store) {
+    async record(p, section, bytes, src, store, build) {
         const m = p.section.manifest;
         const inserted = await this.db
             .insertInto("extension_packages")
@@ -123,6 +162,7 @@ let ExtensionBlobs = class ExtensionBlobs {
             source: src.source,
             source_ref: src.ref,
             uploaded_by: src.actor,
+            display_name: p.displayName === undefined ? null : JSON.stringify(p.displayName),
         })
             .onConflict((oc) => oc.column("sha256").doNothing())
             .returning("sha256")
@@ -134,6 +174,9 @@ let ExtensionBlobs = class ExtensionBlobs {
                 .onConflict((oc) => oc.column("sha256").doNothing())
                 .execute();
         }
+        // the build row commits with the package row it references (or after another replica's)
+        if (build !== undefined)
+            await this.clientBuilds.insert(build);
     }
     /** 已有记录的包：按记录的存放位置检查字节在不在，不在就写回（调用方已校验 sha256）。 */
     async repair(sha256, store, bytes) {
@@ -204,7 +247,8 @@ ExtensionBlobs = __decorate([
     __param(0, Inject(KernelDb)),
     __param(1, Inject(TransactionHost)),
     __param(2, Inject(StorageService)),
-    __param(3, Inject(KernelConfigService))
+    __param(3, Inject(KernelConfigService)),
+    __param(4, Inject(ClientBuilds))
 ], ExtensionBlobs);
 export { ExtensionBlobs };
 async function readCapped(body, max) {

package/dist/extensions/bootstrap.d.ts

@@ -1,6 +1,7 @@
-import { type BootstrapManifest, type Logger } from "@capybara-hq/plugin-sdk";
+import { type BootstrapEntry, type BootstrapManifest, type Logger } from "@capybara-hq/plugin-sdk";
 import { type PluginImport } from "../catalog/plugin-import.js";
 import type { KernelConfig } from "../config/schema.js";
+import { ApiError } from "../errors.js";
 import { KernelConfigService } from "../infra/config/kernel-config.service.js";
 import { KernelDb } from "../infra/database/kernel-db.js";
 import { TransactionHost } from "../infra/database/transaction-host.js";
@@ -11,6 +12,7 @@ import { ExtensionAdminOps } from "./admin-ops.js";
 import { ExtensionInstalls } from "./installs.js";
 import { PluginApproveLock } from "./locks.js";
 import { ExtensionRegistry } from "./registry.js";
+import type { SingleProviderNote } from "./single-provider.js";
 /**
  * The bootstrap step (package ecosystem spec §7, startup bootstrap spec): installs the plugins
  * pinned by the bootstrap manifest (`packages/kernel/bootstrap.json`). It replaced the builtin
@@ -34,9 +36,15 @@ import { ExtensionRegistry } from "./registry.js";
  *   until expiry) aborts the run at its next step. "Done" is a `bootstrap_runs` row for the
  *   manifest's sha256 (written with the audit `bootstrap.completed` only when every entry
  *   succeeded; otherwise the next start runs it again, which is idempotent).
+ * - Two passes per start: the first runs before the extensions load (no object store yet), the
+ *   second after them. A `builtin` entry (not required, but builtin in the catalog) is deferred by
+ *   the first pass (`deferBuiltin`: skipped, not a failure; the run is then not recorded and the
+ *   start runs the second pass), so it may ship skills and templates. A required entry is imported
+ *   by the first pass and must succeed; a builtin one that fails is logged like an optional one.
  * - Only entries new to the manifest are installed (Ruling 5): an optional entry that the last
- *   completed run already handled and that is not installed now (an enterprise admin uninstalled
- *   it) stays uninstalled -> `kept`. A required entry is always installed.
+ *   completed run already handled and that is not installed now (a super admin uninstalled
+ *   it) stays uninstalled -> `kept`. A required or builtin entry is always installed (a
+ *   declarative one imported again when its catalog item is gone).
  * - Per entry, by the installed state of its extension:
  *   - not installed (and new, or required): imported into the catalog (source `bootstrap`, ref
  *     `bootstrap:<name>@<version>`, like a marketplace import), installed, approved (snapshot
@@ -44,8 +52,8 @@ import { ExtensionRegistry } from "./registry.js";
  *     and enabled -> `installed`;
  *   - installed from `bootstrap` at a lower version: upgraded the same way (core's release vouched
  *     for the version, so new permissions are pre-approved and audited), keeping the desired state
- *     an enterprise admin chose (a disabled optional plugin stays disabled) -> `upgraded`;
- *   - installed at a higher version, or from another source (an enterprise admin upgraded it), or
+ *     a super admin chose (a disabled optional plugin stays disabled) -> `upgraded`;
+ *   - installed at a higher version, or from another source (a super admin upgraded it), or
  *     the name is config-declared (`declared`: config wins, no install record is created):
  *     untouched, never downgraded -> `kept`;
  *   - a declarative plugin (no extension): imported into the catalog only -> `imported` (`kept`
@@ -61,7 +69,11 @@ import { ExtensionRegistry } from "./registry.js";
  * A failed entry does not stop the others; the run then throws {@link BootstrapFailedError} naming
  * them. The packages are kept in PG (`blob_store = 'pg'`), so installing an extension never needs
  * the object store (object-store-s3 is itself a bootstrap plugin) and the directory and npm are
- * never needed again; a declarative plugin's catalog files do need the object store.
+ * never needed again. A plugin's icons and setup guide files are kept in PG too (`catalog_files`),
+ * so they never need it either; its skills, connector templates and agent templates do (an entry
+ * shipping them fails while no object store runs; an optional one is retried after the extensions
+ * loaded, and a plugin that must be builtin and ships them is a `builtin` entry, which only the
+ * pass after the extensions load imports).
  */
 /** The bootstrap manifest that ships with the kernel package. */
 export declare const BOOTSTRAP_MANIFEST_FILE: string;
@@ -71,6 +83,11 @@ export interface BootstrapEntryResult {
     action: "installed" | "upgraded" | "kept" | "imported" | "required_cleared";
     /** The version installed / imported now, or the version kept. */
     version: string;
+    /**
+     * An optional entry installed (or upgraded) but left disabled: another enabled extension holds
+     * a single-provider extension point it provides. (A required entry fails instead.)
+     */
+    leftDisabled?: SingleProviderNote;
 }
 export interface BootstrapResult {
     /** sha256 (hex) of the manifest ({@link bootstrapManifestSha256}). */
@@ -84,15 +101,23 @@ export interface BootstrapFailure {
     code: string;
     message: string;
 }
-/** Some entries failed (the others were done); the run is not recorded as done. */
+/**
+ * Some entries failed or were deferred (the others were done); the run is not recorded as done.
+ */
 export declare class BootstrapFailedError extends Error {
     readonly failures: BootstrapFailure[];
     readonly results: BootstrapEntryResult[];
+    readonly deferred: string[];
+    readonly packages: ReadonlyMap<string, Uint8Array>;
     /**
      * @param results what the entries that did not fail did (pass them as `carry` to a later run in
      * the same start, so its record keeps `installed` instead of `kept`).
+     * @param deferred the builtin entries a first pass skipped (`deferBuiltin`): not failures, the
+     * later pass of the same start imports them.
+     * @param packages the packages this run obtained, by entry name (pass them as `packages` to the
+     * later pass of the same start, so nothing is downloaded twice).
      */
-    constructor(failures: BootstrapFailure[], results?: BootstrapEntryResult[]);
+    constructor(failures: BootstrapFailure[], results?: BootstrapEntryResult[], deferred?: string[], packages?: ReadonlyMap<string, Uint8Array>);
     /** A required entry failed: the kernel must not serve without it. */
     get requiredFailed(): boolean;
 }
@@ -112,13 +137,21 @@ export interface BootstrapResolveOptions {
     log?: Pick<Logger, "warn">;
     /** The npm download (default {@link fetchNpmPackage}; tests replace it). */
     fetch?: typeof fetchNpmPackage;
+    /**
+     * Entries whose package failing to be obtained is that entry's failure only: the error goes into
+     * `failed` (by entry name) and the other entries are still obtained. Every other entry's failure
+     * throws, so nothing is written.
+     */
+    soft?: (e: BootstrapEntry) => boolean;
+    failed?: Map<string, ApiError>;
 }
 /**
  * Obtains every entry's tgz, checked against its pinned sha512: from the offline directory when a
  * file there has that sha512, else (a `source: "npm"` entry) downloaded from the npm registry and
  * verified against the pin (the packument's `dist.integrity` is not consulted). Throws an
  * {@link ApiError} naming the plugin for the first entry it cannot obtain
- * (`bootstrap.missing_file`, `bootstrap.download_failed`, `bootstrap.integrity_mismatch`).
+ * (`bootstrap.missing_file`, `bootstrap.download_failed`, `bootstrap.integrity_mismatch`), except
+ * for the `soft` entries (recorded in `failed`, left out of the result).
  */
 export declare function resolveBootstrapPackages(manifest: BootstrapManifest, o: BootstrapResolveOptions): Promise<Map<string, Uint8Array>>;
 /** The lock around a whole run (`lock:bootstrap`). */
@@ -172,9 +205,26 @@ export declare class BootstrapService {
         dir?: string;
         signal?: AbortSignal;
         carry?: readonly BootstrapEntryResult[];
+        /**
+         * The first pass of a start (before the extensions load, so no object store): builtin entries
+         * are skipped, and the run throws {@link BootstrapFailedError} with them as `deferred` instead
+         * of being recorded as done.
+         */
+        deferBuiltin?: boolean;
+        /**
+         * Packages an earlier pass of the same start obtained ({@link BootstrapFailedError.packages});
+         * one whose sha512 still equals its entry's pin is used instead of obtaining it again.
+         */
+        packages?: ReadonlyMap<string, Uint8Array>;
         /** The plugin import a declarative entry (or an entry's catalog version) goes through. */
         imports: BootstrapImports;
     }): Promise<BootstrapResult>;
+    /**
+     * A pin whose plugin name an org holds (a live org-owned catalog item): the bootstrap takes no
+     * name over (ruling P3-28), so the entry fails; this warning tells the super admin why and
+     * which org holds it (final review M4).
+     */
+    private warnIfHeld;
     /** The `bootstrap_runs` row and the `bootstrap.completed` audit, in one transaction. */
     private recordDone;
     /**
@@ -186,16 +236,41 @@ export declare class BootstrapService {
     /** The entry names the last completed run handled (undefined before any run completed). */
     private lastRunNames;
     private entry;
+    /** Whether a live (not deleted) platform catalog item has this name. */
+    private hasLiveItem;
+    /**
+     * The catalog item of a bootstrap entry follows `required` / `builtin`: such an entry is builtin,
+     * offered to every org (scope `all`, which an org admin cannot switch off) and not disabled
+     * (migration 0075 could leave a required plugin disabled); an entry that stops being required or
+     * builtin is no longer builtin and keeps scope `all` (the admin may change it from then on).
+     * Only the platform item: an org-owned item of the same name (the entry then fails with
+     * `plugin.name_taken`) is never touched.
+     */
+    private markBuiltin;
     /** A declarative plugin: catalog only. Returns false when that version is already imported. */
     private importOnly;
     /**
-     * Imports, approves, installs and enables (one review, confirmed with `deployExtension`). When
+     * Imports, approves, installs and enables (one review; its confirm deploys the extension). When
      * the catalog already has this version (an earlier run stopped after importing, or the same
      * package came from a marketplace), the extension is deployed from the package store instead.
      */
     private deploy;
     /** The extension of a catalog version already imported: ingest, approve, install (, enable). */
     private deployFromStore;
+    /**
+     * Enables a bootstrap extension unless another enabled extension provides one of its
+     * single-provider extension points (see {@link leftDisabled}).
+     */
+    private enableOrWarn;
+    /**
+     * A bootstrap extension left disabled because another enabled extension holds a single-provider
+     * extension point it provides. An optional preinstall (a sandbox while the super admin runs
+     * another one) stays disabled with a warning, and the entry's result says so. A required entry
+     * fails (thrown, so the run is not recorded done, `capybara bootstrap` exits 1 and
+     * `capybara start` refuses to serve): the deployment cannot run without it, and only the super
+     * admin can free the point, by disabling the other provider and running the bootstrap again.
+     */
+    private leftDisabled;
     /** The permission snapshot of a stored package (source `bootstrap`), under the approve lock. */
     private approveStored;
     /** Install records still `required` whose entry left the manifest lose `required`. */