@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.1

package/dist/extensions/blobs.d.ts

@@ -0,0 +1,45 @@
+import type { KernelDb } from "../db/kernel-schema.js";
+import type { StorageService } from "../storage/storage.service.js";
+import type { ExtensionPackage } from "./package.js";
+/**
+ * Content-addressed extension package store (C13): one copy per sha256 of the whole tgz.
+ * - With object storage: `deployment/plugins/blobs/<sha256>.tgz` (`blob_store = object`);
+ * - without object storage, or when the caller asks (the CLI process loads no object-store
+ *   extension, F-Q): PG `extension_blobs` (`blob_store = pg`, also limited by `packages.maxBytes`).
+ * Each package's location is recorded in `extension_packages.blob_store`; reads follow the record and
+ * check the sha256.
+ */
+/** 取到的包与 sha256 不符（存储里的字节被篡改或损坏）。 */
+export declare class ExtensionBlobIntegrityError extends Error {
+    constructor(sha256: string);
+}
+/** 对象存储里的键（相对 `deployment/`）。 */
+export declare const blobKey: (sha256: string) => string;
+export interface ExtensionBlobSource {
+    source: "npm" | "file" | "catalog" | "bootstrap";
+    ref: string;
+    /** 上传者账号；CLI 为 null。 */
+    actor: string | null;
+    /** 强制存放位置；缺省有对象存储就用它。 */
+    store?: "object" | "pg";
+}
+export declare class ExtensionBlobs {
+    private readonly d;
+    constructor(d: {
+        db: KernelDb;
+        /** 没有时一律存 PG（如 CLI）。 */
+        storage?: StorageService;
+        /** `packages.maxBytes`（每次调用时读）。 */
+        maxBytes: () => number;
+    });
+    /**
+     * Stores a validated package (the result of {@link readExtensionPackage}) and its bytes, and writes
+     * the `extension_packages` row. Does nothing when a package with the same sha256 exists
+     * (content-addressed, idempotent; replicas writing the same package concurrently is harmless).
+     */
+    put(p: ExtensionPackage, bytes: Uint8Array, src: ExtensionBlobSource): Promise<void>;
+    /** 已有记录的包：按记录的存放位置检查字节在不在，不在就写回（调用方已校验 sha256）。 */
+    private repair;
+    /** Reads the package from its recorded location and checks its sha256; a mismatch throws {@link ExtensionBlobIntegrityError}. */
+    get(sha256: string): Promise<Uint8Array>;
+}

package/dist/extensions/blobs.js

@@ -0,0 +1,160 @@
+import { createHash } from "node:crypto";
+/**
+ * Content-addressed extension package store (C13): one copy per sha256 of the whole tgz.
+ * - With object storage: `deployment/plugins/blobs/<sha256>.tgz` (`blob_store = object`);
+ * - without object storage, or when the caller asks (the CLI process loads no object-store
+ *   extension, F-Q): PG `extension_blobs` (`blob_store = pg`, also limited by `packages.maxBytes`).
+ * Each package's location is recorded in `extension_packages.blob_store`; reads follow the record and
+ * check the sha256.
+ */
+/** 取到的包与 sha256 不符（存储里的字节被篡改或损坏）。 */
+export class ExtensionBlobIntegrityError extends Error {
+    constructor(sha256) {
+        super(`plugin package ${sha256}: stored bytes do not match the sha256`);
+        this.name = "ExtensionBlobIntegrityError";
+    }
+}
+/** 对象存储里的键（相对 `deployment/`）。 */
+export const blobKey = (sha256) => `plugins/blobs/${sha256}.tgz`;
+const SHA256_RE = /^[0-9a-f]{64}$/;
+export class ExtensionBlobs {
+    d;
+    constructor(d) {
+        this.d = d;
+    }
+    /**
+     * Stores a validated package (the result of {@link readExtensionPackage}) and its bytes, and writes
+     * the `extension_packages` row. Does nothing when a package with the same sha256 exists
+     * (content-addressed, idempotent; replicas writing the same package concurrently is harmless).
+     */
+    async put(p, bytes, src) {
+        if (bytes.byteLength > this.d.maxBytes()) {
+            throw new Error(`the package is larger than ${this.d.maxBytes()} bytes`);
+        }
+        if (createHash("sha256").update(bytes).digest("hex") !== p.sha256) {
+            throw new ExtensionBlobIntegrityError(p.sha256);
+        }
+        const existing = await this.d.db
+            .selectFrom("extension_packages")
+            .select("blob_store")
+            .where("sha256", "=", p.sha256)
+            .executeTakeFirst();
+        if (existing) {
+            // 记录已在：字节若丢了（对象被删、库被清）就按原存放位置补上
+            await this.repair(p.sha256, existing.blob_store, bytes);
+            return;
+        }
+        const store = src.store ?? (this.d.storage?.available ? "object" : "pg");
+        if (store === "object") {
+            if (!this.d.storage)
+                throw new Error("object storage is not available");
+            await this.d.storage.deployment().put(blobKey(p.sha256), bytes, {
+                contentType: "application/gzip",
+                size: bytes.byteLength,
+            });
+        }
+        const m = p.section.manifest;
+        // the row stores the plugin's extension section as reviewed (without the name / version copy)
+        const { name: _name, version: _version, ...section } = p.section;
+        await this.d.db.transaction().execute(async (trx) => {
+            const inserted = await trx
+                .insertInto("extension_packages")
+                .values({
+                sha256: p.sha256,
+                name: m.name,
+                package_name: String(p.packageJson.name),
+                version: m.version,
+                size: p.size,
+                section: JSON.stringify(section),
+                blob_store: store,
+                source: src.source,
+                source_ref: src.ref,
+                uploaded_by: src.actor,
+            })
+                .onConflict((oc) => oc.column("sha256").doNothing())
+                .returning("sha256")
+                .executeTakeFirst();
+            if (inserted && store === "pg") {
+                await trx
+                    .insertInto("extension_blobs")
+                    .values({ sha256: p.sha256, bytes: Buffer.from(bytes) })
+                    .onConflict((oc) => oc.column("sha256").doNothing())
+                    .execute();
+            }
+        });
+    }
+    /** 已有记录的包：按记录的存放位置检查字节在不在，不在就写回（调用方已校验 sha256）。 */
+    async repair(sha256, store, bytes) {
+        if (store === "pg") {
+            await this.d.db
+                .insertInto("extension_blobs")
+                .values({ sha256, bytes: Buffer.from(bytes) })
+                .onConflict((oc) => oc.column("sha256").doNothing())
+                .execute();
+            return;
+        }
+        if (!this.d.storage?.available)
+            return;
+        const d = this.d.storage.deployment();
+        if ((await d.head(blobKey(sha256))) === null) {
+            await d.put(blobKey(sha256), bytes, {
+                contentType: "application/gzip",
+                size: bytes.byteLength,
+            });
+        }
+    }
+    /** Reads the package from its recorded location and checks its sha256; a mismatch throws {@link ExtensionBlobIntegrityError}. */
+    async get(sha256) {
+        if (!SHA256_RE.test(sha256))
+            throw new Error(`invalid package sha256 "${sha256}"`);
+        const row = await this.d.db
+            .selectFrom("extension_packages")
+            .select("blob_store")
+            .where("sha256", "=", sha256)
+            .executeTakeFirst();
+        if (!row)
+            throw new Error(`plugin package ${sha256} is not in the package store`);
+        let bytes;
+        if (row.blob_store === "pg") {
+            const b = await this.d.db
+                .selectFrom("extension_blobs")
+                .select("bytes")
+                .where("sha256", "=", sha256)
+                .executeTakeFirst();
+            if (!b)
+                throw new Error(`plugin package ${sha256}: bytes are missing from extension_blobs`);
+            bytes = b.bytes;
+        }
+        else {
+            if (!this.d.storage)
+                throw new Error("object storage is not available");
+            const got = await this.d.storage.deployment().get(blobKey(sha256));
+            if (!got) {
+                // 常见原因：对象存储换了桶 / 地址，旧桶里的包没有带过来
+                throw new Error(`plugin package ${sha256} is recorded in object storage, but deployment/${blobKey(sha256)} is missing there; if the object store's bucket or endpoint was changed, copy the deployment/ prefix from the old bucket, or upload the same package again to restore it`);
+            }
+            bytes = await readCapped(got.body, this.d.maxBytes());
+        }
+        if (createHash("sha256").update(bytes).digest("hex") !== sha256) {
+            throw new ExtensionBlobIntegrityError(sha256);
+        }
+        return bytes;
+    }
+}
+async function readCapped(body, max) {
+    const reader = body.getReader();
+    const chunks = [];
+    let total = 0;
+    for (;;) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        total += value.byteLength;
+        if (total > max) {
+            await reader.cancel().catch(() => { });
+            throw new Error(`stored plugin package is larger than ${max} bytes`);
+        }
+        chunks.push(value);
+    }
+    return chunks.length === 1 ? chunks[0] : Buffer.concat(chunks);
+}

package/dist/extensions/bootstrap.d.ts

@@ -0,0 +1,176 @@
+import { type BootstrapManifest, type Logger } from "@capybara-hq/plugin-sdk";
+import { type PluginImport } from "../catalog/plugin-import.js";
+import type { KernelConfig } from "../config/schema.js";
+import type { KernelDb } from "../db/kernel-schema.js";
+import { fetchNpmPackage } from "../packages/npm.js";
+import type { ExtensionAdminOps } from "./admin-ops.js";
+import type { ExtensionInstalls } from "./installs.js";
+import type { ExtensionSource } from "./sources.js";
+/**
+ * The bootstrap step (package ecosystem spec §7, startup bootstrap spec): installs the plugins
+ * pinned by the bootstrap manifest (`packages/kernel/bootstrap.json`). It replaced the builtin
+ * extensions.
+ *
+ * - When the run is already done for the manifest nothing is read: no directory, no network, so a
+ *   restart of an installed deployment never needs npm.
+ * - Otherwise, inside the lock and before anything is written, every entry's tgz is obtained
+ *   ({@link resolveBootstrapPackages}): a `*.tgz` file of the optional offline directory whose
+ *   sha512 equals the entry's `integrity` (the file name does not matter), else the npm download
+ *   of a `source: "npm"` entry (`packages.npmRegistry` / `npmToken` / `maxBytes`, verified against
+ *   the pin only). A workspace entry without a matching file is `bootstrap.missing_file`, a failed
+ *   download `bootstrap.download_failed`, a download that differs from the pin
+ *   `bootstrap.integrity_mismatch` (all name the plugin), and then nothing is written. Downloading
+ *   inside the lock means a fleet starting together downloads once.
+ * - The whole run holds the session advisory lock `capybara.bootstrap` on a connection of its own
+ *   (taken with `pg_try_advisory_lock` in a backoff loop that observes the signal, and closed
+ *   afterwards, so the lock never outlives the run): when several replicas start together only one
+ *   runs it; the others wait, then find it done. "Done" is a `bootstrap_runs` row for the
+ *   manifest's sha256 (written with the audit `bootstrap.completed` only when every entry
+ *   succeeded; otherwise the next start runs it again, which is idempotent). A lock connection
+ *   that drops mid-run is not detected (its errors are swallowed) and the run goes on without the
+ *   lock, so a second replica could run concurrently: accepted, since a concurrent import or
+ *   install degrades to `catalog.version_exists` and a deploy from the package store.
+ * - Only entries new to the manifest are installed (Ruling 5): an optional entry that the last
+ *   completed run already handled and that is not installed now (an enterprise admin uninstalled
+ *   it) stays uninstalled -> `kept`. A required entry is always installed.
+ * - Per entry, by the installed state of its extension:
+ *   - not installed (and new, or required): imported into the catalog (source `bootstrap`, ref
+ *     `bootstrap:<name>@<version>`, like a marketplace import), installed, approved (snapshot
+ *     source `bootstrap`, audited `extension.permissions_approved` with `source: "bootstrap"`)
+ *     and enabled -> `installed`;
+ *   - installed from `bootstrap` at a lower version: upgraded the same way (core's release vouched
+ *     for the version, so new permissions are pre-approved and audited), keeping the desired state
+ *     an enterprise admin chose (a disabled optional plugin stays disabled) -> `upgraded`;
+ *   - installed at a higher version, or from another source (an enterprise admin upgraded it), or
+ *     the name is config-declared (`declared`: config wins, no install record is created):
+ *     untouched, never downgraded -> `kept`;
+ *   - a declarative plugin (no extension): imported into the catalog only -> `imported` (`kept`
+ *     when that version is already there, or when the entry is not new).
+ *   Every extension entry gets `extension_installs.required` from the manifest, and a required one
+ *   is enabled on the deployment.
+ * - Package bytes are content-addressed: when the very same tgz was ingested before with another
+ *   source (e.g. imported from a marketplace), its `extension_packages` row keeps that source and
+ *   so does the install record, which is then not a bootstrap install and never upgraded by a
+ *   later manifest (`kept`, like a plugin the admin installed).
+ * - An install record still `required` whose entry left the manifest is not uninstalled; it only
+ *   loses `required` -> `required_cleared`.
+ * A failed entry does not stop the others; the run then throws {@link BootstrapFailedError} naming
+ * them. The packages are kept in PG (`blob_store = 'pg'`), so installing an extension never needs
+ * the object store (object-store-s3 is itself a bootstrap plugin) and the directory and npm are
+ * never needed again; a declarative plugin's catalog files do need the object store.
+ */
+/** The bootstrap manifest that ships with the kernel package. */
+export declare const BOOTSTRAP_MANIFEST_FILE: string;
+/** What the bootstrap step did with one entry (or one install record that left the manifest). */
+export interface BootstrapEntryResult {
+    name: string;
+    action: "installed" | "upgraded" | "kept" | "imported" | "required_cleared";
+    /** The version installed / imported now, or the version kept. */
+    version: string;
+}
+export interface BootstrapResult {
+    /** sha256 (hex) of the manifest ({@link bootstrapManifestSha256}). */
+    manifestSha256: string;
+    results: BootstrapEntryResult[];
+}
+/** One entry the bootstrap step could not complete. */
+export interface BootstrapFailure {
+    name: string;
+    required: boolean;
+    code: string;
+    message: string;
+}
+/** Some entries failed (the others were done); the run is not recorded as done. */
+export declare class BootstrapFailedError extends Error {
+    readonly failures: BootstrapFailure[];
+    readonly results: BootstrapEntryResult[];
+    /**
+     * @param results what the entries that did not fail did (pass them as `carry` to a later run in
+     * the same start, so its record keeps `installed` instead of `kept`).
+     */
+    constructor(failures: BootstrapFailure[], results?: BootstrapEntryResult[]);
+    /** A required entry failed: the kernel must not serve without it. */
+    get requiredFailed(): boolean;
+}
+/** sha256 (hex) of a manifest: of its JSON text as parsed (key order kept). */
+export declare function bootstrapManifestSha256(manifest: BootstrapManifest): string;
+/** Reads and parses a bootstrap manifest file; an invalid one throws with its problems. */
+export declare function readBootstrapManifest(file?: string): Promise<BootstrapManifest>;
+/** What {@link resolveBootstrapPackages} needs from the kernel config. */
+export interface BootstrapResolveOptions {
+    /**
+     * The offline directory: every `*.tgz` directly in it is matched to the entries by sha512.
+     * Omitted or missing: no offline packages.
+     */
+    dir?: string;
+    config: Pick<KernelConfig, "packages">;
+    signal?: AbortSignal;
+    log?: Pick<Logger, "warn">;
+    /** The npm download (default {@link fetchNpmPackage}; tests replace it). */
+    fetch?: typeof fetchNpmPackage;
+}
+/**
+ * Obtains every entry's tgz, checked against its pinned sha512: from the offline directory when a
+ * file there has that sha512, else (a `source: "npm"` entry) downloaded from the npm registry and
+ * verified against the pin (the packument's `dist.integrity` is not consulted). Throws an
+ * {@link ApiError} naming the plugin for the first entry it cannot obtain
+ * (`bootstrap.missing_file`, `bootstrap.download_failed`, `bootstrap.integrity_mismatch`).
+ */
+export declare function resolveBootstrapPackages(manifest: BootstrapManifest, o: BootstrapResolveOptions): Promise<Map<string, Uint8Array>>;
+export interface BootstrapServiceDeps {
+    db: KernelDb;
+    /** The lock connection is opened separately (and closed after every run). */
+    dbUrl: string;
+    connectTimeoutMs?: number;
+    config: () => KernelConfig;
+    imports: Pick<PluginImport, "prepare" | "confirm">;
+    installs: Pick<ExtensionInstalls, "ingest">;
+    ops: Pick<ExtensionAdminOps, "install" | "enable">;
+    /**
+     * Config-declared extension names (an entry of the same name is kept, config wins, and no
+     * install record is created). The kernel resolves them from `extensions` before loading.
+     */
+    declared?: () => ReadonlySet<string> | ReadonlyMap<string, ExtensionSource>;
+    log: Pick<Logger, "info" | "warn" | "error">;
+}
+export declare class BootstrapService {
+    private readonly d;
+    constructor(d: BootstrapServiceDeps);
+    /** Whether the bootstrap step completed for the manifest with this sha256. */
+    isDone(manifestSha256: string): Promise<boolean>;
+    /**
+     * Runs the bootstrap step (see the file header); when it is already done for this manifest
+     * (also when another replica finished it while this one waited for the lock) returns the
+     * recorded result without doing anything. `carry`: the results of an earlier, failed run in the
+     * same start ({@link BootstrapFailedError.results}); an entry it installed, upgraded or imported
+     * keeps that action instead of `kept` in this run's record.
+     */
+    run(o: {
+        manifest: BootstrapManifest;
+        /** The offline directory (see {@link resolveBootstrapPackages}); omitted: npm only. */
+        dir?: string;
+        signal?: AbortSignal;
+        carry?: readonly BootstrapEntryResult[];
+    }): Promise<BootstrapResult>;
+    /**
+     * Runs `fn` holding the session advisory lock on a connection of its own: `pg_try_advisory_lock`
+     * retried with a backoff (100 ms doubling to 1 s) that stops when `signal` aborts. The connection
+     * is closed afterwards in every case, which releases the lock even when the unlock failed.
+     */
+    private locked;
+    /** The entry names the last completed run handled (undefined before any run completed). */
+    private lastRunNames;
+    private entry;
+    /** A declarative plugin: catalog only. Returns false when that version is already imported. */
+    private importOnly;
+    /**
+     * Imports, approves, installs and enables (one review, confirmed with `deployExtension`). When
+     * the catalog already has this version (an earlier run stopped after importing, or the same
+     * package came from a marketplace), the extension is deployed from the package store instead.
+     */
+    private deploy;
+    /** The extension of a catalog version already imported: ingest, approve, install (, enable). */
+    private deployFromStore;
+    /** Install records still `required` whose entry left the manifest lose `required`. */
+    private clearRemoved;
+}