@capybara-hq/kernel 0.1.0-rc.0 → 0.1.0-rc.2

package/dist/extensions/blobs.js

@@ -1,4 +1,19 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 import { createHash } from "node:crypto";
+import { Inject, Injectable } from "@nestjs/common";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { Transactional } from "../infra/database/transactional.decorator.js";
+import { StorageService } from "../storage/storage.service.js";
 /**
  * Content-addressed extension package store (C13): one copy per sha256 of the whole tgz.
  * - With object storage: `deployment/plugins/blobs/<sha256>.tgz` (`blob_store = object`);
@@ -17,10 +32,46 @@ export class ExtensionBlobIntegrityError extends Error {
 /** 对象存储里的键（相对 `deployment/`）。 */
 export const blobKey = (sha256) => `plugins/blobs/${sha256}.tgz`;
 const SHA256_RE = /^[0-9a-f]{64}$/;
-export class ExtensionBlobs {
-    d;
-    constructor(d) {
-        this.d = d;
+/** How many packages {@link ExtensionBlobs.remember} keeps; the oldest is dropped beyond it. */
+const REMEMBERED_MAX = 8;
+let ExtensionBlobs = class ExtensionBlobs {
+    db;
+    txHost;
+    storage;
+    config;
+    /**
+     * CLI only: packages the CLI process imported itself and reads back before the store, since such
+     * a package may be recorded in an object store the CLI cannot read. Each entry's sha256 was
+     * checked by {@link remember}; content-addressed and immutable, so it is never stale. Bounded
+     * ({@link REMEMBERED_MAX}, oldest dropped); a kernel replica never fills it.
+     */
+    local = new Map();
+    constructor(db, txHost, 
+    /** Without an object-store extension (the CLI) packages go to PG. */
+    storage, config) {
+        this.db = db;
+        this.txHost = txHost;
+        this.storage = storage;
+        this.config = config;
+    }
+    /** `packages.maxBytes` (read at every call). */
+    maxBytes() {
+        return this.config.get().packages.maxBytes;
+    }
+    /**
+     * CLI only: keeps `bytes` for {@link get} in this process (see {@link local}). Bytes whose sha256
+     * is not `sha256` throw {@link ExtensionBlobIntegrityError} and are not kept.
+     */
+    remember(sha256, bytes) {
+        if (createHash("sha256").update(bytes).digest("hex") !== sha256) {
+            throw new ExtensionBlobIntegrityError(sha256);
+        }
+        this.local.delete(sha256);
+        this.local.set(sha256, bytes);
+        while (this.local.size > REMEMBERED_MAX) {
+            const oldest = this.local.keys().next().value;
+            this.local.delete(oldest);
+        }
     }
     /**
      * Stores a validated package (the result of {@link readExtensionPackage}) and its bytes, and writes
@@ -28,13 +79,13 @@ export class ExtensionBlobs {
      * (content-addressed, idempotent; replicas writing the same package concurrently is harmless).
      */
     async put(p, bytes, src) {
-        if (bytes.byteLength > this.d.maxBytes()) {
-            throw new Error(`the package is larger than ${this.d.maxBytes()} bytes`);
+        if (bytes.byteLength > this.maxBytes()) {
+            throw new Error(`the package is larger than ${this.maxBytes()} bytes`);
         }
         if (createHash("sha256").update(bytes).digest("hex") !== p.sha256) {
             throw new ExtensionBlobIntegrityError(p.sha256);
         }
-        const existing = await this.d.db
+        const existing = await this.db
             .selectFrom("extension_packages")
             .select("blob_store")
             .where("sha256", "=", p.sha256)
@@ -44,58 +95,59 @@ export class ExtensionBlobs {
             await this.repair(p.sha256, existing.blob_store, bytes);
             return;
         }
-        const store = src.store ?? (this.d.storage?.available ? "object" : "pg");
+        const store = src.store ?? (this.storage.available ? "object" : "pg");
         if (store === "object") {
-            if (!this.d.storage)
+            if (!this.storage.available)
                 throw new Error("object storage is not available");
-            await this.d.storage.deployment().put(blobKey(p.sha256), bytes, {
+            await this.storage.deployment().put(blobKey(p.sha256), bytes, {
                 contentType: "application/gzip",
                 size: bytes.byteLength,
             });
         }
-        const m = p.section.manifest;
         // the row stores the plugin's extension section as reviewed (without the name / version copy)
         const { name: _name, version: _version, ...section } = p.section;
-        await this.d.db.transaction().execute(async (trx) => {
-            const inserted = await trx
-                .insertInto("extension_packages")
-                .values({
-                sha256: p.sha256,
-                name: m.name,
-                package_name: String(p.packageJson.name),
-                version: m.version,
-                size: p.size,
-                section: JSON.stringify(section),
-                blob_store: store,
-                source: src.source,
-                source_ref: src.ref,
-                uploaded_by: src.actor,
-            })
+        await this.record(p, section, bytes, src, store);
+    }
+    async record(p, section, bytes, src, store) {
+        const m = p.section.manifest;
+        const inserted = await this.db
+            .insertInto("extension_packages")
+            .values({
+            sha256: p.sha256,
+            name: m.name,
+            package_name: String(p.packageJson.name),
+            version: m.version,
+            size: p.size,
+            section: JSON.stringify(section),
+            blob_store: store,
+            source: src.source,
+            source_ref: src.ref,
+            uploaded_by: src.actor,
+        })
+            .onConflict((oc) => oc.column("sha256").doNothing())
+            .returning("sha256")
+            .executeTakeFirst();
+        if (inserted && store === "pg") {
+            await this.db
+                .insertInto("extension_blobs")
+                .values({ sha256: p.sha256, bytes: Buffer.from(bytes) })
                 .onConflict((oc) => oc.column("sha256").doNothing())
-                .returning("sha256")
-                .executeTakeFirst();
-            if (inserted && store === "pg") {
-                await trx
-                    .insertInto("extension_blobs")
-                    .values({ sha256: p.sha256, bytes: Buffer.from(bytes) })
-                    .onConflict((oc) => oc.column("sha256").doNothing())
-                    .execute();
-            }
-        });
+                .execute();
+        }
     }
     /** 已有记录的包：按记录的存放位置检查字节在不在，不在就写回（调用方已校验 sha256）。 */
     async repair(sha256, store, bytes) {
         if (store === "pg") {
-            await this.d.db
+            await this.db
                 .insertInto("extension_blobs")
                 .values({ sha256, bytes: Buffer.from(bytes) })
                 .onConflict((oc) => oc.column("sha256").doNothing())
                 .execute();
             return;
         }
-        if (!this.d.storage?.available)
+        if (!this.storage.available)
             return;
-        const d = this.d.storage.deployment();
+        const d = this.storage.deployment();
         if ((await d.head(blobKey(sha256))) === null) {
             await d.put(blobKey(sha256), bytes, {
                 contentType: "application/gzip",
@@ -107,7 +159,10 @@ export class ExtensionBlobs {
     async get(sha256) {
         if (!SHA256_RE.test(sha256))
             throw new Error(`invalid package sha256 "${sha256}"`);
-        const row = await this.d.db
+        const local = this.local.get(sha256);
+        if (local)
+            return local;
+        const row = await this.db
             .selectFrom("extension_packages")
             .select("blob_store")
             .where("sha256", "=", sha256)
@@ -116,7 +171,7 @@ export class ExtensionBlobs {
             throw new Error(`plugin package ${sha256} is not in the package store`);
         let bytes;
         if (row.blob_store === "pg") {
-            const b = await this.d.db
+            const b = await this.db
                 .selectFrom("extension_blobs")
                 .select("bytes")
                 .where("sha256", "=", sha256)
@@ -126,21 +181,32 @@ export class ExtensionBlobs {
             bytes = b.bytes;
         }
         else {
-            if (!this.d.storage)
+            if (!this.storage.available)
                 throw new Error("object storage is not available");
-            const got = await this.d.storage.deployment().get(blobKey(sha256));
+            const got = await this.storage.deployment().get(blobKey(sha256));
             if (!got) {
                 // 常见原因：对象存储换了桶 / 地址，旧桶里的包没有带过来
                 throw new Error(`plugin package ${sha256} is recorded in object storage, but deployment/${blobKey(sha256)} is missing there; if the object store's bucket or endpoint was changed, copy the deployment/ prefix from the old bucket, or upload the same package again to restore it`);
             }
-            bytes = await readCapped(got.body, this.d.maxBytes());
+            bytes = await readCapped(got.body, this.maxBytes());
         }
         if (createHash("sha256").update(bytes).digest("hex") !== sha256) {
             throw new ExtensionBlobIntegrityError(sha256);
         }
         return bytes;
     }
-}
+};
+__decorate([
+    Transactional()
+], ExtensionBlobs.prototype, "record", null);
+ExtensionBlobs = __decorate([
+    Injectable(),
+    __param(0, Inject(KernelDb)),
+    __param(1, Inject(TransactionHost)),
+    __param(2, Inject(StorageService)),
+    __param(3, Inject(KernelConfigService))
+], ExtensionBlobs);
+export { ExtensionBlobs };
 async function readCapped(body, max) {
     const reader = body.getReader();
     const chunks = [];

package/dist/extensions/bootstrap.d.ts

@@ -1,28 +1,39 @@
 import { type BootstrapManifest, type Logger } from "@capybara-hq/plugin-sdk";
 import { type PluginImport } from "../catalog/plugin-import.js";
 import type { KernelConfig } from "../config/schema.js";
-import type { KernelDb } from "../db/kernel-schema.js";
-import type { ExtensionAdminOps } from "./admin-ops.js";
-import type { ExtensionInstalls } from "./installs.js";
-import type { ExtensionSource } from "./sources.js";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { LockService } from "../infra/lock/lock.service.js";
+import type { KernelLogger } from "../log.js";
+import { fetchNpmPackage } from "../packages/npm.js";
+import { ExtensionAdminOps } from "./admin-ops.js";
+import { ExtensionInstalls } from "./installs.js";
+import { PluginApproveLock } from "./locks.js";
+import { ExtensionRegistry } from "./registry.js";
 /**
- * The bootstrap step (package ecosystem spec §7): installs the plugins pinned by the bootstrap
- * manifest (`packages/kernel/bootstrap.json`) from the tgz files baked into the image. It replaced
- * the builtin extensions.
+ * The bootstrap step (package ecosystem spec §7, startup bootstrap spec): installs the plugins
+ * pinned by the bootstrap manifest (`packages/kernel/bootstrap.json`). It replaced the builtin
+ * extensions.
  *
- * - Every file (`<dir>/<name>-<version>.tgz`) is checked against the entry's sha512 before anything
- *   is read or written; a missing file is `bootstrap.missing_file`, a mismatch
- *   `bootstrap.integrity_mismatch` (both name the plugin), and then nothing is written. It never
- *   touches the network.
- * - The whole run holds the session advisory lock `capybara.bootstrap` on a connection of its own
- *   (taken with `pg_try_advisory_lock` in a backoff loop that observes the signal, and closed
- *   afterwards, so the lock never outlives the run): when several replicas start together only one
- *   runs it; the others wait, then find it done. "Done" is a `bootstrap_runs` row for the
+ * - When the run is already done for the manifest nothing is read: no directory, no network, so a
+ *   restart of an installed deployment never needs npm.
+ * - Otherwise, inside the lock and before anything is written, every entry's tgz is obtained
+ *   ({@link resolveBootstrapPackages}): a `*.tgz` file of the optional offline directory whose
+ *   sha512 equals the entry's `integrity` (the file name does not matter), else the npm download
+ *   of a `source: "npm"` entry (`packages.npmRegistry` / `npmToken` / `maxBytes`, verified against
+ *   the pin only). A workspace entry without a matching file is `bootstrap.missing_file`, a failed
+ *   download `bootstrap.download_failed`, a download that differs from the pin
+ *   `bootstrap.integrity_mismatch` (all name the plugin), and then nothing is written. Downloading
+ *   inside the lock means a fleet starting together downloads once.
+ * - The whole run holds the Redis lock `lock:bootstrap` (`LockService`, decision R5 of the Nest
+ *   kernel spec: TTL {@link BOOTSTRAP_LOCK_TTL_MS}, renewed every third of it, so a run longer than the TTL keeps
+ *   it and a crashed holder's lock expires within it), taken by an unbounded wait that observes
+ *   the signal: when several replicas (or `capybara bootstrap` and a starting kernel) run it
+ *   together only one runs it; the others wait, then find it done. A lost lock (renewal failed
+ *   until expiry) aborts the run at its next step. "Done" is a `bootstrap_runs` row for the
  *   manifest's sha256 (written with the audit `bootstrap.completed` only when every entry
- *   succeeded; otherwise the next start runs it again, which is idempotent). A lock connection
- *   that drops mid-run is not detected (its errors are swallowed) and the run goes on without the
- *   lock, so a second replica could run concurrently: accepted, since a concurrent import or
- *   install degrades to `catalog.version_exists` and a deploy from the package store.
+ *   succeeded; otherwise the next start runs it again, which is idempotent).
  * - Only entries new to the manifest are installed (Ruling 5): an optional entry that the last
  *   completed run already handled and that is not installed now (an enterprise admin uninstalled
  *   it) stays uninstalled -> `kept`. A required entry is always installed.
@@ -49,8 +60,8 @@ import type { ExtensionSource } from "./sources.js";
  *   loses `required` -> `required_cleared`.
  * A failed entry does not stop the others; the run then throws {@link BootstrapFailedError} naming
  * them. The packages are kept in PG (`blob_store = 'pg'`), so installing an extension never needs
- * the object store (object-store-s3 is itself a bootstrap plugin); a declarative plugin's catalog
- * files do.
+ * the object store (object-store-s3 is itself a bootstrap plugin) and the directory and npm are
+ * never needed again; a declarative plugin's catalog files do need the object store.
  */
 /** The bootstrap manifest that ships with the kernel package. */
 export declare const BOOTSTRAP_MANIFEST_FILE: string;
@@ -89,25 +100,63 @@ export declare class BootstrapFailedError extends Error {
 export declare function bootstrapManifestSha256(manifest: BootstrapManifest): string;
 /** Reads and parses a bootstrap manifest file; an invalid one throws with its problems. */
 export declare function readBootstrapManifest(file?: string): Promise<BootstrapManifest>;
-export interface BootstrapServiceDeps {
-    db: KernelDb;
-    /** The lock connection is opened separately (and closed after every run). */
-    dbUrl: string;
-    connectTimeoutMs?: number;
-    config: () => KernelConfig;
-    imports: Pick<PluginImport, "prepare" | "confirm">;
-    installs: Pick<ExtensionInstalls, "ingest">;
-    ops: Pick<ExtensionAdminOps, "install" | "enable">;
+/** What {@link resolveBootstrapPackages} needs from the kernel config. */
+export interface BootstrapResolveOptions {
     /**
-     * Config-declared extension names (an entry of the same name is kept, config wins, and no
-     * install record is created). The kernel resolves them from `extensions` before loading.
+     * The offline directory: every `*.tgz` directly in it is matched to the entries by sha512.
+     * Omitted or missing: no offline packages.
      */
-    declared?: () => ReadonlySet<string> | ReadonlyMap<string, ExtensionSource>;
-    log: Pick<Logger, "info" | "warn" | "error">;
+    dir?: string;
+    config: Pick<KernelConfig, "packages">;
+    signal?: AbortSignal;
+    log?: Pick<Logger, "warn">;
+    /** The npm download (default {@link fetchNpmPackage}; tests replace it). */
+    fetch?: typeof fetchNpmPackage;
 }
+/**
+ * Obtains every entry's tgz, checked against its pinned sha512: from the offline directory when a
+ * file there has that sha512, else (a `source: "npm"` entry) downloaded from the npm registry and
+ * verified against the pin (the packument's `dist.integrity` is not consulted). Throws an
+ * {@link ApiError} naming the plugin for the first entry it cannot obtain
+ * (`bootstrap.missing_file`, `bootstrap.download_failed`, `bootstrap.integrity_mismatch`).
+ */
+export declare function resolveBootstrapPackages(manifest: BootstrapManifest, o: BootstrapResolveOptions): Promise<Map<string, Uint8Array>>;
+/** The lock around a whole run (`lock:bootstrap`). */
+export declare const BOOTSTRAP_LOCK_KEY = "bootstrap";
+/** Its lifetime without renewal; a run keeps it by renewal, a crashed holder's expires. */
+export declare const BOOTSTRAP_LOCK_TTL_MS = 120000;
+/** Optional provider: the lock's lifetime (ms) instead of {@link BOOTSTRAP_LOCK_TTL_MS} (tests shorten it). */
+export declare const BOOTSTRAP_LOCK_TTL: unique symbol;
+/** What a run imports a declarative entry or an entry's catalog version with. */
+export type BootstrapImports = Pick<PluginImport, "prepare" | "confirm">;
 export declare class BootstrapService {
-    private readonly d;
-    constructor(d: BootstrapServiceDeps);
+    private readonly kdb;
+    readonly txHost: TransactionHost;
+    private readonly locks;
+    private readonly approveLock;
+    private readonly installs;
+    private readonly ops;
+    /**
+     * Config-declared extension names (an entry of the same name is kept, config wins, and no
+     * install record is created): the start sequence records them before the bootstrap step, the
+     * CLI from its inspection.
+     */
+    private readonly registry;
+    private readonly configService;
+    private readonly log;
+    /** The lock's lifetime without renewal. */
+    private readonly lockTtlMs;
+    constructor(kdb: KernelDb, txHost: TransactionHost, locks: LockService, approveLock: PluginApproveLock, installs: ExtensionInstalls, ops: ExtensionAdminOps, 
+    /**
+     * Config-declared extension names (an entry of the same name is kept, config wins, and no
+     * install record is created): the start sequence records them before the bootstrap step, the
+     * CLI from its inspection.
+     */
+    registry: ExtensionRegistry, configService: KernelConfigService, log: KernelLogger, 
+    /** The lock's lifetime without renewal. */
+    lockTtlMs?: number);
+    private get db();
+    private config;
     /** Whether the bootstrap step completed for the manifest with this sha256. */
     isDone(manifestSha256: string): Promise<boolean>;
     /**
@@ -119,20 +168,23 @@ export declare class BootstrapService {
      */
     run(o: {
         manifest: BootstrapManifest;
-        dir: string;
+        /** The offline directory (see {@link resolveBootstrapPackages}); omitted: npm only. */
+        dir?: string;
         signal?: AbortSignal;
         carry?: readonly BootstrapEntryResult[];
+        /** The plugin import a declarative entry (or an entry's catalog version) goes through. */
+        imports: BootstrapImports;
     }): Promise<BootstrapResult>;
+    /** The `bootstrap_runs` row and the `bootstrap.completed` audit, in one transaction. */
+    private recordDone;
     /**
-     * Runs `fn` holding the session advisory lock on a connection of its own: `pg_try_advisory_lock`
-     * retried with a backoff (100 ms doubling to 1 s) that stops when `signal` aborts. The connection
-     * is closed afterwards in every case, which releases the lock even when the unlock failed.
+     * Runs `fn` holding `lock:bootstrap`: waits without bound while another replica holds it
+     * (until `signal` aborts), renews it while `fn` runs and releases it afterwards. `fn` gets a
+     * signal that also aborts when the lock is lost.
      */
     private locked;
     /** The entry names the last completed run handled (undefined before any run completed). */
     private lastRunNames;
-    /** Reads every entry's file and checks its sha512 (before the lock: nothing is written yet). */
-    private verifyFiles;
     private entry;
     /** A declarative plugin: catalog only. Returns false when that version is already imported. */
     private importOnly;
@@ -144,6 +196,8 @@ export declare class BootstrapService {
     private deploy;
     /** The extension of a catalog version already imported: ingest, approve, install (, enable). */
     private deployFromStore;
+    /** The permission snapshot of a stored package (source `bootstrap`), under the approve lock. */
+    private approveStored;
     /** Install records still `required` whose entry left the manifest lose `required`. */
     private clearRemoved;
 }