@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.1

package/dist/extensions/cli.js

@@ -0,0 +1,165 @@
+import { SDK_VERSION } from "@capybara-hq/plugin-sdk";
+import { Redis } from "ioredis";
+import { createBus } from "../ctx/bus.js";
+import { createKeyring } from "../ctx/secrets.js";
+import { kernelDb } from "../db/kernel-schema.js";
+import { createDb } from "../db/pool.js";
+import { ExtensionAdminOps } from "./admin-ops.js";
+import { ExtensionAdmin } from "./admin-view.js";
+import { ExtensionApprovals } from "./approvals.js";
+import { ExtensionBlobs } from "./blobs.js";
+import { mergeDeploymentConfig, readStoredDeploymentConfig, schemaFields, secretPathsOf, } from "./deployment-config.js";
+import { ExtensionHost } from "./host.js";
+import { ExtensionInstalls } from "./installs.js";
+import { PLUGINS_CHANGED_TOPIC } from "./lifecycle.js";
+import { toIssues } from "./org-config.js";
+import { ExtensionPackageError, extensionPackageApiError, permissionSetOfPackage, readExtensionPackage, } from "./package.js";
+/** CLI 发布 bus 消息时连 Redis 的时限（连不上只告警）。 */
+const CLI_REDIS_CONNECT_TIMEOUT_MS = 3_000;
+/**
+ * The CLI's extension management context (C18, F-Q): connects only to PG (and Redis for publishing),
+ * starts no extension host and runs no extension's `setup` (a catalog import sets up the declared
+ * object-store extension itself, see `openCatalogCli`). Packages the CLI imports are always
+ * stored in PG (`blob_store = 'pg'`); replicas read by `blob_store`. Install / enable / disable /
+ * uninstall share {@link ExtensionAdminOps} with the admin console (actor null, audits without an
+ * actor); after writing PG it publishes `kernel.plugins.changed` and running replicas reconcile; a
+ * failed publish only warns (the periodic reconcile catches up).
+ */
+export async function openExtensionCli(config, log, opts = {}) {
+    const inspected = await ExtensionHost.inspect(config.extensions, {
+        ...(opts.resolveFrom !== undefined && { resolveFrom: opts.resolveFrom }),
+    });
+    const raw = createDb(config.db.url, 2, (err) => log.error("pg idle client error", { err }), {
+        connectTimeoutMs: config.db.connectTimeoutMs,
+    });
+    const db = kernelDb(raw);
+    const local = new Map();
+    const stored = new ExtensionBlobs({ db, maxBytes: () => config.packages.maxBytes });
+    const blobs = {
+        get: async (sha256) => local.get(sha256) ?? stored.get(sha256),
+    };
+    const declared = new Map(inspected.map((p) => [p.manifest.name, p.source]));
+    // 部署配置 = 启动配置补上后台保存在 PG 的字段（C10）；不合法的插件内核不加载（待配置）
+    const keyring = createKeyring(Buffer.from(config.masterKey, "base64"), config.masterKeyPrevious.map((k) => Buffer.from(k, "base64")));
+    const deploymentConfigOf = async (name, schema) => {
+        if (!schema)
+            return { ok: true, value: {} };
+        const stored = await readStoredDeploymentConfig(db, keyring, name, {
+            secretPaths: secretPathsOf(schemaFields(schema)),
+        });
+        const r = schema.safeParse(mergeDeploymentConfig(stored, config.extensionConfig[name], schemaFields(schema).map((f) => f.path)));
+        return r.success ? { ok: true, value: r.data } : { ok: false, issues: toIssues(r.error) };
+    };
+    const configIssues = new Map();
+    const deploymentConfigs = new Map();
+    try {
+        for (const p of inspected) {
+            const schema = p.manifest.config?.deployment;
+            if (!schema)
+                continue;
+            const r = await deploymentConfigOf(p.manifest.name, schema);
+            if (r.ok)
+                deploymentConfigs.set(p.manifest.name, r.value);
+            else
+                configIssues.set(p.manifest.name, r.issues);
+        }
+    }
+    catch (e) {
+        await raw.destroy().catch(() => { });
+        throw e;
+    }
+    const declaredInfo = inspected.map((p) => ({
+        name: p.manifest.name,
+        source: p.source,
+        version: p.manifest.version,
+        permissions: p.permissions,
+        configurable: p.manifest.config?.deployment !== undefined,
+        configValid: !configIssues.has(p.manifest.name),
+    }));
+    let redis;
+    const publisher = {
+        async publish(topic, payload) {
+            if (!redis) {
+                redis = new Redis(config.redis.url, {
+                    lazyConnect: true,
+                    enableOfflineQueue: false,
+                    maxRetriesPerRequest: 0,
+                    connectTimeout: CLI_REDIS_CONNECT_TIMEOUT_MS,
+                    retryStrategy: () => null,
+                });
+                redis.on("error", () => { });
+            }
+            if (redis.status !== "ready")
+                await redis.connect();
+            // 与内核 bus 同一个频道命名（cap:kernel:<topic>）；CLI 只发布，不订阅
+            const bus = createBus(redis, {
+                subscribe: () => Promise.reject(new Error("the CLI does not subscribe")),
+            }, "kernel", log);
+            await bus.publish(topic, payload);
+        },
+    };
+    const installs = new ExtensionInstalls({
+        db,
+        blobs: stored,
+        config: () => config,
+        declared: () => declared,
+    });
+    const admin = new ExtensionAdmin({
+        db,
+        blobs,
+        installs,
+        config: () => config,
+        host: { declaredPlugins: () => declaredInfo },
+        bus: publisher,
+        log,
+    });
+    const ops = new ExtensionAdminOps({
+        db,
+        view: (name) => admin.get(name, { allowUnreadable: true }),
+        declared: () => declared,
+        blobs,
+        config: () => config,
+        bus: publisher,
+        log,
+    });
+    return {
+        config,
+        db,
+        inspected,
+        configIssues,
+        deploymentConfigs,
+        deploymentConfigOf,
+        installs,
+        admin,
+        ops,
+        approvals: new ExtensionApprovals(db),
+        remember: (sha256, bytes) => {
+            local.set(sha256, bytes);
+        },
+        readPackage: (sha256) => blobs.get(sha256),
+        async packageSet(sha256) {
+            // a stored package in the pre-0.1.0 format -> extension.legacy_format
+            const pkg = await readExtensionPackage(await blobs.get(sha256), {
+                maxBytes: config.packages.maxBytes,
+                hostSdkVersion: SDK_VERSION,
+            }).catch((e) => {
+                throw e instanceof ExtensionPackageError ? extensionPackageApiError(e) : e;
+            });
+            return permissionSetOfPackage(pkg);
+        },
+        async publishChanged(plugin) {
+            try {
+                await publisher.publish(PLUGINS_CHANGED_TOPIC, { plugin });
+                return true;
+            }
+            catch (err) {
+                log.warn("cannot publish plugins.changed; running replicas pick the change up on their next periodic reconcile", { plugin, err });
+                return false;
+            }
+        },
+        async close() {
+            redis?.disconnect();
+            await raw.destroy().catch(() => { });
+        },
+    };
+}

package/dist/extensions/config-json-schema.d.ts

@@ -0,0 +1,9 @@
+import { z } from "zod";
+export type JsonSchema = Record<string, unknown>;
+/**
+ * 把插件配置 schema 转成 JSON Schema（供界面渲染表单）：
+ * - `secret()` 字段带 `"x-secret": true`（去掉 zod 附带的 `secret` 元数据键）；
+ * - 无法表达的类型为 `{ "x-unsupported": true }`，不抛错；
+ * - 以输入侧形态输出（含 default），不带 `$schema`。
+ */
+export declare function configToJsonSchema(schema: z.ZodType): JsonSchema;

package/dist/extensions/config-json-schema.js

@@ -0,0 +1,73 @@
+import { z } from "zod";
+/** JSON Schema 无法表达的 zod 类型：降级为 `{}` 并标注 `x-unsupported`。 */
+const UNSUPPORTED = new Set([
+    "bigint",
+    "date",
+    "symbol",
+    "undefined",
+    "void",
+    "nan",
+    "map",
+    "set",
+    "function",
+    "promise",
+    "custom",
+    "file",
+]);
+/**
+ * 把插件配置 schema 转成 JSON Schema（供界面渲染表单）：
+ * - `secret()` 字段带 `"x-secret": true`（去掉 zod 附带的 `secret` 元数据键）；
+ * - 无法表达的类型为 `{ "x-unsupported": true }`，不抛错；
+ * - 以输入侧形态输出（含 default），不带 `$schema`。
+ */
+export function configToJsonSchema(schema) {
+    const out = z.toJSONSchema(schema, {
+        io: "input",
+        unrepresentable: "any",
+        override: (ctx) => {
+            const js = ctx.jsonSchema;
+            const def = ctx.zodSchema._zod.def;
+            if (js.secret === true) {
+                delete js.secret;
+                js["x-secret"] = true;
+            }
+            if (UNSUPPORTED.has(def.type))
+                js["x-unsupported"] = true;
+        },
+    });
+    delete out.$schema;
+    scrubSecrets(out);
+    return out;
+}
+/** 会泄露取值的关键字：密钥字段一律不输出。 */
+const VALUE_KEYWORDS = ["default", "examples", "const", "enum"];
+/** 值为「名字 -> 子 schema」映射的关键字：其键是用户字段名，不是关键字。 */
+const SCHEMA_MAPS = new Set(["properties", "patternProperties", "$defs", "definitions"]);
+/**
+ * 递归清掉密钥节点及其所有后代上会暴露取值的关键字（`inSecret` 随递归下传）；无环（输出为纯 JSON 树）。
+ * 只有 schema 关键字 default / examples / const / enum 的值被当作数据跳过；
+ * `properties` 等映射里的键是字段名，名为 default / enum 的字段照常遍历。
+ */
+function scrubSecrets(node, inSecret = false) {
+    if (Array.isArray(node)) {
+        for (const n of node)
+            scrubSecrets(n, inSecret);
+        return;
+    }
+    if (typeof node !== "object" || node === null)
+        return;
+    const o = node;
+    const secret = inSecret || o["x-secret"] === true;
+    if (secret)
+        for (const k of VALUE_KEYWORDS)
+            delete o[k];
+    for (const [k, v] of Object.entries(o)) {
+        if (SCHEMA_MAPS.has(k) && typeof v === "object" && v !== null) {
+            for (const child of Object.values(v))
+                scrubSecrets(child, secret);
+        }
+        else if (!VALUE_KEYWORDS.includes(k)) {
+            scrubSecrets(v, secret);
+        }
+    }
+}

package/dist/extensions/deployment-config.d.ts

@@ -0,0 +1,138 @@
+import { type Transaction } from "kysely";
+import type { core } from "zod";
+import { type ConfigSourceKind } from "../config/sources.js";
+import { type Keyring } from "../ctx/secrets.js";
+import type { KernelDatabase, KernelDb } from "../db/kernel-schema.js";
+import { type JsonSchema } from "./config-json-schema.js";
+import { type ConfigIssue } from "./org-config.js";
+type Trx = Transaction<KernelDatabase>;
+type Obj = Record<string, unknown>;
+/** 部署级插件配置的密钥存在 `capybara.secrets` 的这个 org_id 下（不是真实组织；namespace `kernel`）。 */
+export declare const DEPLOYMENT_SECRETS_ORG = "_deployment";
+/**
+ * 部署配置保存之后（kernel bus 主题，频道 `cap:kernel:plugin-deployment-config`，payload `{ plugin }`）：
+ * 各副本重读并校验——只改了 hot 字段就地替换，改了非 hot 字段重新加载实例，不合法保持当前值并告警；
+ * 插件此刻待配置时，下一次 reconcile 加载它。
+ */
+export declare const PLUGIN_DEPLOYMENT_CONFIG_TOPIC = "plugin-deployment-config";
+export type { ConfigIssue };
+/** 后台一个部署配置字段：来源、是否只读；非密钥字段给出生效值，密钥字段只给 `set`。 */
+export interface DeploymentConfigField {
+    path: string;
+    secret: boolean;
+    hot: boolean;
+    /**
+     * `env` / `file` / `nacos`：启动配置给了（只读）；`db`：后台保存的值；`default`：schema 默认值；`unset`：没有值。
+     */
+    source: ConfigSourceKind | "db" | "default" | "unset";
+    readonly: boolean;
+    /** 非密钥字段的生效值（没有值时省略）。 */
+    value?: unknown;
+    /** 密钥字段是否已设置（任何来源）；值永不回显。 */
+    set?: boolean;
+}
+/** The response of `GET / PUT /api/v1/admin/extensions/:name/config`. */
+export interface DeploymentConfigView {
+    plugin: string;
+    version: string;
+    /** 插件声明了部署级配置。 */
+    configurable: boolean;
+    fields: DeploymentConfigField[];
+    /** 部署级 schema 的 JSON Schema（密钥字段带 `x-secret`）；本副本拿不到模块（未加载的安装记录）时为 null。 */
+    schema: JsonSchema | null;
+    /** 生效配置按 schema 校验通过（`validated` 为 false 时没有校验，恒为 true，界面不要据此显示「合法」）。 */
+    valid: boolean;
+    /**
+     * 本副本拿到了插件模块的 zod schema、`valid` / `issues` 是真实校验结果。false：未加载的安装记录，
+     * 字段按包内静态清单给出，没有校验（加载时才校验，不合法则待配置）。
+     */
+    validated: boolean;
+    /** 校验失败的原因（只有路径与说明，不含值）。 */
+    issues: ConfigIssue[];
+}
+/** 一个插件的部署级配置描述：真实模块的 zod schema（有的话）与字段清单。 */
+export interface DeploymentSchemaInfo {
+    version: string;
+    schema?: core.$ZodType | undefined;
+    fields: {
+        path: string;
+        secret: boolean;
+        hot: boolean;
+    }[];
+}
+/** 由 zod schema 得出字段清单（`configFields`：对象路径上的叶子，secret 向下继承）。 */
+export declare function schemaFields(schema: core.$ZodType | undefined): DeploymentSchemaInfo["fields"];
+/**
+ * 合并一个插件的部署配置（C10 的优先级：环境变量 > 文件 / Nacos > PG > schema 默认值）：以启动配置
+ * （`fileValue`，已含环境变量）为底，逐字段补上 PG 里保存、而启动配置没有覆盖的值。按字段合并：启动配置给了的字段
+ * 整个以它为准（不与 PG 的值深合并）。`fields` 为字段点路径（没有 schema 时用 PG 值的叶子路径）。默认值由校验补上。
+ */
+export declare function mergeDeploymentConfig(stored: Obj, fileValue: unknown, fields?: string[]): unknown;
+/**
+ * PG 里一个插件保存的部署配置（非密钥字段 + 解密的密钥字段，按点路径放回对象）。只并入 `secretPaths`（当前
+ * schema 里是密钥的字段）的密钥：旧版本留下、现在已不是密钥或已删掉的字段不解密、不并入（否则会被当成明文回显
+ * 或保存）。
+ */
+export declare function readStoredDeploymentConfig(db: KernelDb | Trx, keyring: Keyring, plugin: string, opts: {
+    forUpdate?: boolean;
+    secretPaths: readonly string[];
+}): Promise<Obj>;
+/** 字段清单里的密钥字段路径。 */
+export declare const secretPathsOf: (fields: DeploymentSchemaInfo["fields"]) => string[];
+export interface ExtensionDeploymentConfigDeps {
+    db: KernelDb;
+    keyring: Keyring;
+    /** 启动配置里的 `extensionConfig`（文件 / Nacos 与环境变量合并后）。 */
+    fileConfig: () => Record<string, unknown>;
+    /** 启动配置各字段的来源（见 `extensionConfigSources`）；这些字段只读。 */
+    sources: Record<string, Record<string, ConfigSourceKind>>;
+    /** The extension's deployment config description; undefined when the deployment does not know the extension (`extension.not_found`). */
+    schemaOf: (plugin: string) => Promise<DeploymentSchemaInfo | undefined>;
+    /** 保存提交后发布 {@link PLUGIN_DEPLOYMENT_CONFIG_TOPIC}；失败只告警（各副本的周期 reconcile 会重读生效插件的部署配置兜底）。 */
+    bus?: {
+        publish(topic: string, payload: unknown): Promise<void>;
+    };
+    /** 本副本立即应用（不等 bus 消息回到自己；其他副本经 bus）。失败只记 error。 */
+    applyLocal?: (plugin: string) => Promise<unknown>;
+    /** 经手的密钥值（提交后），内核传 `log.addSecrets`。 */
+    onSecrets?: (values: string[]) => void;
+    log: {
+        warn(msg: string, fields?: Record<string, unknown>): void;
+        error(msg: string, fields?: Record<string, unknown>): void;
+    };
+}
+/**
+ * Deployment-level extension config (D8 / C10): non-secret fields are stored in
+ * `extension_deployment_configs`, each secret field encrypted in `capybara.secrets` (namespace
+ * `kernel`, org `_deployment`, key `plugin-deployment-config:<plugin>:<path>`). Fields present in the
+ * startup config (file / Nacos / env) are read-only and show their source; the effective value
+ * follows {@link mergeDeploymentConfig}. No in-process state, shared by every replica.
+ * Secret values are never echoed: responses only say "set", audits only record which paths changed.
+ */
+export declare class ExtensionDeploymentConfigService {
+    private readonly d;
+    constructor(d: ExtensionDeploymentConfigDeps);
+    /** 插件的生效部署配置（未校验）：启动配置为底，补上 PG 里保存的字段。`schema` 给出时按它的字段合并。 */
+    effective(plugin: string, fileValue: unknown, schema?: core.$ZodType): Promise<unknown>;
+    /** Admin view; an extension unknown to the deployment -> `extension.not_found`. */
+    view(plugin: string): Promise<DeploymentConfigView>;
+    /**
+     * Saves some fields: `values` keyed by field dot path, `null` = clear (back to the default),
+     * omitted = unchanged.
+     * - Unknown to the deployment -> `extension.not_found`; no deployment config, unknown field, or
+     *   invalid after merging (except a problem in a read-only startup-config field itself) ->
+     *   `extension.config_invalid` (details carry only paths and messages);
+     * - a field present in the startup config (file / Nacos / env) -> 409 `extension.config_readonly`.
+     * In a transaction, first `lockEnterpriseAdminsAndAuthorize` (re-checks the enterprise admin under
+     * the lock), then an advisory lock per extension; audit `extension.deployment_config_updated`
+     * (detail only `changedPaths`). After the commit publishes {@link PLUGIN_DEPLOYMENT_CONFIG_TOPIC} and
+     * applies it on this replica right away.
+     */
+    put(plugin: string, values: Record<string, unknown>, actor: string, actorTokenId?: string): Promise<DeploymentConfigView>;
+    private info;
+    private render;
+}
+/**
+ * 本副本拿不到模块时（未加载的安装记录）按包内审阅过的静态清单给出字段（没有 zod schema：不校验，加载时再校验）。
+ */
+export declare function staticDeploymentSchema(db: KernelDb, plugin: string): Promise<DeploymentSchemaInfo | undefined>;