@capybara-hq/kernel 0.0.0-stage → 0.1.0-rc.0

package/dist/runtime/maintenance.js

@@ -0,0 +1,147 @@
+import { Queue, Worker } from "bullmq";
+import { sql } from "kysely";
+import { bullConnection } from "./run-queue.js";
+/**
+ * 内核维护队列（不含 `:`）：等待超时任务（`wait-timeout`，见 waits.ts）与周期性维护（输出存档清理、
+ * 工具漂移检测（见 tools/drift.ts）、等待兜底扫描（`wait-sweep`，见 waits.ts）、沙箱的空闲休眠与不活跃销毁
+ * （见 sandbox/lifecycle.ts）等）。整个队列只有一个 Worker（只在 `worker` 角色上启动），
+ * 按任务名分派给登记的处理函数。
+ * 与运行队列分开：运行 worker 的处理函数、stalled / 失败处理都按「一个任务 = 一次运行」设计；
+ * 维护任务短、无会话锁与配额。
+ */
+export const MAINTENANCE_QUEUE_NAME = "cap-kernel-maintenance";
+/** 输出存档清理：任务名、job scheduler id（不含 `:`）、周期与每批删除行数。 */
+export const OUTPUT_CLEANUP_JOB = "output-cleanup";
+export const OUTPUT_CLEANUP_SCHEDULER_ID = "output-cleanup-hourly";
+export const OUTPUT_CLEANUP_EVERY_MS = 3_600_000;
+export const OUTPUT_CLEANUP_BATCH = 1000;
+/**
+ * Periodic marketplace refresh (C26): job name and job scheduler id (no `:`), every
+ * `marketplaces.refreshIntervalMinutes`. One scheduler for all replicas; the handler refreshes each
+ * enabled marketplace in turn (see catalog/marketplaces.service.ts). The values keep their old
+ * names: the scheduler id is stored in Redis, and renaming it would leave the old scheduler behind.
+ */
+export const MARKETPLACES_REFRESH_JOB = "catalogs-refresh";
+export const MARKETPLACES_REFRESH_SCHEDULER_ID = "catalogs-refresh-every";
+/**
+ * Cleanup of plugin (app bundle) import staging (T25, F-AE): job name and job scheduler id (no
+ * `:`), hourly; deletes expired (24-hour) staging rows and their object storage prefixes (see
+ * catalog/plugin-import.ts `PluginImport.cleanupExpired`). The values keep their old names: the
+ * scheduler id is stored in Redis, so renaming it would leave the old scheduler behind.
+ */
+export const PLUGIN_STAGING_CLEANUP_JOB = "app-staging-cleanup";
+export const PLUGIN_STAGING_CLEANUP_SCHEDULER_ID = "app-staging-cleanup-hourly";
+export const PLUGIN_STAGING_CLEANUP_EVERY_MS = 3_600_000;
+/**
+ * 删除已过期（`expires_at < now()`）的输出存档，每批 `batch` 行，直到删完；返回删除的总行数。
+ * 每批一条独立语句（不开长事务）；多个 worker 同时清理时 `SKIP LOCKED` 互不等待。
+ */
+export async function deleteExpiredOutputs(db, batch = OUTPUT_CLEANUP_BATCH) {
+    let total = 0;
+    for (;;) {
+        const r = await db
+            .deleteFrom("tool_outputs")
+            .where("id", "in", db
+            .selectFrom("tool_outputs")
+            .select("id")
+            .where("expires_at", "<", sql `now()`)
+            .limit(batch)
+            .forUpdate()
+            .skipLocked())
+            .executeTakeFirst();
+        const n = Number(r.numDeletedRows);
+        total += n;
+        if (n < batch)
+            return total;
+    }
+}
+/**
+ * 维护队列的生产方与（worker 角色上的）唯一 Worker。状态都在 Redis / PG，没有进程内定时器：
+ * 周期任务用 BullMQ job scheduler（`upsertJobScheduler` 幂等，多副本重复调用只有一个 scheduler）。
+ */
+export class Maintenance {
+    d;
+    queue;
+    queueConn;
+    worker;
+    workerConn;
+    closed = false;
+    handlers = new Map();
+    schedules = [];
+    constructor(d) {
+        this.d = d;
+        this.handle(OUTPUT_CLEANUP_JOB, async () => {
+            const n = await deleteExpiredOutputs(d.db);
+            if (n > 0)
+                d.log.info("expired tool outputs deleted", { count: n });
+        });
+    }
+    /** 登记某个任务名的处理函数（构造期调用；Worker 启动前后都可）。 */
+    handle(name, fn) {
+        this.handlers.set(name, fn);
+    }
+    /** 登记一个周期任务（构造期调用；`ensureSchedules` 时一并 upsert）。 */
+    schedule(s) {
+        this.schedules.push(s);
+    }
+    /** 投递一个维护任务。 */
+    async add(name, data, opts) {
+        await this.getQueue().add(name, data, opts);
+    }
+    /** 登记周期任务的 job scheduler（幂等；worker 角色启动时调用）。 */
+    async ensureSchedules() {
+        await this.getQueue().upsertJobScheduler(OUTPUT_CLEANUP_SCHEDULER_ID, { every: OUTPUT_CLEANUP_EVERY_MS }, {
+            name: OUTPUT_CLEANUP_JOB,
+            data: {},
+            opts: { removeOnComplete: true, removeOnFail: 100 },
+        });
+        for (const s of this.schedules) {
+            await this.getQueue().upsertJobScheduler(s.id, { every: s.everyMs() }, { name: s.name, data: {}, opts: { removeOnComplete: true, removeOnFail: 100 } });
+        }
+    }
+    /** 启动维护队列的 Worker（只在含 `worker` 角色的进程）。 */
+    startWorker() {
+        if (this.worker || this.closed)
+            return;
+        this.workerConn = bullConnection(this.d.redisUrl, this.d.log);
+        this.worker = new Worker(MAINTENANCE_QUEUE_NAME, (job, token) => this.process(job, token), {
+            connection: this.workerConn,
+            concurrency: 5,
+        });
+        this.worker.on("error", (err) => this.d.log.error("maintenance worker error", { err }));
+        this.worker.on("failed", (job, err) => this.d.log.error("maintenance job failed", { name: job?.name, jobId: job?.id, err }));
+    }
+    async close() {
+        this.closed = true;
+        const w = this.worker;
+        const q = this.queue;
+        this.worker = undefined;
+        this.queue = undefined;
+        await w?.close().catch(() => { });
+        await q?.close().catch(() => { });
+        for (const c of [this.workerConn, this.queueConn]) {
+            if (c)
+                await c.quit().catch(() => c.disconnect());
+        }
+        this.workerConn = undefined;
+        this.queueConn = undefined;
+    }
+    getQueue() {
+        if (this.closed)
+            throw new Error("maintenance queue is closed");
+        if (!this.queue) {
+            this.queueConn = bullConnection(this.d.redisUrl, this.d.log);
+            this.queue = new Queue(MAINTENANCE_QUEUE_NAME, { connection: this.queueConn });
+            this.queue.on("error", (err) => this.d.log.error("maintenance queue error", { err }));
+        }
+        return this.queue;
+    }
+    async process(job, token) {
+        const fn = this.handlers.get(job.name);
+        if (!fn) {
+            this.d.log.warn("unknown maintenance job", { name: job.name, jobId: job.id });
+            return;
+        }
+        await fn(job, token);
+    }
+}

package/dist/runtime/media-cache.d.ts

@@ -0,0 +1,21 @@
+import type { MediaBytes, MediaInfo } from "./context/build.js";
+/**
+ * 一次运行内的附件字节缓存（进程内，LRU，按总字节数封顶）：同一次运行的后续步骤不再重复取同一附件。
+ * 上限取 `attachments.requestMediaBytes`：一次请求带的媒体字节不超过它，缓存再大也用不上。
+ *
+ * 多副本下允许：键含 finalize 时的 etag 与大小（内容不可变），只是优化——缓存未命中时照常从存储取，
+ * 不同副本之间不共享、也不需要一致；随运行结束丢弃。
+ */
+export declare class MediaCache {
+    private readonly maxBytes;
+    private readonly entries;
+    private total;
+    constructor(maxBytes: number);
+    /** 缓存键：附件 id、etag、声明大小与取的字节上限（`text/*` 只取开头）。 */
+    static key(attachmentId: string, info: MediaInfo, cap: number | undefined): string;
+    /** 已缓存的总字节数。 */
+    get size(): number;
+    get(key: string): MediaBytes | undefined;
+    /** 放入；比上限还大的不缓存。超出上限时淘汰最久未用的。 */
+    set(key: string, value: MediaBytes): void;
+}

package/dist/runtime/media-cache.js

@@ -0,0 +1,51 @@
+/**
+ * 一次运行内的附件字节缓存（进程内，LRU，按总字节数封顶）：同一次运行的后续步骤不再重复取同一附件。
+ * 上限取 `attachments.requestMediaBytes`：一次请求带的媒体字节不超过它，缓存再大也用不上。
+ *
+ * 多副本下允许：键含 finalize 时的 etag 与大小（内容不可变），只是优化——缓存未命中时照常从存储取，
+ * 不同副本之间不共享、也不需要一致；随运行结束丢弃。
+ */
+export class MediaCache {
+    maxBytes;
+    entries = new Map();
+    total = 0;
+    constructor(maxBytes) {
+        this.maxBytes = maxBytes;
+    }
+    /** 缓存键：附件 id、etag、声明大小与取的字节上限（`text/*` 只取开头）。 */
+    static key(attachmentId, info, cap) {
+        return [attachmentId, info.etag ?? "", info.size, cap ?? ""].join("\u0000");
+    }
+    /** 已缓存的总字节数。 */
+    get size() {
+        return this.total;
+    }
+    get(key) {
+        const v = this.entries.get(key);
+        if (v === undefined)
+            return undefined;
+        // 最近使用的移到末尾
+        this.entries.delete(key);
+        this.entries.set(key, v);
+        return v;
+    }
+    /** 放入；比上限还大的不缓存。超出上限时淘汰最久未用的。 */
+    set(key, value) {
+        const n = value.data.byteLength;
+        if (n > this.maxBytes)
+            return;
+        const old = this.entries.get(key);
+        if (old) {
+            this.total -= old.data.byteLength;
+            this.entries.delete(key);
+        }
+        this.entries.set(key, value);
+        this.total += n;
+        for (const [k, v] of this.entries) {
+            if (this.total <= this.maxBytes)
+                break;
+            this.entries.delete(k);
+            this.total -= v.data.byteLength;
+        }
+    }
+}

package/dist/runtime/model-call.d.ts

@@ -0,0 +1,176 @@
+import { type ContentBlock, type Logger, type ModelProvider, type ModelRequest, type ModelStreamEvent, type ModelUsage, type StopReason } from "@capybara-hq/plugin-sdk";
+import type { AgentDefinition } from "../agents/definition.js";
+import type { KernelConfig } from "../config/schema.js";
+import type { EventLog } from "../conversations/event-log.js";
+import type { KernelDb } from "../db/kernel-schema.js";
+import type { ToolRegistry, UnavailableTool } from "../tools/registry.js";
+import { type BuildModelRequestInput, type MediaBytes, type MediaInfo, type ModelCallMeta, type RequestTool } from "./context/build.js";
+import { type ConversationKindForHooks, type HookRunner } from "./hook-calls.js";
+import type { SubagentParent } from "./loop.js";
+import { MediaCache } from "./media-cache.js";
+import { type ProviderRegistry } from "./providers.js";
+import type { UsageReporter } from "./usage-report.js";
+/** 单轮收尾的尾部提醒（模型可见，英文；spec 6 轮次额度收尾）。 */
+export declare const WRAP_UP_REMINDER = "Step limit reached. Do not call tools. Summarize what you found and what remains for the user.";
+/** 所用模型不支持工具（`capabilities.tools` 为 false）时的提醒（模型可见，英文）。 */
+export declare const TOOLS_UNSUPPORTED_REMINDER = "Tools are unavailable for this model. Answer without calling tools.";
+/** 不可用工具提醒的前缀（模型可见，英文）。 */
+export declare const TOOLS_UNAVAILABLE_PREFIX = "The following tools are unavailable in this run: ";
+/** 本 run 中此前有工具不可用、现在全部恢复时的提醒（模型可见，英文）。 */
+export declare const TOOLS_AVAILABLE_AGAIN = "All previously unavailable tools are available again.";
+/** 不可用工具提醒：`The following tools are unavailable in this run: <ref> (<reason>), ...`。 */
+export declare function toolsUnavailableReminder(unavailable: readonly UnavailableTool[]): string;
+/** 流式增量的出口（推给前端等订阅方）；`step_reset` 表示丢弃本步已推出的增量、重新开始。 */
+export type StreamSink = (delta: {
+    kind: "text" | "thinking" | "step_reset";
+    runId: string;
+    step: number;
+    text?: string;
+}) => void;
+export interface ModelStepDeps {
+    db: KernelDb;
+    eventLog: EventLog;
+    registry: Pick<ProviderRegistry, "resolve">;
+    /** 工具注册表：每步开始时解析 agent 引用的工具（状态可能在步骤之间变化）。 */
+    tools: Pick<ToolRegistry, "forAgent">;
+    config: () => KernelConfig;
+    log: Logger;
+    /**
+     * 会话附件（构建请求时解析日志里的媒体引用；见 AttachmentsService）。未提供时媒体引用一律按「不再可用」处理。
+     * 两个方法都只对「确定不存在」返回缺省 / null；暂时故障（存储不可用、网络错误）一律抛错（Ruling M4）。
+     */
+    media?: MediaSourceDeps;
+    /**
+     * Plugin hooks (`model.request` policy / observe before the call, `model.response` observe after it),
+     * for every attempt of a step and for the compaction call (`purpose: "compaction"`, I-1).
+     * Only chat runs get it: full-run and single-step eval replays omit it (P-18), and hooks also need
+     * `conversationKind: "chat"` on the step input.
+     */
+    hooks?: HookRunner;
+    /**
+     * `usage.report` delivery (P-17): each written `usage` event is enqueued (chat runs and full-run eval
+     * replays, which both log usage). `enqueue` never throws and returns at once without handlers.
+     */
+    usageReport?: Pick<UsageReporter, "enqueue">;
+}
+/** 构建请求用到的附件来源。 */
+export interface MediaSourceDeps {
+    /** 附件行的声明信息（只含绑定在该会话、已 ready 的；没有行的不在结果里）。 */
+    info(orgId: string, conversationId: string, ids: readonly string[]): Promise<Map<string, MediaInfo>>;
+    /** 字节；对象已不在或内容被替换为 null。`caps` 里的附件只取开头若干字节（`text/*` 封顶，见 planMedia）。 */
+    fetch(orgId: string, conversationId: string, ids: readonly string[], caps?: ReadonlyMap<string, number>): Promise<Map<string, MediaBytes | null>>;
+}
+/**
+ * 取附件时存储暂时不可用（Ruling M4）：不把它当成「附件不在」发占位，而是让本次尝试按可重试错误失败
+ * （`run.storage_unavailable`），走降级梯子的重试。
+ */
+export declare class MediaUnavailableError extends Error {
+    constructor(cause: unknown);
+}
+export interface ModelStepInput {
+    /** 会话所属组织（解析工具用）。 */
+    orgId: string;
+    conversationId: string;
+    runId: string;
+    step: number;
+    agent: AgentDefinition;
+    agentVersion: number;
+    signal: AbortSignal;
+    sink: StreamSink;
+    /**
+     * 本步开始时 loop 已吸收到的 user_message seq（R31）：seq 更大的 user_message 不进本步请求，
+     * 由后续步骤吸收或触发下一轮；写入 `model_call.request.absorbedUpToSeq`，请求可由日志还原。
+     */
+    absorbedUpToSeq: number;
+    /** 本次运行的附件字节缓存（同一次运行的各步共用，见 {@link MediaCache}）；缺省时只在本步内缓存。 */
+    mediaCache?: MediaCache;
+    /** Hook payload scope: the run's agent, its trigger account and the conversation kind. */
+    agentId?: string;
+    accountId?: string;
+    /** Hooks fire for `chat` and `subagent` (never for eval replays). */
+    conversationKind?: ConversationKindForHooks;
+    /** A sub-agent's run: where it was dispatched from (hook payloads carry it as `parent`). */
+    parent?: SubagentParent;
+    /**
+     * A sub-agent's run (phase C): the parent run whose `subagent_tokens_used` counts every usage
+     * event this step writes (step, wrap-up and compaction calls; spec §8.3).
+     */
+    parentRunId?: string;
+    /**
+     * The run's step limit (`LoopContext.limits.maxSteps`) for the turn-limit reminder; a child's
+     * may be below `runtime.maxSteps` (final review m4). Defaults to `runtime.maxSteps`.
+     */
+    maxSteps?: number;
+}
+export interface ModelStepResult {
+    content: ContentBlock[];
+    stopReason: StopReason | "aborted";
+    /** 本步的 `assistant_message` 已存在（幂等命中），未调用模型。 */
+    reused: boolean;
+    aborted: boolean;
+    usage: ModelUsage | null;
+    /**
+     * A sub-agent's step stopped before its request: the step's compaction call used up the
+     * parent run's token budget (spec §8.3: at most one call over per active child). The loop ends
+     * the run with `budget`.
+     */
+    budget?: boolean;
+}
+/**
+ * 运行级失败：`code` 写入 `run_failed.error.code`。模型错误为 `model.<kind>`，其余为
+ * `run.<reason>`（如 `run.model_unavailable`、`run.provider_unavailable`、`run.provider_error`）。
+ */
+export declare class RunError extends Error {
+    readonly code: string;
+    constructor(code: string, message: string);
+}
+/** 幂等键。`usage` 与 `assistant_message` 同批写入，带键保证重放时不重复记账。 */
+/** 某步 assistant_message 的幂等键（tools 节点据此找到本步的工具调用）。 */
+export declare const assistantIdemKey: (runId: string, step: number) => string;
+/** 某步第 attempt 次模型调用的 model_call 幂等键（管线据此找到模型实际收到的工具）。 */
+export declare const modelCallIdemKey: (runId: string, step: number, attempt: number) => string;
+/** 一步最多的模型调用尝试次数（主模型、同模型重试、兜底，见 `ladder`）。 */
+export declare const MAX_MODEL_ATTEMPTS = 3;
+/** 一次模型流的结果（{@link consumeModelStream}）。 */
+export interface StreamOutcome {
+    content: ContentBlock[];
+    stopReason: StopReason;
+    usage: ModelUsage;
+}
+/**
+ * 一步模型调用：幂等 → 读日志 → 压缩 → 尾部提醒 → 构建请求 → 流式（看门狗）→ 降级梯子 → 写事件。
+ * 模型看到的一切先写进日志（`model_call` / `reminder` / `compaction_summary`），请求可由日志还原。
+ */
+export declare function runModelStep(deps: ModelStepDeps, input: ModelStepInput): Promise<ModelStepResult>;
+/**
+ * 单轮收尾：到达步数上限时调用一次。写入收尾 `reminder`（位于请求末尾）；照常带工具（历史里有工具调用时
+ * 部分供应商要求请求声明工具，也保持缓存前缀稳定），但 `toolChoice: "none"`——模型只能回复文本。
+ * 其余与 `runModelStep` 相同（幂等、降级、中止）。
+ */
+export declare function runFinalWrapUp(deps: ModelStepDeps, input: ModelStepInput): Promise<ModelStepResult>;
+/** {@link assembleModelRequest} 的输入。 */
+export interface AssembleModelRequestInput extends Omit<BuildModelRequestInput, "tools"> {
+    /** 本步解析出的工具；模型不支持工具（`capabilities.tools` 为 false）时不带。 */
+    tools: readonly RequestTool[];
+    /** 参数里没有 `maxOutputTokens` 旋钮时的输出上限（运行时为 `runtime.maxOutputTokens`）。 */
+    defaultMaxOutputTokens: number;
+}
+/**
+ * 发给模型的请求（纯函数）：`buildModelRequest`（注入 `params` 与本步工具：模型不支持工具时不带；收尾调用带
+ * `toolChoice: "none"`）→ `guardToolPairs`。输出上限：参数里有 `maxOutputTokens` 旋钮时取
+ * `min(模型上限, 旋钮)`，否则 `min(模型上限, defaultMaxOutputTokens)`。
+ * 运行时（{@link runModelStep}）与由日志还原请求（`evals/reconstruct.ts`）共用这一份，二者不会漂移。
+ */
+export declare function assembleModelRequest(input: AssembleModelRequestInput): {
+    request: ModelRequest;
+    meta: ModelCallMeta;
+    mediaTokens: number;
+};
+/**
+ * 消费一次模型流，带空闲看门狗：两次事件之间超过 `idleMs` → 本地中止并按 `ModelError("timeout")` 抛出。
+ * 外部 `signal` 触发 → `StepAborted`。每次 `next()` 都与停止信号竞速，供应商不响应 signal 也不会卡住。
+ * 运行时与评测单步回放（`evals/step-replay.ts`）、评委共用。
+ */
+export declare function consumeModelStream(provider: ModelProvider, request: ModelRequest, idleMs: number, signal: AbortSignal, onDelta: (ev: Exclude<ModelStreamEvent, {
+    type: "message_complete";
+}>) => void): Promise<StreamOutcome>;