@capybara-hq/kernel 0.1.0-rc.0 → 0.1.0-rc.10

package/dist/evals/worker.js

@@ -1,23 +1,42 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __param = (this && this.__param) || function (paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+};
 import { randomUUID } from "node:crypto";
-import { hostname } from "node:os";
 import { isModelError, isRetryableKind } from "@capybara-hq/plugin-sdk";
-import { DelayedError, Worker } from "bullmq";
+import { Inject, Injectable, Optional } from "@nestjs/common";
+import { DelayedError } from "bullmq";
 import { sql } from "kysely";
 import { loadTemplates } from "../agents/templates.js";
+import { EventLog } from "../conversations/event-log.js";
 import { ApiError } from "../errors.js";
 import { ExtensionUnloadedError } from "../extensions/inflight.js";
-import { PurposeAttemptError } from "../models/purposes.js";
+import { waitWithin } from "../infra/close-within.js";
+import { KernelConfigService } from "../infra/config/kernel-config.service.js";
+import { KernelDb } from "../infra/database/kernel-db.js";
+import { TransactionHost } from "../infra/database/transaction-host.js";
+import { Transactional } from "../infra/database/transactional.decorator.js";
+import { KERNEL_LOGGER } from "../infra/tokens.js";
+import { ModelPurposesService, PurposeAttemptError } from "../models/purposes.js";
+import { KyselyCheckpointSaver } from "../runtime/checkpointer.js";
 import { MediaUnavailableError, RunError } from "../runtime/model-call.js";
-import { ProviderListError } from "../runtime/providers.js";
-import { ORG_QUOTA_LEASE_MS } from "../runtime/quota.js";
-import { bullConnection } from "../runtime/run-queue.js";
-import { consumeTokens, recordOutcome, recountCases, runActive, skipPendingJudges, usageTokens, } from "./budget.js";
+import { ProviderListError, ProviderRegistry } from "../runtime/providers.js";
+import { EVAL_QUOTA, ORG_QUOTA_LEASE_MS } from "../runtime/quota.js";
+import { RunFinish } from "../runtime/run-finish.js";
+import { AgentLoops, RunSteps } from "../runtime/run-steps.js";
+import { consumeTokens, EvalBudget, recordOutcome, runActive, skipPendingJudges, usageTokens, } from "./budget.js";
 import { replyParts, structuralDiff } from "./diff.js";
+import { evalJobId } from "./jobs.js";
 import { JUDGE_PROVIDER_UNAVAILABLE, judge } from "./judge.js";
 import { userRequestOf } from "./plan.js";
-import { subagentCaseChanged, subagentStartedOf } from "./reconstruct.js";
-import { deleteEvalConversation, ReplayClaimLost, replayRun, runBaseline, } from "./run-replay.js";
-import { caseBase, EVAL_QUEUE_NAME, evalJobId, versionDefinition, } from "./runs.service.js";
+import { subagentCaseChanged, subagentStartedOf, } from "./reconstruct.js";
+import { deleteEvalConversation, EvalReplaySeeder, ReplayClaimLost, replayRun, runBaseline, } from "./run-replay.js";
+import { caseBase, versionDefinition } from "./runs.service.js";
 import { readSnapshot, replayableCalls, toEvents } from "./snapshot.js";
 import { ReplayAttemptError, replayStep, resolveCandidate, } from "./step-replay.js";
 /**
@@ -41,7 +60,13 @@ export const EVAL_MAX_RETRIES = 3;
 /** 第 n 次重试前的等待（毫秒）；供应商给了 `retryAfterMs` 时取它（不超过 30 秒）。 */
 const RETRY_DELAYS_MS = [1000, 2000, 4000];
 const MAX_RETRY_AFTER_MS = 30_000;
-const MAX_STALLED_COUNT = 3;
+/** The eval worker's BullMQ stalled-job limit (the processor's `maxStalledCount`). */
+export const EVAL_MAX_STALLED_COUNT = 3;
+/**
+ * Test hooks of the full-run replay and the judge (`EvalReplayHooks`): a provider override in tests
+ * (`overrides: [{ provide: EVAL_REPLAY_HOOKS, useValue }]`), absent in production.
+ */
+export const EVAL_REPLAY_HOOKS = Symbol("EVAL_REPLAY_HOOKS");
 /** 供应商插件在调用途中被撤销（错误本身或其 cause）。 */
 const isPluginUnloaded = (err) => err instanceof ExtensionUnloadedError ||
     (err instanceof Error && err.cause instanceof ExtensionUnloadedError);
@@ -134,42 +159,90 @@ function judgeInputOf(s, mode, modelCallSeq, baseline, candidate, rubric) {
         rubric,
     };
 }
-export class EvalWorker {
-    d;
+let EvalWorker = class EvalWorker {
+    txHost;
     worker;
-    conn;
     /** 本进程在执行的任务：停机时中止，任务回到队列。 */
     active = new Set();
     closing = false;
-    constructor(d) {
-        this.d = d;
+    d;
+    constructor(kdb, quota, config, log, purposes, registry, steps, loops, eventLog, runFinish, checkpoints, seeder, budget, txHost, hooks) {
+        this.txHost = txHost;
+        const stepDeps = steps.deps;
+        this.d = {
+            get db() {
+                return kdb.db;
+            },
+            quota,
+            config,
+            log,
+            purposes,
+            budget,
+            // 媒体按快照自己的组织与对话、经组织范围的附件接口取（C3）
+            replay: {
+                registry,
+                config,
+                ...(stepDeps.media !== undefined && { media: stepDeps.media }),
+            },
+            // 整轮回放（C6）：同一个 loop 工厂（同一 LangGraphLoop），只换工具管线（RecordedToolPipeline，
+            // 从不执行工具）、计费的模型注册表与按原对话取附件的来源
+            runReplay: {
+                get db() {
+                    return kdb.db;
+                },
+                eventLog,
+                seeder,
+                runFinish,
+                config,
+                log,
+                registry,
+                ...(stepDeps.media !== undefined && { media: stepDeps.media }),
+                checkpoints,
+                // explicitly without plugin hooks: eval replays never fire them (P-18); nor do they
+                // publish sub-agent progress on a live conversation's stream
+                loop: (d) => {
+                    // nor dispatch fork invocations: run-replay passes the recorded reports instead
+                    const { db: _snapshot, hooks: _noHooks, subagentProgress: _noProgress, skillForks: _noForks, ...replayDeps } = stepDeps;
+                    // the object rest above copied `db` as a value: read it through the step deps' getter
+                    // instead, so the loop always resolves the context-aware handle
+                    return loops.create({
+                        ...replayDeps,
+                        get db() {
+                            return stepDeps.db;
+                        },
+                        ...d,
+                    });
+                },
+                ...(hooks !== undefined && { hooks }),
+            },
+        };
     }
-    start() {
-        if (this.worker)
+    /**
+     * Starts consuming with the `cap-kernel-evals` processor's worker (worker role, after the
+     * extensions loaded; built with `autorun: false`).
+     */
+    start(worker) {
+        if (this.worker || this.closing)
             return;
-        const { runtime, evals } = this.d.config();
-        this.conn = bullConnection(this.d.redisUrl, this.d.log);
-        this.worker = new Worker(EVAL_QUEUE_NAME, (job, token) => this.process(job, token), {
-            name: `${hostname()}-${process.pid}`,
-            connection: this.conn,
-            concurrency: evals.workerConcurrency,
-            lockDuration: runtime.jobLockMs,
-            stalledInterval: runtime.stalledIntervalMs,
-            maxStalledCount: MAX_STALLED_COUNT,
-        });
-        this.worker.on("error", (err) => this.d.log.error("eval worker error", { err }));
-        this.worker.on("failed", (job, err) => {
-            if (!job)
-                return;
-            // 处理函数外的暂时故障（PG / Redis）由 BullMQ 按 attempts 指数退避重试；用尽（或 stalled 超限）才算失败
-            if (job.attemptsMade < (job.opts.attempts ?? 1) && !/stalled/i.test(err.message)) {
-                this.d.log.warn("eval job attempt failed; retrying", { jobId: job.id, err });
-                return;
-            }
-            this.d.log.error("eval job failed", { jobId: job.id, err });
-            // 任务不会再执行：该用例剩下的结果记 error，运行不因此卡住
-            void this.failCase(job.data, err).catch((e) => this.d.log.error("cannot mark failed eval job", { jobId: job.id, err: e }));
-        });
+        this.worker = worker;
+        void worker.run().catch((err) => this.onError(err));
+    }
+    /** The worker's `error` event. */
+    onError(err) {
+        this.d.log.error("eval worker error", { err });
+    }
+    /** The worker's `failed` event. */
+    onFailed(job, err) {
+        if (!job)
+            return;
+        // 处理函数外的暂时故障（PG / Redis）由 BullMQ 按 attempts 指数退避重试；用尽（或 stalled 超限）才算失败
+        if (job.attemptsMade < (job.opts.attempts ?? 1) && !/stalled/i.test(err.message)) {
+            this.d.log.warn("eval job attempt failed; retrying", { jobId: job.id, err });
+            return;
+        }
+        this.d.log.error("eval job failed", { jobId: job.id, err });
+        // 任务不会再执行：该用例剩下的结果记 error，运行不因此卡住
+        void this.failCase(job.data, err).catch((e) => this.d.log.error("cannot mark failed eval job", { jobId: job.id, err: e }));
     }
     /** 停止领取；中止本进程在执行的任务（回到队列由别的 worker 接着做），在 budgetMs 内等它们退出。 */
     async close(budgetMs) {
@@ -181,48 +254,36 @@ export class EvalWorker {
         await worker.pause(true).catch(() => { });
         for (const ac of this.active)
             ac.abort();
-        let timer;
-        const done = await Promise.race([
-            worker.close().then(() => true),
-            new Promise((resolve) => {
-                timer = setTimeout(() => resolve(false), Math.max(0, budgetMs));
-                timer.unref();
-            }),
-        ]);
-        clearTimeout(timer);
+        const done = await waitWithin(worker.close(), budgetMs);
         if (!done)
             await worker.close(true).catch(() => { });
-        this.conn?.disconnect();
     }
+    /** One job: replays one case (the processor runs it in `runJob`). */
     async process(job, token) {
         const data = job.data;
         const { runId, orgId } = data;
         const jobId = job.id ?? evalJobId(runId, data.caseId);
         if (!(await runActive(this.d.db, runId))) {
             await this.skipCase(data);
-            await recountCases(this.d.db, runId);
+            await this.d.budget.recountCases(runId);
             return;
         }
-        const limit = this.d.config().evals.orgConcurrency;
+        const limit = this.d.config.get().evals.orgConcurrency;
         // 每次执行一个新的认领号：失去认领（stalled 后被别的 worker 接手）的旧执行写不进结果（用量照样计入）。
         // 配额成员 = 任务 id + 认领号：旧执行退出时只移除自己的名额，不会把接手执行的名额一起移掉
         const claim = randomUUID();
         const member = `${jobId}-${claim}`;
-        if (!(await this.d.quota.tryEnter(orgId, member, limit, ORG_QUOTA_LEASE_MS))) {
+        // the slot is renewed every lease / 3 by the lock service's watchdog until released
+        const slot = await this.d.quota.enter(orgId, member, limit, ORG_QUOTA_LEASE_MS);
+        if (!slot) {
             // 配额满：固定 5 秒后再试，不按事件唤醒。评测是后台批任务，几秒空等可以接受，换来没有跨副本通知
-            await job.moveToDelayed(Date.now() + (this.d.quotaRetryMs ?? EVAL_QUOTA_RETRY_MS), token);
+            await job.moveToDelayed(Date.now() + EVAL_QUOTA_RETRY_MS, token);
             throw new DelayedError();
         }
         const ac = new AbortController();
         this.active.add(ac);
         if (this.closing)
             ac.abort();
-        const renew = setInterval(() => {
-            this.d.quota
-                .renew(orgId, member, ORG_QUOTA_LEASE_MS)
-                .catch((err) => this.d.log.warn("eval quota renew failed", { jobId, err }));
-        }, ORG_QUOTA_LEASE_MS / 3);
-        renew.unref();
         try {
             await this.d.db
                 .updateTable("eval_runs")
@@ -241,13 +302,12 @@ export class EvalWorker {
             throw err;
         }
         finally {
-            clearInterval(renew);
             this.active.delete(ac);
-            await this.d.quota
-                .leave(orgId, member)
+            await slot
+                .release()
                 .catch((err) => this.d.log.warn("eval quota leave failed", { jobId, err }));
         }
-        await recountCases(this.d.db, runId);
+        await this.d.budget.recountCases(runId);
     }
     /** 单步回放每次请求前：运行仍在执行（{@link runActive}）且结果仍由本次执行认领。 */
     async mayRequest(runId, resultId, claim) {
@@ -277,7 +337,7 @@ export class EvalWorker {
             .executeTakeFirst();
         if (Number(w.numUpdatedRows) === 0)
             return true;
-        const { tripped } = await recordOutcome(db, data.runId, false, this.d.config().evals.maxConsecutiveErrors);
+        const { tripped } = await recordOutcome(db, data.runId, false, this.d.config.get().evals.maxConsecutiveErrors);
         if (tripped) {
             this.d.log.warn("eval run stopped after too many consecutive errors", { runId: data.runId });
             await this.skipCase(data);
@@ -422,34 +482,7 @@ export class EvalWorker {
             const b = replyParts(result.baseline.content);
             const c = replyParts(result.candidate.content);
             const diff = structuralDiff({ ...b, stopReason: result.baseline.stopReason }, { ...c, stopReason: result.candidate.stopReason });
-            const outcome = await db.transaction().execute(async (trx) => {
-                // 只在仍由本次执行认领时写结果：stalled 后被接手的旧执行不覆盖结果。用量照样计入（请求确实发出且已付费，
-                // 与失败尝试一样不以认领为条件），走同一个原子 consumeTokens
-                const w = await trx
-                    .updateTable("eval_results")
-                    .set({
-                    status: "done",
-                    baseline: JSON.stringify(result.baseline),
-                    candidate: JSON.stringify(result.candidate),
-                    diff: JSON.stringify(diff),
-                    // 开了评委：评委做完之前用例不算做完（recountCases 不计 pending 的评委）
-                    judge_status: grade.judge ? "pending" : "none",
-                    drift: result.drift,
-                    error: null,
-                    updated_at: sql `now()`,
-                })
-                    .where("id", "=", r.id)
-                    .where("status", "=", "running")
-                    .where("claimed_by", "=", claim)
-                    .executeTakeFirst();
-                if (Number(w.numUpdatedRows) === 0) {
-                    await consumeTokens(trx, runId, tokens);
-                    return "lost";
-                }
-                await recordOutcome(trx, runId, true, this.d.config().evals.maxConsecutiveErrors);
-                const { capped } = await consumeTokens(trx, runId, tokens);
-                return capped ? "capped" : "ok";
-            });
+            const outcome = await this.writeStepResultIn(runId, r.id, claim, result, diff, grade, tokens);
             // 失去认领：这个用例已由别的执行接手，本次停下
             if (outcome === "lost")
                 return;
@@ -510,7 +543,7 @@ export class EvalWorker {
             try {
                 v = await judge({
                     purposes: this.d.purposes,
-                    maxOutputTokens: this.d.config().evals.judgeMaxOutputTokens,
+                    maxOutputTokens: this.d.config.get().evals.judgeMaxOutputTokens,
                 }, data.orgId, input, signal);
             }
             catch (err) {
@@ -608,8 +641,9 @@ export class EvalWorker {
             out = await replayRun({
                 ...this.d.runReplay,
                 stillClaimed: () => held(),
-                onSeeded: async (trx, conversationId) => {
-                    const w = await trx
+                // runs inside the seed's transaction (the context-aware handle joins it)
+                onSeeded: async (conversationId) => {
+                    const w = await this.d.db
                         .updateTable("eval_results")
                         .set({ eval_conversation_id: conversationId, updated_at: sql `now()` })
                         .where("id", "=", resultId)
@@ -659,33 +693,73 @@ export class EvalWorker {
         const baseText = replyParts(baseline.finalContent).text;
         const candText = replyParts(out.finalContent).text;
         const diff = structuralDiff({ toolCalls: baseline.toolCalls, text: baseText, stopReason: baseline.stopReason }, { toolCalls: out.toolCalls, text: candText, stopReason: out.stopReason });
-        const written = await db.transaction().execute(async (trx) => {
-            const w = await trx
-                .updateTable("eval_results")
-                .set({
-                status: "done",
-                baseline: JSON.stringify(baseline),
-                candidate: JSON.stringify(out),
-                diff: JSON.stringify(diff),
-                judge_status: grade.judge ? "pending" : "none",
-                drift,
-                error: null,
-                eval_conversation_id: out.conversationId,
-                updated_at: sql `now()`,
-            })
-                .where("id", "=", resultId)
-                .where("status", "=", "running")
-                .where("claimed_by", "=", claim)
-                .executeTakeFirst();
-            if (Number(w.numUpdatedRows) === 0)
-                return false;
-            await recordOutcome(trx, runId, true, this.d.config().evals.maxConsecutiveErrors);
-            return true;
+        const written = await this.writeRunResultIn(runId, resultId, claim, {
+            baseline,
+            out,
+            diff,
+            drift,
+            judge: grade.judge,
         });
         if (!written || !grade.judge)
             return;
         await this.judgeResult(data, resultId, claim, signal, judgeInputOf(s, "run", null, baseline, out, grade.rubric));
     }
+    /**
+     * 写一条单步回放的结果：只在仍由本次执行认领时写结果（stalled 后被接手的旧执行不覆盖结果）。用量照样计入（请求确实
+     * 发出且已付费，与失败尝试一样不以认领为条件），走同一个原子 consumeTokens。
+     */
+    async writeStepResultIn(runId, resultId, claim, result, diff, grade, tokens) {
+        const db = this.d.db;
+        const w = await db
+            .updateTable("eval_results")
+            .set({
+            status: "done",
+            baseline: JSON.stringify(result.baseline),
+            candidate: JSON.stringify(result.candidate),
+            diff: JSON.stringify(diff),
+            // 开了评委：评委做完之前用例不算做完（recountCases 不计 pending 的评委）
+            judge_status: grade.judge ? "pending" : "none",
+            drift: result.drift,
+            error: null,
+            updated_at: sql `now()`,
+        })
+            .where("id", "=", resultId)
+            .where("status", "=", "running")
+            .where("claimed_by", "=", claim)
+            .executeTakeFirst();
+        if (Number(w.numUpdatedRows) === 0) {
+            await consumeTokens(db, runId, tokens);
+            return "lost";
+        }
+        await recordOutcome(db, runId, true, this.d.config.get().evals.maxConsecutiveErrors);
+        const { capped } = await consumeTokens(db, runId, tokens);
+        return capped ? "capped" : "ok";
+    }
+    /** 写一条整轮回放的结果（仍由本次执行认领时）；false = 已失去认领。 */
+    async writeRunResultIn(runId, resultId, claim, r) {
+        const db = this.d.db;
+        const w = await db
+            .updateTable("eval_results")
+            .set({
+            status: "done",
+            baseline: JSON.stringify(r.baseline),
+            candidate: JSON.stringify(r.out),
+            diff: JSON.stringify(r.diff),
+            judge_status: r.judge ? "pending" : "none",
+            drift: r.drift,
+            error: null,
+            eval_conversation_id: r.out.conversationId,
+            updated_at: sql `now()`,
+        })
+            .where("id", "=", resultId)
+            .where("status", "=", "running")
+            .where("claimed_by", "=", claim)
+            .executeTakeFirst();
+        if (Number(w.numUpdatedRows) === 0)
+            return false;
+        await recordOutcome(db, runId, true, this.d.config.get().evals.maxConsecutiveErrors);
+        return true;
+    }
     /**
      * 该用例剩下的结果标 skipped。整轮回放留下的评测对话里若还有没结束的运行（上一次执行崩溃在回放中途），删掉它，
      * 不留永远「执行中」的评测对话；已正常收尾（`run_aborted` / `run_finished`）的保留，轨迹里能看到停在哪里。
@@ -773,17 +847,49 @@ export class EvalWorker {
             .execute();
         // 已 done 但评委一直没做成的：评委记 skipped（结果本身保留）
         await skipPendingJudges(this.d.db, data.runId, data.caseId);
-        await recountCases(this.d.db, data.runId);
+        await this.d.budget.recountCases(data.runId);
     }
-}
+};
+__decorate([
+    Transactional({ propagation: "requires_new" })
+], EvalWorker.prototype, "writeStepResultIn", null);
+__decorate([
+    Transactional({ propagation: "requires_new" })
+], EvalWorker.prototype, "writeRunResultIn", null);
+EvalWorker = __decorate([
+    Injectable(),
+    __param(0, Inject(KernelDb)),
+    __param(1, Inject(EVAL_QUOTA)),
+    __param(2, Inject(KernelConfigService)),
+    __param(3, Inject(KERNEL_LOGGER)),
+    __param(4, Inject(ModelPurposesService)),
+    __param(5, Inject(ProviderRegistry)),
+    __param(6, Inject(RunSteps)),
+    __param(7, Inject(AgentLoops)),
+    __param(8, Inject(EventLog)),
+    __param(9, Inject(RunFinish)),
+    __param(10, Inject(KyselyCheckpointSaver)),
+    __param(11, Inject(EvalReplaySeeder)),
+    __param(12, Inject(EvalBudget)),
+    __param(13, Inject(TransactionHost)),
+    __param(14, Optional()),
+    __param(14, Inject(EVAL_REPLAY_HOOKS))
+], EvalWorker);
+export { EvalWorker };
 /**
  * A plugin template's child case: the template's current version, which a new dispatch would use
- * (a template removed since gets version 0, so it always differs). Undefined for other cases.
+ * (a template removed since gets version 0, so it always differs), with the platform template
+ * whose content that is (absent: the org's own; a link or pin change under a copy's id changes
+ * it). Undefined for other cases.
  */
 export async function currentTemplate(db, s) {
     const t = subagentStartedOf(toEvents(s))?.template;
     if (t?.source !== "plugin" || t.id === undefined)
         return undefined;
     const row = (await loadTemplates(db, s.orgId, [t.id])).get(t.id);
-    return { id: t.id, version: row?.version ?? 0 };
+    return {
+        id: t.id,
+        version: row?.version ?? 0,
+        ...(row?.platformTemplateId !== undefined && { platformTemplateId: row.platformTemplateId }),
+    };
 }