@rulemetric/cli 0.2.0 → 0.2.2

package/dist/lib/handlers/cron-suggest-instructions.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"cron-suggest-instructions.js","sourceRoot":"","sources":["../../../src/lib/handlers/cron-suggest-instructions.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,yBAAyB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAEzD,MAAM,2BAA2B,GAAG,CAAC,CAAC;AACtC,MAAM,oBAAoB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC;AAC1F,MAAM,qBAAqB,GAAG,EAAE,CAAC;AACjC,MAAM,SAAS,GAAG,GAAG,CAAC;AACtB,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAwBlC,KAAK,UAAU,wBAAwB;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,YAAY,EAAE,sBAAsB,CAAC,CAAC;IAC9F,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAuB,CAAC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,CAAC;QACpE,OAAO,KAAK,GAAG,qBAAqB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,uBAAuB,GAAS,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE;IACvE,0EAA0E;IAC1E,sEAAsE;IACtE,yEAAyE;IACzE,8BAA8B;IAC9B,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;IACnC,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,wGAAwG,CAAC,CAAC;QAC9H,OAAO;IACT,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;IAEnD,gCAAgC;IAChC,IAAI,YAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAuB,oCAAoC,GAAG,oBAAoB,CAAC,CAAC;QAC5G,YAAY,GAAG,GAAG,CAAC,YAAY,CAAC;IAClC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1G,OAAO;IACT,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;QACjE,OAAO;IACT,CAAC;IAED,uEAAuE;IACvE,wEAAwE;IACxE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC3D,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAe,wBAAwB,CAAC,CAAC;QACzE,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE;gBACrB,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;gBAClC,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;gBACpB,cAAc,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC;aACpE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACtG,CAAC;IACD,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,MAAM,MAAM,CAAiC,6BAA6B,CAAC,CAAC;QAChG,KAAK,MAAM,GAAG,IAAI,WAAW,CAAC,YAAY,IAAI,EAAE,EAAE,CAAC;YACjD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC5B,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE;oBACrB,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,IAAI,EAAE,GAAG,CAAC,IAAI;oBACd,WAAW,EAAE,GAAG,CAAC,WAAW,IAAI,EAAE;oBAClC,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;oBACpB,cAAc,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC;iBACpE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,0CAA0C,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpH,CAAC;IAED,MAAM,aAAa,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC/C,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;QACnE,OAAO;IACT,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,aAAa,CAAC,MAAM,sBAAsB,YAAY,CAAC,MAAM,aAAa,CAAC,CAAC;IAE3G,uBAAuB;IACvB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,WAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC,CAAC;YAClD,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC;YAC/C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,YAAY,WAAW,iCAAiC,CAAC,CAAC;gBAC9E,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAmB;gBAC9B,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAChC,WAAW,EAAE,IAAI;gBACjB,SAAS;gBACT,MAAM,EAAE,EAAE;gBACV,YAAY,EAAE,EAAE;aACjB,CAAC;YAEF,MAAM,MAAM,GAAG,yBAAyB,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;YACjE,2EAA2E;YAC3E,4DAA4D;YAC5D,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;YACpD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,yBAAyB,WAAW,aAAa,OAAO,GAAG,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,eAAe,IAAI,EAAE,CAAmC,CAAC;YAC7E,MAAM,eAAe,GAAqB,GAAG;iBAC1C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,aAAa,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC;iBACjH,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACX,aAAa,EAAE,CAAC,CAAC,aAAuB;gBACxC,KAAK,EAAE,CAAC,CAAC,KAAe;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAgB;aAC3B,CAAC,CAAC,CAAC;YAEN,MAAM,GAAG,GAAG,eAAe;iBACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,SAAS,CAAC;iBACnC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;iBACjC,KAAK,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;YAEzC,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACrB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,qCAAqC,WAAW,EAAE,CAAC,CAAC;gBACxE,SAAS;YACX,CAAC;YAED,MAAM,OAAO,CAAsB,2CAA2C,EAAE;gBAC9E,WAAW;gBACX,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3B,aAAa,EAAE,CAAC,CAAC,aAAa;oBAC9B,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM,EAAE,CAAC,CAAC,MAAM;iBACjB,CAAC,CAAC;aACJ,CAAC,CAAC;YACH,MAAM,EAAE,CAAC;QACX,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,OAAO,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,IAAI,MAAM,wBAAwB,EAAE,EAAE,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,yCAAyC,EAAE,EAAE,CAAC,CAAC;YAC7D,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QACvF,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,MAAM,IAAI,YAAY,CAAC,MAAM,WAAW,CAAC,CAAC;AAChF,CAAC,CAAC"}
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/handlers/process-announcement.js

@@ -0,0 +1,11 @@
+import {
+  processAnnouncement
+} from "../../chunk-GQCISMAH.js";
+import "../../chunk-DGHWRQXL.js";
+import "../../chunk-NIDLJO6F.js";
+import "../../chunk-7FAA3YNB.js";
+import "../../chunk-NSBPE2FW.js";
+export {
+  processAnnouncement
+};
+//# sourceMappingURL=process-announcement.js.map

package/dist/lib/handlers/process-announcement.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/handlers/process-changelog.js

@@ -0,0 +1,11 @@
+import {
+  processChangelog
+} from "../../chunk-R5MFHGHR.js";
+import "../../chunk-K3BFZ7KG.js";
+import "../../chunk-NIDLJO6F.js";
+import "../../chunk-7FAA3YNB.js";
+import "../../chunk-NSBPE2FW.js";
+export {
+  processChangelog
+};
+//# sourceMappingURL=process-changelog.js.map

package/dist/lib/handlers/process-changelog.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/handlers/process-conversation.js

@@ -1,35 +1,14 @@
-import { apiPatch } from '../api-client.js';
-import { processConversationJob } from '../eval-conversation-handler.js';
-/**
- * Graphile-shaped handler for the `process_conversation` task.
- *
- * Thin wrapper around `eval-conversation-handler.processConversationJob` —
- * the existing function already does all the work (action decision,
- * run_evals/generate_cases/analyze branches, idempotency guard via
- * already-posted assistant messages). The handler just unpacks the payload
- * and threads the helpers' logger through.
- *
- * Mechanical contract vs. the legacy worker:
- *   - no claim — Graphile claimed the job before invoking us
- *   - no /complete POST — we return a small marker on success (matches the
- *     `{ type: 'conversation' }` blob the legacy code POSTed)
- *   - no surrounding try/catch — errors propagate to Graphile, which flips
- *     the mirror row to `failed` via the runner.events handler
- *
- * Conversation jobs run with `maxAttempts: 1` (set at enqueue time in
- * `apps/api/src/lib/queue.ts`) — chat is interactive and silent retries
- * would confuse users mid-flow.
- */
-export async function processConversation(payload, helpers) {
-    const { mirrorId, evalTargetId, config } = payload;
-    const log = (msg) => helpers.logger.info(msg);
-    await processConversationJob(config, evalTargetId, log);
-    const result = { type: 'conversation', conversationId: config.conversationId };
-    // Graphile discards task return values, so persist a small marker onto
-    // the mirror row so the dashboard can tell the job finished. Best-effort.
-    await apiPatch(`/api/eval-jobs/${mirrorId}/state`, { result }).catch((err) => {
-        log(`result persist failed: ${err instanceof Error ? err.message : String(err)}`);
-    });
-    return result;
-}
-//# sourceMappingURL=process-conversation.js.map
+import {
+  processConversation
+} from "../../chunk-BD7CY42U.js";
+import "../../chunk-IWYCRLEK.js";
+import "../../chunk-3TIMQ3O6.js";
+import "../../chunk-YNIH3KMU.js";
+import "../../chunk-QN34TGD3.js";
+import "../../chunk-EJOI2ZGU.js";
+import "../../chunk-KZRFUXGF.js";
+import "../../chunk-NSBPE2FW.js";
+export {
+  processConversation
+};
+//# sourceMappingURL=process-conversation.js.map

package/dist/lib/handlers/process-conversation.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"process-conversation.js","sourceRoot":"","sources":["../../../src/lib/handlers/process-conversation.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAezE;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,OAA4B,EAC5B,OAAmB;IAEnB,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEtD,MAAM,sBAAsB,CAAC,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,EAAE,IAAI,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;IAE/E,uEAAuE;IACvE,0EAA0E;IAC1E,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACpF,GAAG,CAAC,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}

package/dist/lib/handlers/process-eval.js

@@ -1,207 +1,15 @@
-import { apiGet, apiPost, apiPatch } from '../api-client.js';
-import { executeEval } from '../eval-executor.js';
-import { gradeEvalOutput } from '../eval-grader.js';
-import { judgeEvalQuality } from '../eval-meta-judge.js';
-import { createEvalWorktree } from '../eval-worktree.js';
-import { PermanentJobError } from '../permanent-job-error.js';
-/**
- * Graphile-shaped handler for the `process_eval` task.
- *
- * Lifted verbatim from the legacy `eval-worker.ts:processJob` body, with the
- * queue plumbing stripped out:
- *   - no claim — Graphile already claimed the job before invoking us
- *   - no /complete POST — we return the result object (or throw)
- *   - no `processing` flag — that lived in the polling loop's closure
- *   - no conversation branch — split into its own handler (Task 14)
- *
- * Mid-job progress patches still hit the existing
- * `PATCH /api/eval-jobs/:id/progress` endpoint, just keyed by the mirror row's
- * id instead of the legacy queue id.
- */
-export async function processEval(payload, helpers) {
-    const { mirrorId, evalTargetId, config } = payload;
-    const log = (msg) => helpers.logger.info(msg);
-    // Fail-fast on retry. Each eval_run is INSERT'd as it completes, but
-    // there's no idempotency key tying a row to a specific job — re-running
-    // the whole batch would duplicate every previously-completed row in
-    // eval_runs. Until proper resume-from-progress-checkpoint lands, surface
-    // the failure to the user instead of silently doubling-up rows. See
-    // docs/audits/2026-05-07-worker-audit.md "Handlers not idempotent".
-    if (helpers.job.attempts > 1) {
-        throw new PermanentJobError(`process_eval is not safe to retry (attempt ${helpers.job.attempts}). Re-trigger the eval run from the UI.`);
-    }
-    const target = await apiGet(`/api/eval-targets/${evalTargetId}`);
-    const evalCases = await apiGet(`/api/eval-targets/${evalTargetId}/evals`);
-    if (evalCases.length === 0) {
-        // No retries — eval cases don't materialize on their own; the user has
-        // to add one via the UI/CLI. Skip Graphile's retry loop.
-        throw new PermanentJobError('No evals defined for target');
-    }
-    const runsPerEval = config.runsPerEval ?? 1;
-    const timeout = config.timeout ?? 120_000;
-    const iteration = config.iteration ?? 1;
-    const configurations = [
-        { name: 'with_target', content: target.content },
-    ];
-    if (!config.skipBaseline) {
-        configurations.push({ name: 'without_target' });
-    }
-    // Build work queue
-    const workQueue = [];
-    for (const eval_ of evalCases) {
-        for (const cfg of configurations) {
-            for (let run = 1; run <= runsPerEval; run++) {
-                workQueue.push({ eval_: eval_, configuration: cfg.name, targetContent: cfg.content, runNumber: run });
-            }
-        }
-    }
-    const total = workQueue.length;
-    let completed = 0;
-    const concurrency = Math.min(config.parallel ?? 1, 4); // cap at 4 parallel
-    await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
-        status: 'running',
-        progress: { total, completed: 0, current: null, message: `Starting ${total} eval runs (concurrency: ${concurrency})...` },
-    });
-    const results = [];
-    // Process work items — sequential if concurrency=1, parallel otherwise
-    async function processItem(item) {
-        const label = `${item.configuration} · ${item.eval_.name} · run#${item.runNumber}`;
-        log(`[${completed + 1}/${total}] ${label}`);
-        // Create isolated worktree for parallel execution
-        const worktree = concurrency > 1 ? createEvalWorktree() : null;
-        try {
-            const execResult = await executeEval({
-                prompt: item.eval_.prompt,
-                targetContent: item.targetContent,
-                workDir: worktree?.workDir,
-                timeout,
-                model: config.model,
-            });
-            let grading = {};
-            if (!config.skipGrading && item.eval_.expectations.length > 0 && !execResult.error) {
-                const gradingResult = await gradeEvalOutput({
-                    prompt: item.eval_.prompt,
-                    output: execResult.output,
-                    expectations: item.eval_.expectations,
-                    model: config.graderModel ?? config.model,
-                    timeout,
-                });
-                grading = gradingResult;
-            }
-            return {
-                evalId: item.eval_.id,
-                evalTargetId: target.id,
-                evalTargetVersion: target.version,
-                configuration: item.configuration,
-                runNumber: item.runNumber,
-                iteration,
-                status: execResult.error ? 'failed' : 'completed',
-                outputs: execResult.error ? { error: execResult.error } : { transcript: execResult.output },
-                timing: { duration_ms: execResult.durationMs, total_tokens: execResult.totalTokens },
-                grading,
-                worktree: worktree?.isWorktree ? true : undefined,
-            };
-        }
-        finally {
-            worktree?.cleanup();
-        }
-    }
-    if (concurrency <= 1) {
-        // Sequential execution (existing behavior)
-        for (const item of workQueue) {
-            const label = `${item.configuration} · ${item.eval_.name} · run#${item.runNumber}`;
-            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
-                status: 'running',
-                progress: { total, completed, current: label, message: `Executing ${label}...` },
-            });
-            const runResult = await processItem(item);
-            await apiPost('/api/eval-runs', runResult);
-            results.push(runResult);
-            completed++;
-            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
-                status: 'running',
-                progress: { total, completed, current: label, message: `Completed ${label}` },
-            });
-        }
-    }
-    else {
-        // Parallel execution with worktree isolation
-        log(`Running ${total} evals with concurrency=${concurrency} (worktree isolation)`);
-        for (let i = 0; i < workQueue.length; i += concurrency) {
-            const batch = workQueue.slice(i, i + concurrency);
-            const batchLabel = `batch ${Math.floor(i / concurrency) + 1}/${Math.ceil(workQueue.length / concurrency)}`;
-            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
-                status: 'running',
-                progress: { total, completed, current: batchLabel, message: `Executing ${batchLabel} (${batch.length} parallel)...` },
-            });
-            const batchResults = await Promise.all(batch.map(processItem));
-            for (const runResult of batchResults) {
-                await apiPost('/api/eval-runs', runResult);
-                results.push(runResult);
-                completed++;
-            }
-            await apiPatch(`/api/eval-jobs/${mirrorId}/progress`, {
-                status: 'running',
-                progress: { total, completed, current: batchLabel, message: `Completed ${batchLabel}` },
-            });
-        }
-    }
-    const summary = buildSummary(results);
-    // Meta-judge: assess eval quality if we have grading results
-    let metaJudgement;
-    if (!config.skipGrading && results.some(r => r.grading && Object.keys(r.grading).length > 0)) {
-        try {
-            log(`Running meta-judge for job ${mirrorId}...`);
-            const evalInputs = evalCases.map(e => {
-                const matchingResults = results.filter(r => r.evalId === e.id && r.configuration === 'with_target');
-                const firstResult = matchingResults[0];
-                return {
-                    evalName: e.name,
-                    prompt: e.prompt,
-                    expectations: e.expectations,
-                    grading: firstResult?.grading ? firstResult.grading : null,
-                    output: firstResult?.outputs?.transcript?.slice(0, 500),
-                };
-            });
-            metaJudgement = await judgeEvalQuality({
-                targetName: target.name,
-                evalResults: evalInputs,
-                model: config.graderModel ?? config.model,
-                timeout: 60000,
-            });
-            log(`Meta-judge: quality=${metaJudgement.quality_score}/5, issues=${metaJudgement.issues.length}`);
-        }
-        catch (err) {
-            log(`Meta-judge failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`);
-        }
-    }
-    const result = metaJudgement
-        ? { ...summary, meta_judgement: metaJudgement }
-        : summary;
-    log(`Job ${mirrorId} completed: ${completed}/${total} runs`);
-    // Graphile discards task return values, so persist the summary onto the
-    // mirror row here. Best-effort: a failure to write the result must not
-    // mark the job as failed (it succeeded; we just lose the summary blob).
-    await apiPatch(`/api/eval-jobs/${mirrorId}/state`, { result }).catch((err) => {
-        log(`result persist failed: ${err instanceof Error ? err.message : String(err)}`);
-    });
-    return result;
-}
-function buildSummary(results) {
-    const completed = results.filter((r) => r.status === 'completed');
-    const failed = results.filter((r) => r.status === 'failed');
-    const gradedRuns = completed.filter((r) => {
-        const grading = r.grading;
-        return grading && Object.keys(grading).length > 0;
-    });
-    const avgPassRate = gradedRuns.length > 0
-        ? gradedRuns.reduce((sum, r) => sum + (r.grading?.summary?.pass_rate ?? 0), 0) / gradedRuns.length
-        : null;
-    return {
-        total: results.length,
-        completed: completed.length,
-        failed: failed.length,
-        avgPassRate,
-    };
-}
-//# sourceMappingURL=process-eval.js.map
+import {
+  processEval
+} from "../../chunk-QJXAULRC.js";
+import "../../chunk-DGHWRQXL.js";
+import "../../chunk-J7N3DLH6.js";
+import "../../chunk-52WIYRZH.js";
+import "../../chunk-YNIH3KMU.js";
+import "../../chunk-QN34TGD3.js";
+import "../../chunk-EJOI2ZGU.js";
+import "../../chunk-KZRFUXGF.js";
+import "../../chunk-NSBPE2FW.js";
+export {
+  processEval
+};
+//# sourceMappingURL=process-eval.js.map

package/dist/lib/handlers/process-eval.js.map

@@ -1 +1,7 @@
-{"version":3,"file":"process-eval.js","sourceRoot":"","sources":["../../../src/lib/handlers/process-eval.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAsB,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,gBAAgB,EAAsB,MAAM,uBAAuB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAiC9D;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAoB,EACpB,OAAmB;IAEnB,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;IACnD,MAAM,GAAG,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAEtD,qEAAqE;IACrE,wEAAwE;IACxE,oEAAoE;IACpE,yEAAyE;IACzE,oEAAoE;IACpE,oEAAoE;IACpE,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,iBAAiB,CACzB,8CAA8C,OAAO,CAAC,GAAG,CAAC,QAAQ,yCAAyC,CAC5G,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,MAAM,CAAa,qBAAqB,YAAY,EAAE,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,MAAM,MAAM,CAAS,qBAAqB,YAAY,QAAQ,CAAC,CAAC;IAElF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,uEAAuE;QACvE,yDAAyD;QACzD,MAAM,IAAI,iBAAiB,CAAC,6BAA6B,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,OAAO,CAAC;IAC1C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;IAExC,MAAM,cAAc,GAA8C;QAChE,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE;KACjD,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;QACzB,cAAc,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,mBAAmB;IACnB,MAAM,SAAS,GAA6F,EAAE,CAAC;IAC/G,KAAK,MAAM,KAAK,IAAI,SAAS,EAAE,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;YACjC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,WAAW,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC5C,SAAS,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,CAAC,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;IAC/B,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAE3E,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;QACpD,MAAM,EAAE,SAAS;QACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,YAAY,KAAK,4BAA4B,WAAW,MAAM,EAAE;KAC1H,CAAC,CAAC;IAEH,MAAM,OAAO,GAA8B,EAAE,CAAC;IAE9C,uEAAuE;IACvE,KAAK,UAAU,WAAW,CAAC,IAAyB;QAClD,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,aAAa,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;QACnF,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK,KAAK,KAAK,EAAE,CAAC,CAAC;QAE5C,kDAAkD;QAClD,MAAM,QAAQ,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAE/D,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC;gBACnC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,OAAO,EAAE,QAAQ,EAAE,OAAO;gBAC1B,OAAO;gBACP,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC;YAEH,IAAI,OAAO,GAA4B,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnF,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC;oBAC1C,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM;oBACzB,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,YAAY;oBACrC,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK;oBACzC,OAAO;iBACR,CAAC,CAAC;gBACH,OAAO,GAAG,aAAmD,CAAC;YAChE,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE;gBACrB,YAAY,EAAE,MAAM,CAAC,EAAE;gBACvB,iBAAiB,EAAE,MAAM,CAAC,OAAO;gBACjC,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,SAAS;gBACT,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW;gBACjD,OAAO,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,MAAM,EAAE;gBAC3F,MAAM,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC,UAAU,EAAE,YAAY,EAAE,UAAU,CAAC,WAAW,EAAE;gBACpF,OAAO;gBACP,QAAQ,EAAE,QAAQ,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;aAClD,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,QAAQ,EAAE,OAAO,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,2CAA2C;QAC3C,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,aAAa,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;YACnF,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,KAAK,KAAK,EAAE;aACjF,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,OAAO,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;YAC3C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,SAAS,EAAE,CAAC;YAEZ,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE;aAC9E,CAAC,CAAC;QACL,CAAC;IACH,CAAC;SAAM,CAAC;QACN,6CAA6C;QAC7C,GAAG,CAAC,WAAW,KAAK,2BAA2B,WAAW,uBAAuB,CAAC,CAAC;QACnF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;YAClD,MAAM,UAAU,GAAG,SAAS,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,WAAW,CAAC,EAAE,CAAC;YAE3G,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,UAAU,KAAK,KAAK,CAAC,MAAM,eAAe,EAAE;aACtH,CAAC,CAAC;YAEH,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;YAE/D,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;gBACrC,MAAM,OAAO,CAAC,gBAAgB,EAAE,SAAS,CAAC,CAAC;gBAC3C,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxB,SAAS,EAAE,CAAC;YACd,CAAC;YAED,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,WAAW,EAAE;gBACpD,MAAM,EAAE,SAAS;gBACjB,QAAQ,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,UAAU,EAAE,EAAE;aACxF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IAEtC,6DAA6D;IAC7D,IAAI,aAAwC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAiB,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QACvG,IAAI,CAAC;YACH,GAAG,CAAC,8BAA8B,QAAQ,KAAK,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;gBACnC,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,aAAa,KAAK,aAAa,CAAC,CAAC;gBACpG,MAAM,WAAW,GAAG,eAAe,CAAC,CAAC,CAAC,CAAC;gBACvC,OAAO;oBACL,QAAQ,EAAE,CAAC,CAAC,IAAI;oBAChB,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,YAAY,EAAE,CAAC,CAAC,YAAY;oBAC5B,OAAO,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,OAAmC,CAAC,CAAC,CAAC,IAAI;oBACtF,MAAM,EAAG,WAAW,EAAE,OAAkC,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;iBACpF,CAAC;YACJ,CAAC,CAAC,CAAC;YACH,aAAa,GAAG,MAAM,gBAAgB,CAAC;gBACrC,UAAU,EAAE,MAAM,CAAC,IAAI;gBACvB,WAAW,EAAE,UAAU;gBACvB,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,KAAK;gBACzC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YACH,GAAG,CAAC,uBAAuB,aAAa,CAAC,aAAa,cAAc,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACrG,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,kCAAkC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,aAAa;QAC1B,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE;QAC/C,CAAC,CAAC,OAAO,CAAC;IACZ,GAAG,CAAC,OAAO,QAAQ,eAAe,SAAS,IAAI,KAAK,OAAO,CAAC,CAAC;IAE7D,wEAAwE;IACxE,uEAAuE;IACvE,wEAAwE;IACxE,MAAM,QAAQ,CAAC,kBAAkB,QAAQ,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;QACpF,GAAG,CAAC,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,OAAkC;IACtD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;IAE5D,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,CAAC,CAAC,OAA8B,CAAC;QACjD,OAAO,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC;QACvC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAE,CAAC,CAAC,OAAe,EAAE,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;QAC3G,CAAC,CAAC,IAAI,CAAC;IAET,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,SAAS,EAAE,SAAS,CAAC,MAAM;QAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW;KACZ,CAAC;AACJ,CAAC"}
+{
+  "version": 3,
+  "sources": [],
+  "sourcesContent": [],
+  "mappings": "",
+  "names": []
+}