@agentdeck/bridge 0.2.2 → 1.0.0

package/dist/apme/runner.js

@@ -6,16 +6,34 @@
  * 0/1 in `evals` (metrics: lint_clean, build_ok, tests_pass).
  *
  * Layer 2 (llm_judge): G-Eval style rubric against the latest `rubrics` row.
- * Backend is pluggable — default is local MLX (cost-free), API/OpenClaw are
- * opt-in via `~/.agentdeck/settings.json`. Gated by `shouldJudge()` so the
- * common "clear pass" case skips layer 2 entirely.
+ * Backend is pluggable — default is Foundation Models via the Swift daemon or
+ * bundled CLI Swift helper, with local MLX fallback for CLI-only runs. Gated
+ * by `shouldJudge()` so the common "clear pass" case skips layer 2 entirely.
  */
 import { spawn } from 'child_process';
 import { existsSync, readdirSync } from 'fs';
 import { debug } from '../logger.js';
-import { loadApmeConfig, shouldJudge } from './settings.js';
+import { loadApmeConfig, shouldJudge, DEFAULT_APME_CONFIG } from './settings.js';
 import { loadMlxSettings, mlxChatUrl } from '@agentdeck/shared';
+import { callFoundationModelsHelper, probeFoundationModelsHelper } from '../foundation-models-helper.js';
+import { runSampleScorers } from './scorers/index.js';
 import { execSync } from 'child_process';
+function deriveTaskOutcome(score) {
+    if (score == null)
+        return 'pending';
+    if (score >= 0.75)
+        return 'success';
+    if (score >= 0.5)
+        return 'partial';
+    return 'fail';
+}
+/** Narrow an arbitrary DB `outcome` string to the typed union, so a row
+ *  written via the manual path (`closeTaskExternal` outcome override) can
+ *  flow back into `TaskEvaluatedEvent` without losing type-safety. */
+function isPreservableOutcome(value) {
+    return value === 'success' || value === 'partial' || value === 'fail'
+        || value === 'pending' || value === 'abandoned';
+}
 const DEFAULT_COMMANDS = {
     typescript: {
         lint: 'pnpm -w lint',
@@ -38,13 +56,27 @@ export class ApmeRunner {
     store;
     queue = [];
     drainPromise = null;
+    queuedRunIds = new Set();
+    runningRunIds = new Set();
+    runningTurnIds = new Set();
+    runningTaskIds = new Set();
     listeners = new Set();
+    taskListeners = new Set();
     configOverride = null;
     judgeOverride = null;
     detOverride = null;
+    /** Cached startup judge readiness probe. Populated by `refreshBackendProbe`,
+     *  surfaced on /health. Null until the first probe completes. */
+    lastBackendProbe = null;
     constructor(store) {
         this.store = store;
     }
+    /** Probe the configured judge backend and cache the result. Safe to call
+     *  fire-and-forget at daemon startup — failures don't throw. */
+    async refreshBackendProbe(cfg) {
+        this.lastBackendProbe = await probeJudgeBackend(cfg);
+        return this.lastBackendProbe;
+    }
     _setConfig(cfg) { this.configOverride = cfg; }
     _setJudgeFn(fn) {
         this.judgeOverride = fn;
@@ -56,9 +88,23 @@ export class ApmeRunner {
         this.listeners.add(fn);
         return () => this.listeners.delete(fn);
     }
+    /** Subscribe to per-task judge completions. The event fires AFTER the
+     *  task_judge axis scores and summary are persisted, so listeners can read
+     *  the final state directly from the event payload without re-querying.
+     *  Used by the timeline emitter to upsert the corresponding `task_end` row
+     *  with score + outcome metadata. */
+    onTaskEvaluated(fn) {
+        this.taskListeners.add(fn);
+        return () => this.taskListeners.delete(fn);
+    }
     enqueue(job) {
         if (!this.store.enabled)
             return;
+        if (this.queuedRunIds.has(job.runId) || this.runningRunIds.has(job.runId)) {
+            debug('APME', `skip duplicate eval enqueue runId=${job.runId}`);
+            return;
+        }
+        this.queuedRunIds.add(job.runId);
         this.queue.push(job);
         debug('APME', `enqueue eval runId=${job.runId} (queue=${this.queue.length})`);
         void this.drain();
@@ -69,7 +115,14 @@ export class ApmeRunner {
     enqueueTurn(job) {
         if (!this.store.enabled)
             return;
-        void this.runTurnEval(job);
+        if (this.runningTurnIds.has(job.turnId)) {
+            debug('APME', `skip duplicate turn eval turnId=${job.turnId}`);
+            return;
+        }
+        this.runningTurnIds.add(job.turnId);
+        void this.runTurnEval(job).finally(() => {
+            this.runningTurnIds.delete(job.turnId);
+        });
     }
     /** Judge a closed task (group of turns between boundary signals —
      *  TodoWrite all-completed, /clear, session_end). Fires-and-forgets;
@@ -77,7 +130,14 @@ export class ApmeRunner {
     enqueueTask(job) {
         if (!this.store.enabled)
             return;
-        void this.runTaskEval(job);
+        if (this.runningTaskIds.has(job.taskId)) {
+            debug('APME', `skip duplicate task eval taskId=${job.taskId}`);
+            return;
+        }
+        this.runningTaskIds.add(job.taskId);
+        void this.runTaskEval(job).finally(() => {
+            this.runningTaskIds.delete(job.taskId);
+        });
     }
     async runTaskEval({ runId, taskId, category, boundarySignal }) {
         const cfg = this.configOverride ?? loadApmeConfig();
@@ -121,6 +181,13 @@ export class ApmeRunner {
         if (turns.length > TURN_CAP) {
             lines.push(`… (${turns.length - TURN_CAP} more turns omitted)`);
         }
+        // The canonical SessionSample gives the judge the typed tool trajectory
+        // (req #5/#6) and per-sample cost (req #7) — context the turn rows lack.
+        const sample = this.store.getSample(taskId);
+        const trajectoryLines = sample ? buildTrajectoryLines(sample) : [];
+        const costLine = sample
+            ? `cost: ${sample.cost.inputTokens}in/${sample.cost.outputTokens}out tok, $${sample.cost.costUsd.toFixed(4)}, model ${sample.model.modelId}`
+            : '';
         const judgePrompt = [
             rubric.prompt,
             '',
@@ -128,23 +195,28 @@ export class ApmeRunner {
             `task_category: ${category ?? task.taskCategory ?? 'unknown'}`,
             `turn_count: ${turns.length}`,
             `boundary_signal: ${boundarySignal ?? task.boundarySignal}`,
+            ...(costLine ? [costLine] : []),
             '',
             '--- TURNS ---',
             ...lines,
+            ...(trajectoryLines.length ? ['', '--- TOOL TRAJECTORY ---', ...trajectoryLines] : []),
             '',
             'Respond with strict JSON only.',
         ].join('\n');
         try {
-            const judgeText = this.judgeOverride
-                ? await this.judgeOverride(judgePrompt, cfg.judge)
-                : await callJudge(judgePrompt, cfg.judge);
-            const parsed = parseJudgeJson(judgeText);
+            // callJudgeWithMeta carries the effective backend label across the
+            // FM→MLX fallback path. Without it, eval rows produced by the MLX
+            // fallback would be misattributed to foundationModels in the DB.
+            const judgeResult = this.judgeOverride
+                ? { text: await this.judgeOverride(judgePrompt, cfg.judge), effectiveLabel: effectiveJudgeModelTag(cfg.judge) }
+                : await callJudgeWithMeta(judgePrompt, cfg.judge);
+            const parsed = parseJudgeJson(judgeResult.text);
             if (!parsed) {
                 debug('APME', `runTaskEval parse failed task=${taskId.slice(0, 8)}`);
                 return;
             }
             const now = Date.now();
-            const judgeModel = effectiveJudgeModelTag(cfg.judge);
+            const judgeModel = judgeResult.effectiveLabel;
             for (const [axis, score] of Object.entries(parsed.scores)) {
                 this.store.insertEvalForTask({
                     id: 0,
@@ -165,22 +237,85 @@ export class ApmeRunner {
                     createdAt: now,
                 });
             }
+            // Pure sample-trajectory scorers (tool churn, error rate) — they add
+            // signal the LLM judge can miss and are cheap/deterministic. Stored under
+            // layer='trajectory' so they don't collide with task_judge axes.
+            if (sample) {
+                try {
+                    for (const r of runSampleScorers(sample)) {
+                        this.store.insertEvalForTask({
+                            id: 0, runId, taskId,
+                            layer: r.layer,
+                            metric: r.metric,
+                            score: r.score,
+                            raw: r.reasoning ? JSON.stringify({ reasoning: r.reasoning, scorer: r.scorer }) : null,
+                            rubricVer: null,
+                            judgeModel: `scorer:${r.scorer}`,
+                            createdAt: now,
+                        });
+                    }
+                }
+                catch (err) {
+                    debug('APME', `sample scorers failed task=${taskId.slice(0, 8)}: ${String(err)}`);
+                }
+            }
+            const compositeScore = parsed.scores.overall ?? null;
+            const derivedOutcome = deriveTaskOutcome(compositeScore);
+            // Preserve a previously-set outcome — that only happens when the user
+            // explicitly closed the task via `agentdeck task done/cancel` (or the
+            // macOS detail-pane button) with an outcome override. Without this
+            // guard the async judge resolves 5–30 s after the manual close and
+            // overwrites e.g. `abandoned` with `partial`, silently losing the
+            // user's gesture. `closeTask` itself never writes outcome — only the
+            // manual path does — so a non-null read here is unambiguous.
+            const existingOutcome = this.store.getTask(taskId)?.outcome ?? null;
+            const taskOutcome = isPreservableOutcome(existingOutcome)
+                ? existingOutcome
+                : derivedOutcome;
             this.store.updateTask(taskId, {
                 summary: parsed.summary ?? null,
-                compositeScore: parsed.scores.overall ?? null,
+                compositeScore,
+                outcome: taskOutcome,
                 notesJson: JSON.stringify({
                     reasoning: parsed.reasoning,
                     done: parsed.done,
                     missed: parsed.missed,
                 }),
             });
-            debug('APME', `task eval ${taskId.slice(0, 8)}: overall=${parsed.scores.overall} summary=${parsed.summary?.slice(0, 40) ?? '-'}`);
+            debug('APME', `task eval ${taskId.slice(0, 8)}: overall=${compositeScore} outcome=${taskOutcome} summary=${parsed.summary?.slice(0, 40) ?? '-'}`);
             for (const fn of this.listeners) {
                 try {
-                    fn({ runId, taskId, layer1Ran: false, layer2Ran: true, overall: parsed.scores.overall });
+                    fn({ runId, taskId, layer1Ran: false, layer2Ran: true, overall: compositeScore ?? undefined });
                 }
                 catch { /* ignore */ }
             }
+            if (this.taskListeners.size > 0) {
+                const run = this.store.getRun(runId);
+                const updatedTask = this.store.getTask(taskId) ?? task;
+                const event = {
+                    runId,
+                    taskId,
+                    sessionId: run?.sessionId ?? '',
+                    agentType: run?.agentType ?? undefined,
+                    projectName: run?.projectName ?? undefined,
+                    startedAt: updatedTask.startedAt,
+                    endedAt: updatedTask.endedAt ?? Date.now(),
+                    compositeScore,
+                    outcome: taskOutcome,
+                    taskCategory: category ?? updatedTask.taskCategory ?? undefined,
+                    summary: parsed.summary ?? undefined,
+                    boundarySignal: boundarySignal ?? updatedTask.boundarySignal,
+                    turns: updatedTask.firstTurnIndex != null
+                        ? (updatedTask.lastTurnIndex ?? updatedTask.firstTurnIndex) - updatedTask.firstTurnIndex + 1
+                        : undefined,
+                };
+                for (const fn of this.taskListeners) {
+                    try {
+                        fn(event);
+                    }
+                    catch { /* ignore */ }
+                }
+            }
         }
         catch (err) {
             debug('APME', `task eval error taskId=${taskId.slice(0, 8)}: ${String(err)}`);
@@ -226,14 +361,15 @@ export class ApmeRunner {
             'Respond with strict JSON only.',
         ].join('\n');
         try {
-            const judgeText = this.judgeOverride
-                ? await this.judgeOverride(judgePrompt, cfg.judge)
-                : await callJudge(judgePrompt, cfg.judge);
-            const parsed = parseJudgeJson(judgeText);
+            // Same fallback-aware labelling rule as runTaskEval — see comment there.
+            const judgeResult = this.judgeOverride
+                ? { text: await this.judgeOverride(judgePrompt, cfg.judge), effectiveLabel: effectiveJudgeModelTag(cfg.judge) }
+                : await callJudgeWithMeta(judgePrompt, cfg.judge);
+            const parsed = parseJudgeJson(judgeResult.text);
             if (!parsed)
                 return;
             const now = Date.now();
-            const judgeModel = effectiveJudgeModelTag(cfg.judge);
+            const judgeModel = judgeResult.effectiveLabel;
             for (const [axis, score] of Object.entries(parsed.scores)) {
                 this.store.insertEvalForTurn({
                     runId, turnId,
@@ -274,8 +410,14 @@ export class ApmeRunner {
     async doDrain() {
         while (this.queue.length > 0) {
             const job = this.queue.shift();
+            this.queuedRunIds.delete(job.runId);
+            this.runningRunIds.add(job.runId);
             try {
                 const result = await this.runOne(job);
+                if (!result.layer1Ran && !result.layer2Ran && result.overall === undefined) {
+                    debug('APME', `runner no-op runId=${job.runId} (no eval rows produced)`);
+                    continue;
+                }
                 for (const fn of this.listeners) {
                     try {
                         fn(result);
@@ -286,6 +428,9 @@ export class ApmeRunner {
             catch (err) {
                 debug('APME', `runner error runId=${job.runId}: ${String(err)}`);
             }
+            finally {
+                this.runningRunIds.delete(job.runId);
+            }
         }
     }
     async runOne(job) {
@@ -295,6 +440,10 @@ export class ApmeRunner {
             debug('APME', `runOne: run ${job.runId} not found`);
             return { runId: job.runId, layer1Ran: false, layer2Ran: false };
         }
+        if (this.store.listEvalsForRun(run.id).length > 0) {
+            debug('APME', `runOne: run ${run.id} already has eval rows; skip duplicate`);
+            return { runId: job.runId, layer1Ran: false, layer2Ran: false };
+        }
         // ── Layer 1 — deterministic ───────────────────────────────────────────────
         let layer1Ran = false;
         let layer1Passed = null;
@@ -339,13 +488,14 @@ export class ApmeRunner {
             if (rubric) {
                 try {
                     const prompt = buildJudgePrompt(run, rubric.prompt, layer1Passed, this.store);
-                    const judgeText = this.judgeOverride
-                        ? await this.judgeOverride(prompt, cfg.judge)
-                        : await callJudge(prompt, cfg.judge);
-                    const parsed = parseJudgeJson(judgeText);
+                    // Same fallback-aware labelling rule as runTaskEval — see comment there.
+                    const judgeResult = this.judgeOverride
+                        ? { text: await this.judgeOverride(prompt, cfg.judge), effectiveLabel: effectiveJudgeModelTag(cfg.judge) }
+                        : await callJudgeWithMeta(prompt, cfg.judge);
+                    const parsed = parseJudgeJson(judgeResult.text);
                     if (parsed) {
                         const now = Date.now();
-                        const judgeModel = effectiveJudgeModelTag(cfg.judge);
+                        const judgeModel = judgeResult.effectiveLabel;
                         for (const [axis, score] of Object.entries(parsed.scores)) {
                             this.store.insertEval({
                                 runId: run.id,
@@ -530,6 +680,38 @@ function runCommand(command, cwd, timeoutMs) {
     });
 }
 // ─── Layer 2 execution ────────────────────────────────────────────────────────
+/** Render a SessionSample's typed trajectory as compact judge-prompt lines.
+ *  Gives the judge tool-call sequences + model usage the turn rows omit. */
+export function buildTrajectoryLines(sample, cap = 30) {
+    const lines = [];
+    const events = sample.events.slice(0, cap);
+    for (const e of events) {
+        switch (e.kind) {
+            case 'tool': {
+                let input = '';
+                try {
+                    input = e.input == null ? '' : JSON.stringify(e.input).slice(0, 120);
+                }
+                catch {
+                    input = '';
+                }
+                lines.push(`  tool ${e.name}(${input})${e.status ? ` → ${e.status}` : ''}${e.error ? ` [err: ${String(e.error).slice(0, 80)}]` : ''}`);
+                break;
+            }
+            case 'model':
+                lines.push(`  model ${e.model}: ${e.inputTokens}in/${e.outputTokens}out tok${e.costUsd ? ` ($${e.costUsd.toFixed(4)})` : ''}`);
+                break;
+            case 'state':
+                lines.push(`  state → ${e.to}`);
+                break;
+            default:
+                break; // user/assistant messages already shown in the TURNS section
+        }
+    }
+    if (sample.events.length > cap)
+        lines.push(`  … (${sample.events.length - cap} more events)`);
+    return lines;
+}
 // `ParsedJudge` is the canonical eval v1 type imported from `@agentdeck/shared`.
 const NON_CODE_CATEGORIES = new Set(['conversation', 'planning', 'research', 'review']);
 export function buildJudgePrompt(run, rubricPrompt, layer1Passed, store) {
@@ -619,29 +801,351 @@ export function effectiveJudgeModelTag(cfg) {
         return 'foundationModels:apple-intelligence';
     return `${cfg.backend}:${cfg.model}`;
 }
-export async function callJudge(prompt, judgeCfg) {
-    if (judgeCfg.backend === 'mlx')
-        return callMlx(prompt, judgeCfg);
-    if (judgeCfg.backend === 'openclaw')
-        return callOpenClaw(prompt, judgeCfg);
-    if (judgeCfg.backend === 'api')
-        return callApi(prompt, judgeCfg);
+/** Strip backend-specific fields when forcing a cfg through a different
+ *  adapter. Without this, a FM cfg (`endpoint:"http://.../apme/judge/foundation-models"`,
+ *  `model:"apple-intelligence"`) handed to `callMlx` would POST to the FM URL
+ *  and request a model the MLX server has never heard of — silent failure.
+ *  Mirrors the `resetBackendCoupledFields` path inside `loadApmeConfig`. */
+export function sanitizeForMlx(judgeCfg) {
+    if (judgeCfg.backend === 'mlx' && !judgeCfg.endpoint && (!judgeCfg.model || judgeCfg.model === DEFAULT_APME_CONFIG.judge.model)) {
+        return judgeCfg;
+    }
+    return {
+        ...judgeCfg,
+        backend: 'mlx',
+        endpoint: undefined,
+        model: DEFAULT_APME_CONFIG.judge.model,
+    };
+}
+const FOUNDATION_MODELS_MISSING_DAEMON_TTL_MS = 15_000;
+const FOUNDATION_MODELS_UNAVAILABLE_TTL_MS = 60_000;
+const FOUNDATION_MODELS_READY_TTL_MS = 60_000;
+let foundationModelsAutoCache = null;
+let foundationModelsResolveInFlight = null;
+function getFoundationModelsAutoCache(now = Date.now()) {
+    if (!foundationModelsAutoCache)
+        return null;
+    if (foundationModelsAutoCache.expiresAt <= now) {
+        foundationModelsAutoCache = null;
+        return null;
+    }
+    return foundationModelsAutoCache;
+}
+function markFoundationModelsAutoReady(url) {
+    foundationModelsAutoCache = {
+        state: 'ready',
+        url,
+        expiresAt: Date.now() + FOUNDATION_MODELS_READY_TTL_MS,
+    };
+}
+function markFoundationModelsAutoUnavailable(reason, ttlMs = FOUNDATION_MODELS_UNAVAILABLE_TTL_MS) {
+    foundationModelsAutoCache = {
+        state: 'unavailable',
+        reason,
+        expiresAt: Date.now() + ttlMs,
+    };
+}
+/** Test hook for cache-sensitive runner tests. Production code should not call this. */
+export function clearFoundationModelsAutoCacheForTests() {
+    foundationModelsAutoCache = null;
+    foundationModelsResolveInFlight = null;
+}
+/** Like `callJudge`, but returns the effective backend + label so callers
+ *  can record `judge_model` correctly across fallback paths. */
+export async function callJudgeWithMeta(prompt, judgeCfg) {
     if (judgeCfg.backend === 'foundationModels') {
         try {
-            return await callFoundationModels(prompt, judgeCfg);
+            const text = await callFoundationModels(prompt, judgeCfg);
+            return { text, effectiveBackend: 'foundationModels', effectiveLabel: effectiveJudgeModelTag(judgeCfg) };
         }
         catch (err) {
-            // Cost-sensitive default: do not silently route to a network backend.
-            // Only retry via MLX when the user explicitly opts in — otherwise
+            // Cost-sensitive default: never route to a paid/network backend.
+            // Retry via local MLX only when fallbackToMlx is enabled; otherwise
             // propagate the error so the runner's try/catch skips this eval.
             if (judgeCfg.fallbackToMlx) {
                 debug('APME', `foundationModels unavailable, fallback to MLX: ${String(err)}`);
-                return callMlx(prompt, judgeCfg);
+                // sanitizeForMlx wipes the FM-specific endpoint/model so callMlx
+                // never POSTs to the FM endpoint or asks MLX for `apple-intelligence`.
+                // We then derive the label from the SANITIZED cfg — recording the
+                // effective backend, not the cfg the user originally requested.
+                const mlxCfg = sanitizeForMlx(judgeCfg);
+                const text = await callMlx(prompt, mlxCfg);
+                return { text, effectiveBackend: 'mlx', effectiveLabel: effectiveJudgeModelTag(mlxCfg) };
             }
             throw err;
         }
     }
-    throw new Error(`unknown judge backend: ${String(judgeCfg.backend)}`);
+    let text;
+    if (judgeCfg.backend === 'mlx')
+        text = await callMlx(prompt, judgeCfg);
+    else if (judgeCfg.backend === 'openai')
+        text = await callOpenAICompatible(prompt, judgeCfg);
+    else if (judgeCfg.backend === 'openclaw')
+        text = await callOpenClaw(prompt, judgeCfg);
+    else if (judgeCfg.backend === 'api')
+        text = await callApi(prompt, judgeCfg);
+    else
+        throw new Error(`unknown judge backend: ${String(judgeCfg.backend)}`);
+    return { text, effectiveBackend: judgeCfg.backend, effectiveLabel: effectiveJudgeModelTag(judgeCfg) };
+}
+/** Backwards-compatible thin wrapper. Internal callers that need the actual
+ *  effective backend (for DB labelling) should call `callJudgeWithMeta`. */
+export async function callJudge(prompt, judgeCfg) {
+    return (await callJudgeWithMeta(prompt, judgeCfg)).text;
+}
+export async function probeJudgeBackend(cfg) {
+    const start = Date.now();
+    const checkedAt = start;
+    // Each branch must establish that the backend can ACTUALLY produce a judge
+    // response — not just that some HTTP port answers. Reachability without a
+    // usable model / SDK / on-device LLM yields false-positive "ready" that
+    // hides the real failure mode (silent eval skip later). Follow the rule:
+    // either invoke (cheap ping) or downgrade to `unavailable` with a reason.
+    try {
+        if (cfg.backend === 'mlx') {
+            const mlx = loadMlxSettings();
+            const url = cfg.endpoint ?? mlx.endpoint;
+            const base = url.replace(/\/v1\/chat\/completions$/, '').replace(/\/chat\/completions$/, '');
+            let model;
+            let modelsReachable = false;
+            for (const path of ['/v1/models', '/models']) {
+                const resp = await fetch(`${base}${path}`, { signal: AbortSignal.timeout(5000) }).catch(() => null);
+                if (resp?.ok) {
+                    modelsReachable = true;
+                    const json = await resp.json().catch(() => ({}));
+                    model = json.data?.find(m => m.id && !m.id.toLowerCase().includes('nanollava'))?.id;
+                    break;
+                }
+            }
+            if (!modelsReachable) {
+                return {
+                    backend: 'mlx', status: 'unavailable',
+                    reason: `MLX server unreachable at ${base}. Start with \`mlx_lm.server\` or set apme.judge.endpoint.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            // Pinned/configured model overrides catalog discovery — the real call uses
+            // the same fallback chain as callMlx().
+            const pickedModel = mlx.model ?? cfg.model ?? model;
+            if (!pickedModel) {
+                return {
+                    backend: 'mlx', status: 'unavailable',
+                    reason: `MLX server reachable at ${base} but advertises no chat-capable model (only nanollava-class found). Load a chat model with \`mlx_lm.server --model …\`.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            // Cheapest possible inference probe: max_tokens=1, temperature=0. If MLX
+            // accepts this without a model error, callMlx() will succeed too.
+            const ping = await fetch(`${base}/v1/chat/completions`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    model: pickedModel,
+                    messages: [{ role: 'user', content: 'ping' }],
+                    max_tokens: 1, temperature: 0,
+                }),
+                signal: AbortSignal.timeout(8000),
+            }).catch((e) => ({ ok: false, status: 0, statusText: String(e).slice(0, 80) }));
+            if (!ping.ok) {
+                const detail = ping.status ? `HTTP ${ping.status}` : ping.statusText ?? 'no response';
+                return {
+                    backend: 'mlx', status: 'unavailable',
+                    reason: `MLX inference failed for model "${pickedModel}" (${detail}). Check that the model is actually loaded.`,
+                    endpoint: base, model: pickedModel, checkedAt,
+                };
+            }
+            return { backend: 'mlx', status: 'ready', latencyMs: Date.now() - start, model: pickedModel, endpoint: base, checkedAt };
+        }
+        if (cfg.backend === 'openai') {
+            // Generic OpenAI-compatible (Ollama / OpenRouter / LM Studio / vLLM / …).
+            // "Ready" requires a reachable model catalog and a resolvable model id.
+            if (!cfg.endpoint) {
+                return {
+                    backend: 'openai', status: 'unavailable',
+                    reason: 'set apme.judge.endpoint (e.g. http://127.0.0.1:11434/v1 for Ollama, or https://openrouter.ai/api/v1 for OpenRouter)',
+                    checkedAt,
+                };
+            }
+            const base = openAIBase(cfg.endpoint);
+            const isRemote = /^https?:\/\/(?!127\.0\.0\.1|localhost|\[::1\])/.test(cfg.endpoint);
+            if (isRemote && !cfg.apiKey) {
+                return {
+                    backend: 'openai', status: 'unavailable',
+                    reason: `remote endpoint ${base} needs an API key — set apme.judge.apiKey (OpenRouter etc.)`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            const model = await resolveOpenAIModel(base, cfg.apiKey, cfg.model);
+            if (!model || model === 'default') {
+                return {
+                    backend: 'openai', status: 'unavailable',
+                    reason: `endpoint ${base} unreachable or advertises no model — is the server running / the key valid?`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            return { backend: 'openai', status: 'ready', latencyMs: Date.now() - start, model, endpoint: base, checkedAt };
+        }
+        if (cfg.backend === 'openclaw') {
+            // OpenClaw Gateway: /health proves the gateway socket is up but does NOT
+            // prove /chat will route. The bridge's own gateway adapter performs an
+            // Ed25519 handshake against the same gateway; we don't replicate it
+            // here, but we DO require both /health AND a model catalog response so
+            // a stub gateway with /health = 200 doesn't pass.
+            const url = cfg.endpoint ?? 'http://127.0.0.1:18789';
+            const base = url.replace(/\/chat$/, '');
+            const health = await fetch(`${base}/health`, { signal: AbortSignal.timeout(5000) }).catch(() => null);
+            if (!health?.ok) {
+                return {
+                    backend: 'openclaw', status: 'unavailable',
+                    reason: `OpenClaw Gateway /health unreachable at ${base}.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            const models = await fetch(`${base}/models`, { signal: AbortSignal.timeout(5000) }).catch(() => null);
+            if (!models?.ok) {
+                return {
+                    backend: 'openclaw', status: 'unavailable',
+                    reason: `OpenClaw Gateway /health responds but /models does not — gateway not fully initialised. Wait for handshake or check apme.judge.endpoint.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            const json = await models.json().catch(() => ({}));
+            const list = json.data ?? json.models ?? [];
+            const requested = cfg.model;
+            if (requested && !list.some(m => m.id === requested)) {
+                return {
+                    backend: 'openclaw', status: 'unavailable',
+                    reason: `OpenClaw Gateway is up but model "${requested}" is not advertised. Available: ${list.slice(0, 5).map(m => m.id).join(', ')}`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            return {
+                backend: 'openclaw', status: 'ready',
+                latencyMs: Date.now() - start,
+                model: requested ?? list[0]?.id, endpoint: base, checkedAt,
+            };
+        }
+        if (cfg.backend === 'foundationModels') {
+            // Mirror callFoundationModels: explicit endpoint wins over auto-resolve.
+            const url = cfg.endpoint ?? await resolveFoundationModelsUrl();
+            if (!url) {
+                const helper = await probeFoundationModelsHelper();
+                if (helper.available) {
+                    return {
+                        backend: 'foundationModels',
+                        status: 'ready',
+                        latencyMs: Date.now() - start,
+                        endpoint: helper.path ? `helper:${helper.path}` : 'helper',
+                        checkedAt,
+                    };
+                }
+                return {
+                    backend: 'foundationModels', status: 'unavailable',
+                    reason: `Swift daemon not found and helper unavailable: ${helper.reason ?? 'unknown'}`,
+                    checkedAt,
+                };
+            }
+            // Foundation Models adapter on the Swift side returns either { text }
+            // (ready) or { error: "unavailable", reason } (Apple Intelligence not
+            // downloaded, model still warming, etc). A trivial ping forces that
+            // signal up to us so we don't claim ready when the on-device LLM is
+            // actually unusable.
+            const ping = await fetch(url, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ prompt: 'ping' }),
+                signal: AbortSignal.timeout(8000),
+            }).catch((e) => ({ ok: false, status: 0, statusText: String(e).slice(0, 80) }));
+            if (!ping.ok) {
+                const detail = ping.status ? `HTTP ${ping.status}` : ping.statusText ?? 'no response';
+                if (!cfg.endpoint) {
+                    markFoundationModelsAutoUnavailable(`Swift daemon FM endpoint did not accept probe (${detail}).`);
+                    const helper = await probeFoundationModelsHelper();
+                    if (helper.available) {
+                        return {
+                            backend: 'foundationModels',
+                            status: 'ready',
+                            latencyMs: Date.now() - start,
+                            endpoint: helper.path ? `helper:${helper.path}` : 'helper',
+                            checkedAt,
+                        };
+                    }
+                }
+                return {
+                    backend: 'foundationModels', status: 'unavailable',
+                    reason: `Swift daemon FM endpoint did not accept probe (${detail}).`,
+                    endpoint: url, checkedAt,
+                };
+            }
+            const json = await ping.json().catch(() => ({}));
+            if (json.error) {
+                if (!cfg.endpoint) {
+                    markFoundationModelsAutoUnavailable(`Foundation Models ${json.error}: ${json.reason ?? 'no reason given'}.`);
+                    const helper = await probeFoundationModelsHelper();
+                    if (helper.available) {
+                        return {
+                            backend: 'foundationModels',
+                            status: 'ready',
+                            latencyMs: Date.now() - start,
+                            endpoint: helper.path ? `helper:${helper.path}` : 'helper',
+                            checkedAt,
+                        };
+                    }
+                }
+                return {
+                    backend: 'foundationModels', status: 'unavailable',
+                    reason: `Foundation Models ${json.error}: ${json.reason ?? 'no reason given'}. Apple Intelligence may not be downloaded yet.`,
+                    endpoint: url, checkedAt,
+                };
+            }
+            if (!cfg.endpoint)
+                markFoundationModelsAutoReady(url);
+            return { backend: 'foundationModels', status: 'ready', latencyMs: Date.now() - start, endpoint: url, checkedAt };
+        }
+        if (cfg.backend === 'api') {
+            // Opt-in Anthropic API judge. "Ready" must mean a judge call can
+            // actually succeed, so verify a credential exists and the model id
+            // resolves via the free Models endpoint (no token spend).
+            const hasCredential = Boolean(cfg.apiKey || process.env.ANTHROPIC_API_KEY || process.env.ANTHROPIC_AUTH_TOKEN);
+            if (!hasCredential) {
+                return {
+                    backend: 'api', status: 'unavailable',
+                    reason: 'no Anthropic API credential — set apme.judge.apiKey in settings.json, export ANTHROPIC_API_KEY, or run `ant auth login`',
+                    checkedAt,
+                };
+            }
+            try {
+                const { default: Anthropic } = await import('@anthropic-ai/sdk');
+                const client = new Anthropic({
+                    ...(cfg.apiKey ? { apiKey: cfg.apiKey } : {}),
+                    timeout: 8_000,
+                    maxRetries: 0,
+                });
+                const model = apiJudgeModel(cfg);
+                await client.models.retrieve(model);
+                return {
+                    backend: 'api', status: 'ready', model,
+                    latencyMs: Date.now() - start, checkedAt,
+                };
+            }
+            catch (err) {
+                return {
+                    backend: 'api', status: 'unavailable',
+                    reason: `Anthropic API probe failed: ${String(err).slice(0, 200)}`,
+                    checkedAt,
+                };
+            }
+        }
+        return { backend: cfg.backend, status: 'unknown', checkedAt };
+    }
+    catch (err) {
+        return {
+            backend: cfg.backend, status: 'unavailable',
+            reason: String(err).slice(0, 200),
+            latencyMs: Date.now() - start,
+            checkedAt,
+        };
+    }
 }
 async function callMlx(prompt, cfg) {
     // MLX server speaks OpenAI chat-completions. The llm.mlx pin (shared with
@@ -691,6 +1195,95 @@ async function callMlx(prompt, cfg) {
     }
     return text;
 }
+/** Normalize a user-supplied base/endpoint to the chat-completions URL.
+ *  Accepts a bare host (`http://127.0.0.1:11434`), a base with `/v1`, or the
+ *  full `/v1/chat/completions` — all resolve to the same POST target. */
+export function openAIChatUrl(endpoint) {
+    let e = endpoint.trim().replace(/\/+$/, '');
+    if (/\/chat\/completions$/.test(e))
+        return e;
+    if (/\/v1$/.test(e))
+        return `${e}/chat/completions`;
+    return `${e}/v1/chat/completions`;
+}
+function openAIBase(endpoint) {
+    return endpoint.trim().replace(/\/+$/, '')
+        .replace(/\/chat\/completions$/, '')
+        .replace(/\/v1$/, '');
+}
+/** Resolve a model id for an OpenAI-compatible server when the user left it
+ *  unset. Ollama exposes `/api/tags`; everything else exposes `/v1/models`. */
+async function resolveOpenAIModel(base, apiKey, configured) {
+    if (configured && configured !== 'qwen3-30b' && configured !== 'default')
+        return configured;
+    const headers = apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
+    // Ollama first (its /v1/models also works, but /api/tags is the canonical list).
+    try {
+        const r = await fetch(`${base}/api/tags`, { headers, signal: AbortSignal.timeout(3000) }).catch(() => null);
+        if (r?.ok) {
+            const j = await r.json();
+            const first = j.models?.find((m) => m.name)?.name;
+            if (first)
+                return first;
+        }
+    }
+    catch { /* try openai path */ }
+    for (const path of ['/v1/models', '/models']) {
+        try {
+            const r = await fetch(`${base}${path}`, { headers, signal: AbortSignal.timeout(3000) }).catch(() => null);
+            if (r?.ok) {
+                const j = await r.json();
+                const first = j.data?.find((m) => m.id && !m.id.toLowerCase().includes('nanollava'))?.id;
+                if (first)
+                    return first;
+            }
+        }
+        catch { /* next */ }
+    }
+    return configured || 'default';
+}
+/**
+ * Generic OpenAI-compatible chat-completions judge. One implementation covers
+ * the de-facto standard local + cloud providers:
+ *   - Ollama       endpoint http://127.0.0.1:11434/v1   (no key)
+ *   - LM Studio    endpoint http://127.0.0.1:1234/v1    (no key)
+ *   - vLLM/llama.cpp/LiteLLM/MLX  (local OpenAI servers, no key)
+ *   - OpenRouter   endpoint https://openrouter.ai/api/v1 (Bearer apiKey)
+ *   - any other OpenAI-compatible endpoint
+ * `apiKey` is sent as a Bearer only when set — local servers ignore it.
+ */
+async function callOpenAICompatible(prompt, cfg) {
+    if (!cfg.endpoint) {
+        throw new Error('openai judge: apme.judge.endpoint is required (e.g. http://127.0.0.1:11434/v1 for Ollama)');
+    }
+    const url = openAIChatUrl(cfg.endpoint);
+    const base = openAIBase(cfg.endpoint);
+    const model = await resolveOpenAIModel(base, cfg.apiKey, cfg.model);
+    const headers = { 'Content-Type': 'application/json' };
+    if (cfg.apiKey)
+        headers.Authorization = `Bearer ${cfg.apiKey}`;
+    const resp = await fetch(url, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+            model,
+            messages: [
+                { role: 'system', content: 'You are an exacting code evaluator. Reply with strict JSON only.' },
+                { role: 'user', content: prompt },
+            ],
+            temperature: 0,
+            max_tokens: 1024,
+        }),
+        signal: AbortSignal.timeout(90_000),
+    });
+    if (!resp.ok)
+        throw new Error(`openai judge HTTP ${resp.status} (${url})`);
+    const json = await resp.json();
+    const text = json.choices?.[0]?.message?.content;
+    if (typeof text !== 'string' || text.trim().length === 0)
+        throw new Error('openai judge returned empty content');
+    return text;
+}
 async function callOpenClaw(prompt, cfg) {
     // OpenClaw Gateway exposes the user's configured models. Route through it
     // when the user wants to reuse their existing subscription models for judge.
@@ -709,58 +1302,136 @@ async function callOpenClaw(prompt, cfg) {
     return json.text;
 }
 /**
- * Route a judge call to the Swift daemon's Foundation Models adapter.
+ * Route a judge call to Foundation Models.
  *
- * The actual on-device LLMSession lives in
- * `apple/AgentDeck/Daemon/Apme/ApmeJudgeFoundationModels.swift`; this TS path
- * just forwards the prompt. Available only when a Swift in-process daemon is
- * running on the same machine (App Store macOS build). Node-only setups don't
- * ship Foundation Models — callers should either opt into `fallbackToMlx` or
- * accept the resulting eval skip.
+ * Prefer the Swift daemon HTTP adapter when it is running; otherwise use the
+ * bundled CLI Swift helper process. Default CLI config enables
+ * `fallbackToMlx`, while callers can set it false to force a skip when
+ * neither Foundation Models path works.
  *
  * Shape contract:
  *   Request  : POST /apme/judge/foundation-models { prompt: string }
  *   Response : { text: string } | { error: "unavailable", reason: string }
  */
 async function callFoundationModels(prompt, cfg) {
+    const explicitEndpoint = Boolean(cfg.endpoint);
+    const cached = explicitEndpoint ? null : getFoundationModelsAutoCache();
+    if (cached?.state === 'unavailable') {
+        try {
+            return await callFoundationModelsHelper(prompt);
+        }
+        catch (helperErr) {
+            throw new Error(`foundationModels cached unavailable: ${cached.reason}; helper unavailable: ${String(helperErr)}`);
+        }
+    }
     const url = cfg.endpoint ?? await resolveFoundationModelsUrl();
-    if (!url)
-        throw new Error('foundationModels: no Swift daemon found — FM is only available in App Store macOS builds');
-    const resp = await fetch(url, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ prompt }),
-        signal: AbortSignal.timeout(60_000),
-    });
-    if (!resp.ok)
-        throw new Error(`foundationModels HTTP ${resp.status}`);
-    const json = await resp.json();
-    if (json.error) {
-        throw new Error(`foundationModels ${json.error}: ${json.reason ?? 'no reason'}`);
+    if (!url) {
+        try {
+            return await callFoundationModelsHelper(prompt);
+        }
+        catch (helperErr) {
+            throw new Error(`foundationModels: no Swift daemon found and helper unavailable: ${String(helperErr)}`);
+        }
     }
-    if (typeof json.text !== 'string' || json.text.length === 0) {
-        throw new Error('foundationModels returned empty text');
+    try {
+        const resp = await fetch(url, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ prompt }),
+            signal: AbortSignal.timeout(60_000),
+        });
+        if (!resp.ok)
+            throw new Error(`foundationModels HTTP ${resp.status}`);
+        const json = await resp.json();
+        if (json.error) {
+            throw new Error(`foundationModels ${json.error}: ${json.reason ?? 'no reason'}`);
+        }
+        if (typeof json.text !== 'string' || json.text.length === 0) {
+            throw new Error('foundationModels returned empty text');
+        }
+        if (!explicitEndpoint)
+            markFoundationModelsAutoReady(url);
+        return json.text;
+    }
+    catch (err) {
+        if (!explicitEndpoint) {
+            markFoundationModelsAutoUnavailable(String(err));
+            try {
+                return await callFoundationModelsHelper(prompt);
+            }
+            catch (helperErr) {
+                throw new Error(`${String(err)}; helper unavailable: ${String(helperErr)}`);
+            }
+        }
+        throw err;
     }
-    return json.text;
 }
 /** Best-effort resolver for the Swift daemon's FM endpoint. Returns null when
  *  no Swift daemon (httpPort ≠ port) is reachable. */
 async function resolveFoundationModelsUrl() {
+    const cached = getFoundationModelsAutoCache();
+    if (cached?.state === 'ready')
+        return cached.url;
+    if (cached?.state === 'unavailable')
+        return null;
+    if (foundationModelsResolveInFlight)
+        return foundationModelsResolveInFlight;
     // Lazy-require to avoid pulling session-registry into every test bundle
     // that imports runner.ts for its pure helpers.
-    const { findDaemonPortAsync } = await import('../session-registry.js');
-    const info = await findDaemonPortAsync();
-    if (!info)
-        return null;
-    const port = info.httpPort ?? info.port;
-    return `http://127.0.0.1:${port}/apme/judge/foundation-models`;
-}
-async function callApi(_prompt, _cfg) {
-    // Anthropic API — opt-in only. Requires ANTHROPIC_API_KEY and the @anthropic-ai/sdk
-    // package, neither of which we depend on by default. Surface a clear error so
-    // users understand they enabled API mode but didn't wire credentials.
-    throw new Error('APME judge backend "api" requires explicit setup: install @anthropic-ai/sdk and set ANTHROPIC_API_KEY. ' +
-        'Phase 2 ships with MLX (local, free) by default.');
+    foundationModelsResolveInFlight = (async () => {
+        try {
+            const { findDaemonPortAsync } = await import('../session-registry.js');
+            const info = await findDaemonPortAsync();
+            if (!info) {
+                markFoundationModelsAutoUnavailable('Swift daemon not found', FOUNDATION_MODELS_MISSING_DAEMON_TTL_MS);
+                return null;
+            }
+            const port = info.httpPort ?? info.port;
+            const url = `http://127.0.0.1:${port}/apme/judge/foundation-models`;
+            markFoundationModelsAutoReady(url);
+            return url;
+        }
+        finally {
+            foundationModelsResolveInFlight = null;
+        }
+    })();
+    return foundationModelsResolveInFlight;
+}
+/** Default model for the opt-in Anthropic API judge when the configured
+ *  `model` belongs to another backend (e.g. an MLX id left over from a
+ *  backend switch). */
+const API_JUDGE_DEFAULT_MODEL = 'claude-opus-4-8';
+function apiJudgeModel(cfg) {
+    return cfg.model && cfg.model.startsWith('claude') ? cfg.model : API_JUDGE_DEFAULT_MODEL;
+}
+async function callApi(prompt, cfg) {
+    // Opt-in Anthropic API judge (mirrors the Swift ApmeJudgeApi adapter).
+    // Credential chain: settings.json apme.judge.apiKey -> the SDK's standard
+    // resolution (ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN / `ant auth login`
+    // profile). Hard client timeout — a wedged judge must never wedge an eval.
+    const { default: Anthropic } = await import('@anthropic-ai/sdk');
+    const client = new Anthropic({
+        ...(cfg.apiKey ? { apiKey: cfg.apiKey } : {}),
+        timeout: 90_000,
+        maxRetries: 1,
+    });
+    const response = await client.messages.create({
+        model: apiJudgeModel(cfg),
+        max_tokens: 8192,
+        thinking: { type: 'adaptive' },
+        messages: [{ role: 'user', content: prompt }],
+    });
+    if (response.stop_reason === 'refusal') {
+        throw new Error('API judge refused the request (stop_reason=refusal)');
+    }
+    const text = response.content
+        .filter((b) => b.type === 'text')
+        .map((b) => b.text)
+        .join('\n')
+        .trim();
+    if (!text)
+        throw new Error(`API judge returned no text (stop_reason=${response.stop_reason})`);
+    return text;
 }
 export function parseJudgeJson(text) {
     // Models often wrap JSON in prose or code fences — grab the first {...} block.