@agentdeck/bridge 0.2.1 → 0.2.3

package/dist/apme/runner.js

@@ -6,16 +6,35 @@
  * 0/1 in `evals` (metrics: lint_clean, build_ok, tests_pass).
  *
  * Layer 2 (llm_judge): G-Eval style rubric against the latest `rubrics` row.
- * Backend is pluggable — default is local MLX (cost-free), API/OpenClaw are
- * opt-in via `~/.agentdeck/settings.json`. Gated by `shouldJudge()` so the
- * common "clear pass" case skips layer 2 entirely.
+ * Backend is pluggable — default is Foundation Models via the Swift daemon or
+ * bundled CLI Swift helper, with local MLX fallback for CLI-only runs. Gated
+ * by `shouldJudge()` so the common "clear pass" case skips layer 2 entirely.
  */
 import { spawn } from 'child_process';
 import { existsSync, readdirSync } from 'fs';
+import { createRequire } from 'module';
 import { debug } from '../logger.js';
-import { loadApmeConfig, shouldJudge } from './settings.js';
+import { loadApmeConfig, shouldJudge, DEFAULT_APME_CONFIG } from './settings.js';
 import { loadMlxSettings, mlxChatUrl } from '@agentdeck/shared';
+import { callFoundationModelsHelper, probeFoundationModelsHelper } from '../foundation-models-helper.js';
+import { runSampleScorers } from './scorers/index.js';
 import { execSync } from 'child_process';
+function deriveTaskOutcome(score) {
+    if (score == null)
+        return 'pending';
+    if (score >= 0.75)
+        return 'success';
+    if (score >= 0.5)
+        return 'partial';
+    return 'fail';
+}
+/** Narrow an arbitrary DB `outcome` string to the typed union, so a row
+ *  written via the manual path (`closeTaskExternal` outcome override) can
+ *  flow back into `TaskEvaluatedEvent` without losing type-safety. */
+function isPreservableOutcome(value) {
+    return value === 'success' || value === 'partial' || value === 'fail'
+        || value === 'pending' || value === 'abandoned';
+}
 const DEFAULT_COMMANDS = {
     typescript: {
         lint: 'pnpm -w lint',
@@ -38,13 +57,27 @@ export class ApmeRunner {
     store;
     queue = [];
     drainPromise = null;
+    queuedRunIds = new Set();
+    runningRunIds = new Set();
+    runningTurnIds = new Set();
+    runningTaskIds = new Set();
     listeners = new Set();
+    taskListeners = new Set();
     configOverride = null;
     judgeOverride = null;
     detOverride = null;
+    /** Cached startup judge readiness probe. Populated by `refreshBackendProbe`,
+     *  surfaced on /health. Null until the first probe completes. */
+    lastBackendProbe = null;
     constructor(store) {
         this.store = store;
     }
+    /** Probe the configured judge backend and cache the result. Safe to call
+     *  fire-and-forget at daemon startup — failures don't throw. */
+    async refreshBackendProbe(cfg) {
+        this.lastBackendProbe = await probeJudgeBackend(cfg);
+        return this.lastBackendProbe;
+    }
     _setConfig(cfg) { this.configOverride = cfg; }
     _setJudgeFn(fn) {
         this.judgeOverride = fn;
@@ -56,9 +89,23 @@ export class ApmeRunner {
         this.listeners.add(fn);
         return () => this.listeners.delete(fn);
     }
+    /** Subscribe to per-task judge completions. The event fires AFTER the
+     *  task_judge axis scores and summary are persisted, so listeners can read
+     *  the final state directly from the event payload without re-querying.
+     *  Used by the timeline emitter to upsert the corresponding `task_end` row
+     *  with score + outcome metadata. */
+    onTaskEvaluated(fn) {
+        this.taskListeners.add(fn);
+        return () => this.taskListeners.delete(fn);
+    }
     enqueue(job) {
         if (!this.store.enabled)
             return;
+        if (this.queuedRunIds.has(job.runId) || this.runningRunIds.has(job.runId)) {
+            debug('APME', `skip duplicate eval enqueue runId=${job.runId}`);
+            return;
+        }
+        this.queuedRunIds.add(job.runId);
         this.queue.push(job);
         debug('APME', `enqueue eval runId=${job.runId} (queue=${this.queue.length})`);
         void this.drain();
@@ -69,7 +116,14 @@ export class ApmeRunner {
     enqueueTurn(job) {
         if (!this.store.enabled)
             return;
-        void this.runTurnEval(job);
+        if (this.runningTurnIds.has(job.turnId)) {
+            debug('APME', `skip duplicate turn eval turnId=${job.turnId}`);
+            return;
+        }
+        this.runningTurnIds.add(job.turnId);
+        void this.runTurnEval(job).finally(() => {
+            this.runningTurnIds.delete(job.turnId);
+        });
     }
     /** Judge a closed task (group of turns between boundary signals —
      *  TodoWrite all-completed, /clear, session_end). Fires-and-forgets;
@@ -77,7 +131,14 @@ export class ApmeRunner {
     enqueueTask(job) {
         if (!this.store.enabled)
             return;
-        void this.runTaskEval(job);
+        if (this.runningTaskIds.has(job.taskId)) {
+            debug('APME', `skip duplicate task eval taskId=${job.taskId}`);
+            return;
+        }
+        this.runningTaskIds.add(job.taskId);
+        void this.runTaskEval(job).finally(() => {
+            this.runningTaskIds.delete(job.taskId);
+        });
     }
     async runTaskEval({ runId, taskId, category, boundarySignal }) {
         const cfg = this.configOverride ?? loadApmeConfig();
@@ -121,6 +182,13 @@ export class ApmeRunner {
         if (turns.length > TURN_CAP) {
             lines.push(`… (${turns.length - TURN_CAP} more turns omitted)`);
         }
+        // The canonical SessionSample gives the judge the typed tool trajectory
+        // (req #5/#6) and per-sample cost (req #7) — context the turn rows lack.
+        const sample = this.store.getSample(taskId);
+        const trajectoryLines = sample ? buildTrajectoryLines(sample) : [];
+        const costLine = sample
+            ? `cost: ${sample.cost.inputTokens}in/${sample.cost.outputTokens}out tok, $${sample.cost.costUsd.toFixed(4)}, model ${sample.model.modelId}`
+            : '';
         const judgePrompt = [
             rubric.prompt,
             '',
@@ -128,23 +196,28 @@ export class ApmeRunner {
             `task_category: ${category ?? task.taskCategory ?? 'unknown'}`,
             `turn_count: ${turns.length}`,
             `boundary_signal: ${boundarySignal ?? task.boundarySignal}`,
+            ...(costLine ? [costLine] : []),
             '',
             '--- TURNS ---',
             ...lines,
+            ...(trajectoryLines.length ? ['', '--- TOOL TRAJECTORY ---', ...trajectoryLines] : []),
             '',
             'Respond with strict JSON only.',
         ].join('\n');
         try {
-            const judgeText = this.judgeOverride
-                ? await this.judgeOverride(judgePrompt, cfg.judge)
-                : await callJudge(judgePrompt, cfg.judge);
-            const parsed = parseJudgeJson(judgeText);
+            // callJudgeWithMeta carries the effective backend label across the
+            // FM→MLX fallback path. Without it, eval rows produced by the MLX
+            // fallback would be misattributed to foundationModels in the DB.
+            const judgeResult = this.judgeOverride
+                ? { text: await this.judgeOverride(judgePrompt, cfg.judge), effectiveLabel: effectiveJudgeModelTag(cfg.judge) }
+                : await callJudgeWithMeta(judgePrompt, cfg.judge);
+            const parsed = parseJudgeJson(judgeResult.text);
             if (!parsed) {
                 debug('APME', `runTaskEval parse failed task=${taskId.slice(0, 8)}`);
                 return;
             }
             const now = Date.now();
-            const judgeModel = effectiveJudgeModelTag(cfg.judge);
+            const judgeModel = judgeResult.effectiveLabel;
             for (const [axis, score] of Object.entries(parsed.scores)) {
                 this.store.insertEvalForTask({
                     id: 0,
@@ -165,22 +238,82 @@ export class ApmeRunner {
                     createdAt: now,
                 });
             }
+            // Pure sample-trajectory scorers (tool churn, error rate) — they add
+            // signal the LLM judge can miss and are cheap/deterministic. Stored under
+            // layer='trajectory' so they don't collide with task_judge axes.
+            if (sample) {
+                try {
+                    for (const r of runSampleScorers(sample)) {
+                        this.store.insertEvalForTask({
+                            id: 0, runId, taskId,
+                            layer: r.layer,
+                            metric: r.metric,
+                            score: r.score,
+                            raw: r.reasoning ? JSON.stringify({ reasoning: r.reasoning, scorer: r.scorer }) : null,
+                            rubricVer: null,
+                            judgeModel: `scorer:${r.scorer}`,
+                            createdAt: now,
+                        });
+                    }
+                }
+                catch (err) {
+                    debug('APME', `sample scorers failed task=${taskId.slice(0, 8)}: ${String(err)}`);
+                }
+            }
+            const compositeScore = parsed.scores.overall ?? null;
+            const derivedOutcome = deriveTaskOutcome(compositeScore);
+            // Preserve a previously-set outcome — that only happens when the user
+            // explicitly closed the task via `agentdeck task done/cancel` (or the
+            // macOS detail-pane button) with an outcome override. Without this
+            // guard the async judge resolves 5–30 s after the manual close and
+            // overwrites e.g. `abandoned` with `partial`, silently losing the
+            // user's gesture. `closeTask` itself never writes outcome — only the
+            // manual path does — so a non-null read here is unambiguous.
+            const existingOutcome = this.store.getTask(taskId)?.outcome ?? null;
+            const taskOutcome = isPreservableOutcome(existingOutcome)
+                ? existingOutcome
+                : derivedOutcome;
             this.store.updateTask(taskId, {
                 summary: parsed.summary ?? null,
-                compositeScore: parsed.scores.overall ?? null,
+                compositeScore,
+                outcome: taskOutcome,
                 notesJson: JSON.stringify({
                     reasoning: parsed.reasoning,
                     done: parsed.done,
                     missed: parsed.missed,
                 }),
             });
-            debug('APME', `task eval ${taskId.slice(0, 8)}: overall=${parsed.scores.overall} summary=${parsed.summary?.slice(0, 40) ?? '-'}`);
+            debug('APME', `task eval ${taskId.slice(0, 8)}: overall=${compositeScore} outcome=${taskOutcome} summary=${parsed.summary?.slice(0, 40) ?? '-'}`);
             for (const fn of this.listeners) {
                 try {
-                    fn({ runId, taskId, layer1Ran: false, layer2Ran: true, overall: parsed.scores.overall });
+                    fn({ runId, taskId, layer1Ran: false, layer2Ran: true, overall: compositeScore ?? undefined });
                 }
                 catch { /* ignore */ }
             }
+            if (this.taskListeners.size > 0) {
+                const run = this.store.getRun(runId);
+                const updatedTask = this.store.getTask(taskId) ?? task;
+                const event = {
+                    runId,
+                    taskId,
+                    sessionId: run?.sessionId ?? '',
+                    agentType: run?.agentType ?? undefined,
+                    projectName: run?.projectName ?? undefined,
+                    startedAt: updatedTask.startedAt,
+                    endedAt: updatedTask.endedAt ?? Date.now(),
+                    compositeScore,
+                    outcome: taskOutcome,
+                    taskCategory: category ?? updatedTask.taskCategory ?? undefined,
+                    summary: parsed.summary ?? undefined,
+                    boundarySignal: boundarySignal ?? updatedTask.boundarySignal,
+                };
+                for (const fn of this.taskListeners) {
+                    try {
+                        fn(event);
+                    }
+                    catch { /* ignore */ }
+                }
+            }
         }
         catch (err) {
             debug('APME', `task eval error taskId=${taskId.slice(0, 8)}: ${String(err)}`);
@@ -226,14 +359,15 @@ export class ApmeRunner {
             'Respond with strict JSON only.',
         ].join('\n');
         try {
-            const judgeText = this.judgeOverride
-                ? await this.judgeOverride(judgePrompt, cfg.judge)
-                : await callJudge(judgePrompt, cfg.judge);
-            const parsed = parseJudgeJson(judgeText);
+            // Same fallback-aware labelling rule as runTaskEval — see comment there.
+            const judgeResult = this.judgeOverride
+                ? { text: await this.judgeOverride(judgePrompt, cfg.judge), effectiveLabel: effectiveJudgeModelTag(cfg.judge) }
+                : await callJudgeWithMeta(judgePrompt, cfg.judge);
+            const parsed = parseJudgeJson(judgeResult.text);
             if (!parsed)
                 return;
             const now = Date.now();
-            const judgeModel = effectiveJudgeModelTag(cfg.judge);
+            const judgeModel = judgeResult.effectiveLabel;
             for (const [axis, score] of Object.entries(parsed.scores)) {
                 this.store.insertEvalForTurn({
                     runId, turnId,
@@ -274,8 +408,14 @@ export class ApmeRunner {
     async doDrain() {
         while (this.queue.length > 0) {
             const job = this.queue.shift();
+            this.queuedRunIds.delete(job.runId);
+            this.runningRunIds.add(job.runId);
             try {
                 const result = await this.runOne(job);
+                if (!result.layer1Ran && !result.layer2Ran && result.overall === undefined) {
+                    debug('APME', `runner no-op runId=${job.runId} (no eval rows produced)`);
+                    continue;
+                }
                 for (const fn of this.listeners) {
                     try {
                         fn(result);
@@ -286,6 +426,9 @@ export class ApmeRunner {
             catch (err) {
                 debug('APME', `runner error runId=${job.runId}: ${String(err)}`);
             }
+            finally {
+                this.runningRunIds.delete(job.runId);
+            }
         }
     }
     async runOne(job) {
@@ -295,6 +438,10 @@ export class ApmeRunner {
             debug('APME', `runOne: run ${job.runId} not found`);
             return { runId: job.runId, layer1Ran: false, layer2Ran: false };
         }
+        if (this.store.listEvalsForRun(run.id).length > 0) {
+            debug('APME', `runOne: run ${run.id} already has eval rows; skip duplicate`);
+            return { runId: job.runId, layer1Ran: false, layer2Ran: false };
+        }
         // ── Layer 1 — deterministic ───────────────────────────────────────────────
         let layer1Ran = false;
         let layer1Passed = null;
@@ -339,13 +486,14 @@ export class ApmeRunner {
             if (rubric) {
                 try {
                     const prompt = buildJudgePrompt(run, rubric.prompt, layer1Passed, this.store);
-                    const judgeText = this.judgeOverride
-                        ? await this.judgeOverride(prompt, cfg.judge)
-                        : await callJudge(prompt, cfg.judge);
-                    const parsed = parseJudgeJson(judgeText);
+                    // Same fallback-aware labelling rule as runTaskEval — see comment there.
+                    const judgeResult = this.judgeOverride
+                        ? { text: await this.judgeOverride(prompt, cfg.judge), effectiveLabel: effectiveJudgeModelTag(cfg.judge) }
+                        : await callJudgeWithMeta(prompt, cfg.judge);
+                    const parsed = parseJudgeJson(judgeResult.text);
                     if (parsed) {
                         const now = Date.now();
-                        const judgeModel = effectiveJudgeModelTag(cfg.judge);
+                        const judgeModel = judgeResult.effectiveLabel;
                         for (const [axis, score] of Object.entries(parsed.scores)) {
                             this.store.insertEval({
                                 runId: run.id,
@@ -530,6 +678,38 @@ function runCommand(command, cwd, timeoutMs) {
     });
 }
 // ─── Layer 2 execution ────────────────────────────────────────────────────────
+/** Render a SessionSample's typed trajectory as compact judge-prompt lines.
+ *  Gives the judge tool-call sequences + model usage the turn rows omit. */
+export function buildTrajectoryLines(sample, cap = 30) {
+    const lines = [];
+    const events = sample.events.slice(0, cap);
+    for (const e of events) {
+        switch (e.kind) {
+            case 'tool': {
+                let input = '';
+                try {
+                    input = e.input == null ? '' : JSON.stringify(e.input).slice(0, 120);
+                }
+                catch {
+                    input = '';
+                }
+                lines.push(`  tool ${e.name}(${input})${e.status ? ` → ${e.status}` : ''}${e.error ? ` [err: ${String(e.error).slice(0, 80)}]` : ''}`);
+                break;
+            }
+            case 'model':
+                lines.push(`  model ${e.model}: ${e.inputTokens}in/${e.outputTokens}out tok${e.costUsd ? ` ($${e.costUsd.toFixed(4)})` : ''}`);
+                break;
+            case 'state':
+                lines.push(`  state → ${e.to}`);
+                break;
+            default:
+                break; // user/assistant messages already shown in the TURNS section
+        }
+    }
+    if (sample.events.length > cap)
+        lines.push(`  … (${sample.events.length - cap} more events)`);
+    return lines;
+}
 // `ParsedJudge` is the canonical eval v1 type imported from `@agentdeck/shared`.
 const NON_CODE_CATEGORIES = new Set(['conversation', 'planning', 'research', 'review']);
 export function buildJudgePrompt(run, rubricPrompt, layer1Passed, store) {
@@ -619,29 +799,312 @@ export function effectiveJudgeModelTag(cfg) {
         return 'foundationModels:apple-intelligence';
     return `${cfg.backend}:${cfg.model}`;
 }
-export async function callJudge(prompt, judgeCfg) {
-    if (judgeCfg.backend === 'mlx')
-        return callMlx(prompt, judgeCfg);
-    if (judgeCfg.backend === 'openclaw')
-        return callOpenClaw(prompt, judgeCfg);
-    if (judgeCfg.backend === 'api')
-        return callApi(prompt, judgeCfg);
+/** Strip backend-specific fields when forcing a cfg through a different
+ *  adapter. Without this, a FM cfg (`endpoint:"http://.../apme/judge/foundation-models"`,
+ *  `model:"apple-intelligence"`) handed to `callMlx` would POST to the FM URL
+ *  and request a model the MLX server has never heard of — silent failure.
+ *  Mirrors the `resetBackendCoupledFields` path inside `loadApmeConfig`. */
+export function sanitizeForMlx(judgeCfg) {
+    if (judgeCfg.backend === 'mlx' && !judgeCfg.endpoint && (!judgeCfg.model || judgeCfg.model === DEFAULT_APME_CONFIG.judge.model)) {
+        return judgeCfg;
+    }
+    return {
+        ...judgeCfg,
+        backend: 'mlx',
+        endpoint: undefined,
+        model: DEFAULT_APME_CONFIG.judge.model,
+    };
+}
+const FOUNDATION_MODELS_MISSING_DAEMON_TTL_MS = 15_000;
+const FOUNDATION_MODELS_UNAVAILABLE_TTL_MS = 60_000;
+const FOUNDATION_MODELS_READY_TTL_MS = 60_000;
+let foundationModelsAutoCache = null;
+let foundationModelsResolveInFlight = null;
+function getFoundationModelsAutoCache(now = Date.now()) {
+    if (!foundationModelsAutoCache)
+        return null;
+    if (foundationModelsAutoCache.expiresAt <= now) {
+        foundationModelsAutoCache = null;
+        return null;
+    }
+    return foundationModelsAutoCache;
+}
+function markFoundationModelsAutoReady(url) {
+    foundationModelsAutoCache = {
+        state: 'ready',
+        url,
+        expiresAt: Date.now() + FOUNDATION_MODELS_READY_TTL_MS,
+    };
+}
+function markFoundationModelsAutoUnavailable(reason, ttlMs = FOUNDATION_MODELS_UNAVAILABLE_TTL_MS) {
+    foundationModelsAutoCache = {
+        state: 'unavailable',
+        reason,
+        expiresAt: Date.now() + ttlMs,
+    };
+}
+/** Test hook for cache-sensitive runner tests. Production code should not call this. */
+export function clearFoundationModelsAutoCacheForTests() {
+    foundationModelsAutoCache = null;
+    foundationModelsResolveInFlight = null;
+}
+/** Like `callJudge`, but returns the effective backend + label so callers
+ *  can record `judge_model` correctly across fallback paths. */
+export async function callJudgeWithMeta(prompt, judgeCfg) {
     if (judgeCfg.backend === 'foundationModels') {
         try {
-            return await callFoundationModels(prompt, judgeCfg);
+            const text = await callFoundationModels(prompt, judgeCfg);
+            return { text, effectiveBackend: 'foundationModels', effectiveLabel: effectiveJudgeModelTag(judgeCfg) };
         }
         catch (err) {
-            // Cost-sensitive default: do not silently route to a network backend.
-            // Only retry via MLX when the user explicitly opts in — otherwise
+            // Cost-sensitive default: never route to a paid/network backend.
+            // Retry via local MLX only when fallbackToMlx is enabled; otherwise
             // propagate the error so the runner's try/catch skips this eval.
             if (judgeCfg.fallbackToMlx) {
                 debug('APME', `foundationModels unavailable, fallback to MLX: ${String(err)}`);
-                return callMlx(prompt, judgeCfg);
+                // sanitizeForMlx wipes the FM-specific endpoint/model so callMlx
+                // never POSTs to the FM endpoint or asks MLX for `apple-intelligence`.
+                // We then derive the label from the SANITIZED cfg — recording the
+                // effective backend, not the cfg the user originally requested.
+                const mlxCfg = sanitizeForMlx(judgeCfg);
+                const text = await callMlx(prompt, mlxCfg);
+                return { text, effectiveBackend: 'mlx', effectiveLabel: effectiveJudgeModelTag(mlxCfg) };
             }
             throw err;
         }
     }
-    throw new Error(`unknown judge backend: ${String(judgeCfg.backend)}`);
+    let text;
+    if (judgeCfg.backend === 'mlx')
+        text = await callMlx(prompt, judgeCfg);
+    else if (judgeCfg.backend === 'openclaw')
+        text = await callOpenClaw(prompt, judgeCfg);
+    else if (judgeCfg.backend === 'api')
+        text = await callApi(prompt, judgeCfg);
+    else
+        throw new Error(`unknown judge backend: ${String(judgeCfg.backend)}`);
+    return { text, effectiveBackend: judgeCfg.backend, effectiveLabel: effectiveJudgeModelTag(judgeCfg) };
+}
+/** Backwards-compatible thin wrapper. Internal callers that need the actual
+ *  effective backend (for DB labelling) should call `callJudgeWithMeta`. */
+export async function callJudge(prompt, judgeCfg) {
+    return (await callJudgeWithMeta(prompt, judgeCfg)).text;
+}
+export async function probeJudgeBackend(cfg) {
+    const start = Date.now();
+    const checkedAt = start;
+    // Each branch must establish that the backend can ACTUALLY produce a judge
+    // response — not just that some HTTP port answers. Reachability without a
+    // usable model / SDK / on-device LLM yields false-positive "ready" that
+    // hides the real failure mode (silent eval skip later). Follow the rule:
+    // either invoke (cheap ping) or downgrade to `unavailable` with a reason.
+    try {
+        if (cfg.backend === 'mlx') {
+            const mlx = loadMlxSettings();
+            const url = cfg.endpoint ?? mlx.endpoint;
+            const base = url.replace(/\/v1\/chat\/completions$/, '').replace(/\/chat\/completions$/, '');
+            let model;
+            let modelsReachable = false;
+            for (const path of ['/v1/models', '/models']) {
+                const resp = await fetch(`${base}${path}`, { signal: AbortSignal.timeout(5000) }).catch(() => null);
+                if (resp?.ok) {
+                    modelsReachable = true;
+                    const json = await resp.json().catch(() => ({}));
+                    model = json.data?.find(m => m.id && !m.id.toLowerCase().includes('nanollava'))?.id;
+                    break;
+                }
+            }
+            if (!modelsReachable) {
+                return {
+                    backend: 'mlx', status: 'unavailable',
+                    reason: `MLX server unreachable at ${base}. Start with \`mlx_lm.server\` or set apme.judge.endpoint.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            // Pinned/configured model overrides catalog discovery — the real call uses
+            // the same fallback chain as callMlx().
+            const pickedModel = mlx.model ?? cfg.model ?? model;
+            if (!pickedModel) {
+                return {
+                    backend: 'mlx', status: 'unavailable',
+                    reason: `MLX server reachable at ${base} but advertises no chat-capable model (only nanollava-class found). Load a chat model with \`mlx_lm.server --model …\`.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            // Cheapest possible inference probe: max_tokens=1, temperature=0. If MLX
+            // accepts this without a model error, callMlx() will succeed too.
+            const ping = await fetch(`${base}/v1/chat/completions`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    model: pickedModel,
+                    messages: [{ role: 'user', content: 'ping' }],
+                    max_tokens: 1, temperature: 0,
+                }),
+                signal: AbortSignal.timeout(8000),
+            }).catch((e) => ({ ok: false, status: 0, statusText: String(e).slice(0, 80) }));
+            if (!ping.ok) {
+                const detail = ping.status ? `HTTP ${ping.status}` : ping.statusText ?? 'no response';
+                return {
+                    backend: 'mlx', status: 'unavailable',
+                    reason: `MLX inference failed for model "${pickedModel}" (${detail}). Check that the model is actually loaded.`,
+                    endpoint: base, model: pickedModel, checkedAt,
+                };
+            }
+            return { backend: 'mlx', status: 'ready', latencyMs: Date.now() - start, model: pickedModel, endpoint: base, checkedAt };
+        }
+        if (cfg.backend === 'openclaw') {
+            // OpenClaw Gateway: /health proves the gateway socket is up but does NOT
+            // prove /chat will route. The bridge's own gateway adapter performs an
+            // Ed25519 handshake against the same gateway; we don't replicate it
+            // here, but we DO require both /health AND a model catalog response so
+            // a stub gateway with /health = 200 doesn't pass.
+            const url = cfg.endpoint ?? 'http://127.0.0.1:18789';
+            const base = url.replace(/\/chat$/, '');
+            const health = await fetch(`${base}/health`, { signal: AbortSignal.timeout(5000) }).catch(() => null);
+            if (!health?.ok) {
+                return {
+                    backend: 'openclaw', status: 'unavailable',
+                    reason: `OpenClaw Gateway /health unreachable at ${base}.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            const models = await fetch(`${base}/models`, { signal: AbortSignal.timeout(5000) }).catch(() => null);
+            if (!models?.ok) {
+                return {
+                    backend: 'openclaw', status: 'unavailable',
+                    reason: `OpenClaw Gateway /health responds but /models does not — gateway not fully initialised. Wait for handshake or check apme.judge.endpoint.`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            const json = await models.json().catch(() => ({}));
+            const list = json.data ?? json.models ?? [];
+            const requested = cfg.model;
+            if (requested && !list.some(m => m.id === requested)) {
+                return {
+                    backend: 'openclaw', status: 'unavailable',
+                    reason: `OpenClaw Gateway is up but model "${requested}" is not advertised. Available: ${list.slice(0, 5).map(m => m.id).join(', ')}`,
+                    endpoint: base, checkedAt,
+                };
+            }
+            return {
+                backend: 'openclaw', status: 'ready',
+                latencyMs: Date.now() - start,
+                model: requested ?? list[0]?.id, endpoint: base, checkedAt,
+            };
+        }
+        if (cfg.backend === 'foundationModels') {
+            // Mirror callFoundationModels: explicit endpoint wins over auto-resolve.
+            const url = cfg.endpoint ?? await resolveFoundationModelsUrl();
+            if (!url) {
+                const helper = await probeFoundationModelsHelper();
+                if (helper.available) {
+                    return {
+                        backend: 'foundationModels',
+                        status: 'ready',
+                        latencyMs: Date.now() - start,
+                        endpoint: helper.path ? `helper:${helper.path}` : 'helper',
+                        checkedAt,
+                    };
+                }
+                return {
+                    backend: 'foundationModels', status: 'unavailable',
+                    reason: `Swift daemon not found and helper unavailable: ${helper.reason ?? 'unknown'}`,
+                    checkedAt,
+                };
+            }
+            // Foundation Models adapter on the Swift side returns either { text }
+            // (ready) or { error: "unavailable", reason } (Apple Intelligence not
+            // downloaded, model still warming, etc). A trivial ping forces that
+            // signal up to us so we don't claim ready when the on-device LLM is
+            // actually unusable.
+            const ping = await fetch(url, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ prompt: 'ping' }),
+                signal: AbortSignal.timeout(8000),
+            }).catch((e) => ({ ok: false, status: 0, statusText: String(e).slice(0, 80) }));
+            if (!ping.ok) {
+                const detail = ping.status ? `HTTP ${ping.status}` : ping.statusText ?? 'no response';
+                if (!cfg.endpoint) {
+                    markFoundationModelsAutoUnavailable(`Swift daemon FM endpoint did not accept probe (${detail}).`);
+                    const helper = await probeFoundationModelsHelper();
+                    if (helper.available) {
+                        return {
+                            backend: 'foundationModels',
+                            status: 'ready',
+                            latencyMs: Date.now() - start,
+                            endpoint: helper.path ? `helper:${helper.path}` : 'helper',
+                            checkedAt,
+                        };
+                    }
+                }
+                return {
+                    backend: 'foundationModels', status: 'unavailable',
+                    reason: `Swift daemon FM endpoint did not accept probe (${detail}).`,
+                    endpoint: url, checkedAt,
+                };
+            }
+            const json = await ping.json().catch(() => ({}));
+            if (json.error) {
+                if (!cfg.endpoint) {
+                    markFoundationModelsAutoUnavailable(`Foundation Models ${json.error}: ${json.reason ?? 'no reason given'}.`);
+                    const helper = await probeFoundationModelsHelper();
+                    if (helper.available) {
+                        return {
+                            backend: 'foundationModels',
+                            status: 'ready',
+                            latencyMs: Date.now() - start,
+                            endpoint: helper.path ? `helper:${helper.path}` : 'helper',
+                            checkedAt,
+                        };
+                    }
+                }
+                return {
+                    backend: 'foundationModels', status: 'unavailable',
+                    reason: `Foundation Models ${json.error}: ${json.reason ?? 'no reason given'}. Apple Intelligence may not be downloaded yet.`,
+                    endpoint: url, checkedAt,
+                };
+            }
+            if (!cfg.endpoint)
+                markFoundationModelsAutoReady(url);
+            return { backend: 'foundationModels', status: 'ready', latencyMs: Date.now() - start, endpoint: url, checkedAt };
+        }
+        if (cfg.backend === 'api') {
+            // The 'api' backend is currently a STUB — callApi() always throws,
+            // regardless of ANTHROPIC_API_KEY or @anthropic-ai/sdk presence. A probe
+            // that returned 'ready' here would be a lie: the next judge call still
+            // throws. Per cost-sensitive-defaults policy (memory:
+            // feedback_cost_sensitive_defaults.md), local MLX is the supported path;
+            // wiring real API judging is deliberately deferred. Surface that gap
+            // honestly so users redirect to MLX/OpenClaw instead of debugging a
+            // never-firing eval.
+            const hasKey = !!process.env.ANTHROPIC_API_KEY;
+            let sdkPresent = false;
+            try {
+                const r = createRequire(import.meta.url);
+                r.resolve('@anthropic-ai/sdk');
+                sdkPresent = true;
+            }
+            catch { /* not installed */ }
+            const env = hasKey
+                ? (sdkPresent ? 'key+SDK present' : 'key set, SDK missing')
+                : 'no ANTHROPIC_API_KEY';
+            return {
+                backend: 'api', status: 'unavailable',
+                reason: `Anthropic API judge backend is not implemented — callApi() is a stub that always throws. Switch apme.judge.backend to "mlx" or "openclaw". Environment: ${env}.`,
+                checkedAt,
+            };
+        }
+        return { backend: cfg.backend, status: 'unknown', checkedAt };
+    }
+    catch (err) {
+        return {
+            backend: cfg.backend, status: 'unavailable',
+            reason: String(err).slice(0, 200),
+            latencyMs: Date.now() - start,
+            checkedAt,
+        };
+    }
 }
 async function callMlx(prompt, cfg) {
     // MLX server speaks OpenAI chat-completions. The llm.mlx pin (shared with
@@ -709,58 +1172,111 @@ async function callOpenClaw(prompt, cfg) {
     return json.text;
 }
 /**
- * Route a judge call to the Swift daemon's Foundation Models adapter.
+ * Route a judge call to Foundation Models.
  *
- * The actual on-device LLMSession lives in
- * `apple/AgentDeck/Daemon/Apme/ApmeJudgeFoundationModels.swift`; this TS path
- * just forwards the prompt. Available only when a Swift in-process daemon is
- * running on the same machine (App Store macOS build). Node-only setups don't
- * ship Foundation Models — callers should either opt into `fallbackToMlx` or
- * accept the resulting eval skip.
+ * Prefer the Swift daemon HTTP adapter when it is running; otherwise use the
+ * bundled CLI Swift helper process. Default CLI config enables
+ * `fallbackToMlx`, while callers can set it false to force a skip when
+ * neither Foundation Models path works.
  *
  * Shape contract:
  *   Request  : POST /apme/judge/foundation-models { prompt: string }
  *   Response : { text: string } | { error: "unavailable", reason: string }
  */
 async function callFoundationModels(prompt, cfg) {
+    const explicitEndpoint = Boolean(cfg.endpoint);
+    const cached = explicitEndpoint ? null : getFoundationModelsAutoCache();
+    if (cached?.state === 'unavailable') {
+        try {
+            return await callFoundationModelsHelper(prompt);
+        }
+        catch (helperErr) {
+            throw new Error(`foundationModels cached unavailable: ${cached.reason}; helper unavailable: ${String(helperErr)}`);
+        }
+    }
     const url = cfg.endpoint ?? await resolveFoundationModelsUrl();
-    if (!url)
-        throw new Error('foundationModels: no Swift daemon found — FM is only available in App Store macOS builds');
-    const resp = await fetch(url, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ prompt }),
-        signal: AbortSignal.timeout(60_000),
-    });
-    if (!resp.ok)
-        throw new Error(`foundationModels HTTP ${resp.status}`);
-    const json = await resp.json();
-    if (json.error) {
-        throw new Error(`foundationModels ${json.error}: ${json.reason ?? 'no reason'}`);
+    if (!url) {
+        try {
+            return await callFoundationModelsHelper(prompt);
+        }
+        catch (helperErr) {
+            throw new Error(`foundationModels: no Swift daemon found and helper unavailable: ${String(helperErr)}`);
+        }
     }
-    if (typeof json.text !== 'string' || json.text.length === 0) {
-        throw new Error('foundationModels returned empty text');
+    try {
+        const resp = await fetch(url, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ prompt }),
+            signal: AbortSignal.timeout(60_000),
+        });
+        if (!resp.ok)
+            throw new Error(`foundationModels HTTP ${resp.status}`);
+        const json = await resp.json();
+        if (json.error) {
+            throw new Error(`foundationModels ${json.error}: ${json.reason ?? 'no reason'}`);
+        }
+        if (typeof json.text !== 'string' || json.text.length === 0) {
+            throw new Error('foundationModels returned empty text');
+        }
+        if (!explicitEndpoint)
+            markFoundationModelsAutoReady(url);
+        return json.text;
+    }
+    catch (err) {
+        if (!explicitEndpoint) {
+            markFoundationModelsAutoUnavailable(String(err));
+            try {
+                return await callFoundationModelsHelper(prompt);
+            }
+            catch (helperErr) {
+                throw new Error(`${String(err)}; helper unavailable: ${String(helperErr)}`);
+            }
+        }
+        throw err;
     }
-    return json.text;
 }
 /** Best-effort resolver for the Swift daemon's FM endpoint. Returns null when
  *  no Swift daemon (httpPort ≠ port) is reachable. */
 async function resolveFoundationModelsUrl() {
+    const cached = getFoundationModelsAutoCache();
+    if (cached?.state === 'ready')
+        return cached.url;
+    if (cached?.state === 'unavailable')
+        return null;
+    if (foundationModelsResolveInFlight)
+        return foundationModelsResolveInFlight;
     // Lazy-require to avoid pulling session-registry into every test bundle
     // that imports runner.ts for its pure helpers.
-    const { findDaemonPortAsync } = await import('../session-registry.js');
-    const info = await findDaemonPortAsync();
-    if (!info)
-        return null;
-    const port = info.httpPort ?? info.port;
-    return `http://127.0.0.1:${port}/apme/judge/foundation-models`;
+    foundationModelsResolveInFlight = (async () => {
+        try {
+            const { findDaemonPortAsync } = await import('../session-registry.js');
+            const info = await findDaemonPortAsync();
+            if (!info) {
+                markFoundationModelsAutoUnavailable('Swift daemon not found', FOUNDATION_MODELS_MISSING_DAEMON_TTL_MS);
+                return null;
+            }
+            const port = info.httpPort ?? info.port;
+            const url = `http://127.0.0.1:${port}/apme/judge/foundation-models`;
+            markFoundationModelsAutoReady(url);
+            return url;
+        }
+        finally {
+            foundationModelsResolveInFlight = null;
+        }
+    })();
+    return foundationModelsResolveInFlight;
 }
 async function callApi(_prompt, _cfg) {
-    // Anthropic API — opt-in only. Requires ANTHROPIC_API_KEY and the @anthropic-ai/sdk
-    // package, neither of which we depend on by default. Surface a clear error so
-    // users understand they enabled API mode but didn't wire credentials.
-    throw new Error('APME judge backend "api" requires explicit setup: install @anthropic-ai/sdk and set ANTHROPIC_API_KEY. ' +
-        'Phase 2 ships with MLX (local, free) by default.');
+    // Anthropic API judging is NOT implemented on the Node bridge. The macOS
+    // Swift daemon (apple/AgentDeck/Daemon/Apme/ApmeJudgeApi.swift) is the only
+    // place this backend works today. `loadApmeConfig` silently downgrades
+    // settings.json `backend:"api"` to `"mlx"`, so reaching this stub means
+    // either (a) a programmatic caller bypassed loadApmeConfig, or (b) something
+    // raced between settings load and dispatch. Either way, throw with the
+    // SAME diagnostic the probe surfaces so logs remain consistent.
+    throw new Error('APME judge backend "api" is a stub on the Node bridge — callApi() always throws. ' +
+        'Use the macOS app for Anthropic API judging, or set apme.judge.backend to "mlx" / "openclaw" / "foundationModels".');
 }
 export function parseJudgeJson(text) {
     // Models often wrap JSON in prose or code fences — grab the first {...} block.