mixdog 0.9.3 → 0.9.4

package/scripts/bench/round-r8-fulldefault.json

@@ -0,0 +1,255 @@
+{
+  "round": "8",
+  "runner": "lead",
+  "opts": {
+    "provider": null,
+    "model": null,
+    "effort": null,
+    "fast": false,
+    "env": {
+      "MIXDOG_DATA_DIR": "C:\\Users\\tempe\\.mixdog\\bench-data"
+    }
+  },
+  "tasks": 3,
+  "completed": 3,
+  "completion_rate": 100,
+  "sessions": [
+    "sess_51468_1783039949771_1783039950019_986225c84c3411f10ae399d9084cae21",
+    "sess_51564_1783039949781_1783039950027_596c0770872549478e285dccd2390597",
+    "sess_19712_1783039949794_1783039950038_d4b96a9272d27d7f810e51341f8c58b7"
+  ],
+  "results": [
+    {
+      "id": "r4-debug-forwarding-gap",
+      "agent": "reviewer",
+      "ok": true,
+      "ms": 144598,
+      "sessionId": "sess_51468_1783039949771_1783039950019_986225c84c3411f10ae399d9084cae21",
+      "output": "sessionId: sess_51468_1783039949771_1783039950019_986225c84c3411f10ae399d9084cae21\n조사 완료: 활성화 직후 경로에는 “지연 후 복구”가 대부분이지만, 새 transcript가 이미 존재하는 상태로 잡히면 첫 출력이 실제로 조용히 스킵될 수 있습니다.\n\n**경로**: `start()`가 `channelBridgeActive=true`와 `bridge-state.json`을 먼저 씁니다(`src/runtime/channels/index.mjs:1734`), 이후 ownership/connect/bind를 거쳐 `forwarder.forwardNewText()`가 `extractNewText()`→queue→`backend.sendMessage()`로 첫 전송합니다(`index.mjs:801`, `lib/output-forwarder.mjs:351`, `index.mjs:284`).\n\n**최대 영향 — 새 transcript 기존 파일을 EOF로 바인딩**: `catchUpFromPersisted`가 켜져도 persisted path가 다르면 cursor를 `currentSize`로 둡니다(`lib/output-forwarder.mjs:117`), 그리고 그 값을 즉시 상태에 저장합니다(`index.mjs:326`); 바인딩 전에 이미 쓰인 assistant 출력은 backlog catch-up으로 복구되지 않습니다.\n\n**높은 영향 — discovery polling 중 파일이 생겨 이미 내용이 있는 경우**: `discoverSessionBoundTranscript()` 루프가 `exists`를 잡으면 초기 activate 경로는 `catchUpFromPersisted:true`만 넘기고 `replayFromStart`는 거의 false입니다(`index.mjs:433`, `index.mjs:441`, `index.mjs:625`); persisted가 같은 transcript일 때만 복구되고, 아니면 위 EOF 스킵으로 유실됩니다.\n\n**중간 영향 — not-yet-created transcript watch 실패**: `startWatch()`는 없는 파일에서 catch 후 watcher를 닫습니다(`lib/output-forwarder.mjs:647`, `lib/output-forwarder.mjs:667`); pending re-arm이 250ms polling으로 파일 생성 후 `startWatch()`와 `forwardNewText()`를 실행해 60초 안에는 backlog가 복구됩니다(`index.mjs:357`, `index.mjs:376`), 60초 timeout 후에는 자동 복구가 없습니다(`index.mjs:384`).\n\n**중간 영향 — bridge-state 이후 connect/ownership 전 flush**: watch flush가 먼저 떠도 `_isOwner()`는 `bridgeRuntimeConnected && currentOwnerState().owned`라서 false이면 읽지 않고 반환합니다(`index.mjs:316`, `lib/output-forwarder.mjs:351`); cursor가 전진하지 않아 post-connect `forwardNewText()` 또는 re-arm이 backlog를 복구합니다(`index.mjs:776`, `index.mjs:801`).\n\n**낮은 영향 — ownership probe 예외/일시 false**: `_isOwner()`는 예외를 non-owner로 처리합니다(`lib/output-forwarder.mjs:36`), 해당 flush는 재시도 예약 없이 끝납니다; bytes는 소비되지 않아 다음 fs.watch/turn-end/post-connect flush가 있으면 복구되지만, 추가 트리거가 없으면 지연될 수 있습니다.\n\n**낮은 영향 — 바인딩 전 channelId 없음**: persisted status나 main channel fallback이 없으면 바인딩 자체를 중단합니다(`index.mjs:610`, `index.mjs:624`), 이후 channel bind가 생길 때까지 전송은 지연되며, 그때 새 transcript를 EOF로 잡으면 위 유실 조건을 다시 탑니다.\n\nVerdict: not-yet-created 파일과 pre-connect ownership 창은 대체로 backlog로 복구되지만, “이미 존재하는 새 transcript를 persisted mismatch 상태로 바인딩”하는 경로는 첫 assistant 출력을 조용히 유실할 수 있습니다.",
+      "card": {
+        "session": "sess_51468_1783039949771_1783039950019_986225c84c3411f10ae399d9084cae21",
+        "provider": "openai-oauth",
+        "model": "gpt-5.5",
+        "wall_ms": 134065,
+        "turns": 9,
+        "tool_calls": 8,
+        "tools_per_turn": 0.9,
+        "total_tool_ms": 497,
+        "llm_stream_ms": 133367,
+        "cache_ratio": 0.8301802744876187,
+        "cached_tokens": 186368,
+        "prompt_tokens": 224491,
+        "uncached_tokens": 38123,
+        "output_tokens": 13835,
+        "cache_weighted_input_10": 56759.8,
+        "cache_weighted_input_25": 84715,
+        "prompt_growth": 35644,
+        "antipatterns": 1,
+        "issues": 15,
+        "issue_types": [
+          "cache",
+          "cache_break",
+          "readonly_stall",
+          "broad_tool_result",
+          "service_tier",
+          "missed_parallelism",
+          "tool_churn_cluster"
+        ],
+        "read_fragmentation_paths": [],
+        "grep_sweep_count": 0
+      }
+    },
+    {
+      "id": "r4-impl-usage-summary",
+      "agent": "worker",
+      "ok": true,
+      "ms": 90877,
+      "sessionId": "sess_51564_1783039949781_1783039950027_596c0770872549478e285dccd2390597",
+      "output": "sessionId: sess_51564_1783039949781_1783039950027_596c0770872549478e285dccd2390597\n요청 범위가 작고 실행 승인이 포함되어 있어 바로 확인·수정·검증까지 진행하겠습니다.구현 파일과 npm script가 이미 작업트리에 있어 요구사항 대비 검증하고 필요한 보정만 하겠습니다.출력이 비어 있는 날에도 `GRAND TOTAL`이 보이도록 보정하겠습니다.패치 도구가 환경 문제로 실패해 PowerShell로 동일 변경을 적용하겠습니다.검증을 실행하겠습니다.차이 요약을 확인하겠습니다.새 스크립트가 untracked라 `git diff`에 빠져 상태까지 확인합니다.재영님, 완료했습니다.\n\n- **추가 기능** `usage:today` npm script와 `scripts/usage-today.mjs`를 추가해 오늘 0시 이후 usage를 모델별로 집계합니다.\n- **집계 내용** calls, input/cached/output tokens, costUsd와 `GRAND TOTAL` 라인을 출력하며 `MIXDOG_DATA_DIR`와 `--trace`, `--json`을 지원합니다.\n- **검증** `node --check scripts/usage-today.mjs` 통과, `npm run usage:today` 실행 성공.\n- **Diff 요약** `package.json` 1줄 추가, `scripts/usage-today.mjs` 신규 파일 추가.",
+      "card": {
+        "session": "sess_51564_1783039949781_1783039950027_596c0770872549478e285dccd2390597",
+        "provider": "openai-oauth",
+        "model": "gpt-5.5",
+        "wall_ms": 80356,
+        "turns": 11,
+        "tool_calls": 16,
+        "tools_per_turn": 1.5,
+        "total_tool_ms": 3261,
+        "llm_stream_ms": 76951,
+        "cache_ratio": 0.7513836864762118,
+        "cached_tokens": 161280,
+        "prompt_tokens": 214644,
+        "uncached_tokens": 53364,
+        "output_tokens": 5943,
+        "cache_weighted_input_10": 69492,
+        "cache_weighted_input_25": 93684,
+        "prompt_growth": 23910,
+        "antipatterns": 1,
+        "issues": 15,
+        "issue_types": [
+          "cache",
+          "cache_break",
+          "tool_error",
+          "service_tier",
+          "missed_parallelism",
+          "tool_churn_cluster"
+        ],
+        "read_fragmentation_paths": [],
+        "grep_sweep_count": 0
+      }
+    },
+    {
+      "id": "r4-explain-compact-engine",
+      "agent": "explore",
+      "ok": true,
+      "ms": 74444,
+      "sessionId": "sess_19712_1783039949794_1783039950038_d4b96a9272d27d7f810e51341f8c58b7",
+      "output": "op.mjs:329`), then fires if `shouldCompactForSession(...)` is true (`src/runtime/agent/orchestrator/session/loop.mjs:353`).\n\n- **Trigger logic**: `shouldCompactForSession` returns false without `auto`/boundary, true on reactive overflow, true on unknown token estimate, otherwise compares `messageTokens + reserveTokens` to `triggerTokens || boundaryTokens` (`src/runtime/agent/orchestrator/session/loop/compact-policy.mjs:170`).\n\n- **Reactive path**: provider context overflow sets `reactiveOverflowRetryPending = true`, causing the next loop to force compaction with trigger `reactive` (`src/runtime/agent/orchestrator/session/loop.mjs:996`, `src/runtime/agent/orchestrator/session/loop.mjs:1002`, `src/runtime/agent/orchestrator/session/loop.mjs:352`).\n\n- **Budgeting**: compact target is capped to the boundary and includes reserve (`src/runtime/agent/orchestrator/session/loop/compact-policy.mjs:163`); manual runner uses the same boundary/trigger/reserve concepts and skips when not forced and below threshold (`src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:150`, `src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:176`).\n\n- **Semantic flow**: `semanticCompactMessages` no-ops if sanitized messages already fit and not forced, otherwise redacts, selects a head/tail window, summarizes the head with provider.send, then emits `system + summary + tail` (`src/runtime/agent/orchestrator/session/compact/engine.mjs:278`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:289`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:312`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:367`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:392`).\n\n- **Semantic tail selection**: `selectCompactionWindow` splits protected context from live history, uses user-anchored recent turns when users exist, otherwise valid non-tool cut points, then sanitizes/reconciles the preserved tail (`src/runtime/agent/orchestrator/session/compact/engine.mjs:235`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:240`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:250`).\n\n- **User-tail policy**: user-anchored mode keeps up to `tailTurns` recent turns within `preserveRecentBudget`; if the newest turn alone is too large it keeps a fitting suffix or moves everything into the head (`src/runtime/agent/orchestrator/session/compact/engine.mjs:134`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:141`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:151`).\n\n- **No-user tail policy**: cut-point mode never starts a tail on a `tool` result, grows the newest valid suffix while it fits, and falls back to minimal/empty tail so there is still a compactable head (`src/runtime/agent/orchestrator/session/compact/engine.mjs:121`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:193`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:207`).\n\n- **Recall-fasttrack flow**: loop first builds bounded recall text, then calls `recallFastTrackCompactMessages` with force and tail options (`src/runtime/agent/orchestrator/session/loop/recall-fasttrack.mjs:148`, `src/runtime/agent/orchestrator/session/loop/recall-fasttrack.mjs:161`).\n\n- **Recall tail policy**: recall preserves newest structured turns verbatim, default max 2 user turns and 8k token cap, dropping oldest turns then truncating oldest surviving content while preserving newest content (`src/runtime/agent/orchestrator/session/compact/engine.mjs:449`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:467`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:601`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:610`).\n\n- **Recall assembly**: recall path redacts after no-op check, splits protected/live/previous summary, selects recall tail/head, requires compactable prior history or recall/prior text, then emits `system + recall summary + recallTail` (`src/runtime/agent/orchestrator/session/compact/engine.mjs:625`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:644`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:646`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:651`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:653`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:695`).\n\n- **Redaction hook**: sensitive tool-call argument values are redacted only when compaction proceeds, before window selection, so preserved system/tail are measured and emitted redacted; no-op returns sanitized but unredacted messages (`src/runtime/agent/orchestrator/session/compact/engine.mjs:286`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:306`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:312`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:389`).\n\n- **Budget fallback**: both semantic and recall raise budget to fit mandatory `system + tail` plus minimum summary room instead of failing when preserved content exceeds target (`src/runtime/agent/orchestrator/session/compact/engine.mjs:319`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:328`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:658`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:661`).\n\n- **Semantic failures**: provider missing, empty live window, no compactable head, prompt/summary fit failure, empty provider summary, or final budget overflow throw from `engine.mjs` (`src/runtime/agent/orchestrator/session/compact/engine.mjs:279`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:254`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:315`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:335`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:372`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:395`).\n\n- **Recall failures**: recall path throws on no compactable prior history, empty recall text when not explicitly allowed, summary fit failure, or final budget overflow (`src/runtime/agent/orchestrator/session/compact/engine.mjs:653`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:666`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:691`, `src/runtime/agent/orchestrator/session/compact/engine.mjs:698`).\n\n- **Loop fallback**: semantic auto failure can recover via `pruneToolOutputsUnanchored` if the pruned transcript fits; recall-fasttrack failure deliberately skips this prune and escalates overflow (`src/runtime/agent/orchestrator/session/loop.mjs:501`, `src/runtime/agent/orchestrator/session/loop.mjs:517`, `src/runtime/agent/orchestrator/session/loop.mjs:527`, `src/runtime/agent/orchestrator/session/loop.mjs:618`).\n\n- **Manual fallback**: `runSessionCompaction` falls back from recall-fasttrack to semantic when enabled/provider exists; if non-recall still fails, it tries the same unanchored prune, otherwise records `manual_failed`/`post_turn_failed` without replacing history (`src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:237`, `src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:245`, `src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:322`, `src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:348`).\n\n- **Success side effects**: successful loop compaction mutates in-memory messages, clears provider state, resets iterations, and records compact telemetry/events (`src/runtime/agent/orchestrator/session/loop.mjs:630`, `src/runtime/agent/orchestrator/session/loop.mjs:639`, `src/runtime/agent/orchestrator/session/loop.mjs:644`, `src/runtime/agent/orchestrator/session/loop.mjs:651`); manual runner writes `session.messages`, clears provider state, and stores compaction metadata (`src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:410`, `src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:411`, `src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs:412`).",
+      "card": {
+        "session": "sess_19712_1783039949794_1783039950038_d4b96a9272d27d7f810e51341f8c58b7",
+        "provider": "openai-oauth",
+        "model": "gpt-5.5",
+        "wall_ms": 63903,
+        "turns": 8,
+        "tool_calls": 12,
+        "tools_per_turn": 1.5,
+        "total_tool_ms": 1254,
+        "llm_stream_ms": 62907,
+        "cache_ratio": 0.5888853058153284,
+        "cached_tokens": 109568,
+        "prompt_tokens": 186060,
+        "uncached_tokens": 76492,
+        "output_tokens": 4420,
+        "cache_weighted_input_10": 87448.8,
+        "cache_weighted_input_25": 103884,
+        "prompt_growth": 36532,
+        "antipatterns": 0,
+        "issues": 10,
+        "issue_types": [
+          "cache",
+          "cache_break",
+          "service_tier"
+        ],
+        "read_fragmentation_paths": [],
+        "grep_sweep_count": 0
+      }
+    }
+  ],
+  "task_complete": true,
+  "task_errors": [],
+  "score_complete": true,
+  "score_errors": [],
+  "group": {
+    "n": 3,
+    "wall_ms": 92774.7,
+    "turns": 9.3,
+    "tool_calls": 12,
+    "tools_per_turn": 1.3,
+    "total_tool_ms": 1670.7,
+    "llm_stream_ms": 91075,
+    "cache_ratio": 0.7,
+    "cached_tokens": 152405.3,
+    "prompt_tokens": 208398.3,
+    "uncached_tokens": 55993,
+    "output_tokens": 8066,
+    "cache_weighted_input_10": 71233.5,
+    "cache_weighted_input_25": 94094.3,
+    "prompt_growth": 32028.7,
+    "antipatterns": 0.7,
+    "issues": 13.3,
+    "grep_sweep_count": 0
+  },
+  "cards": [
+    {
+      "session": "sess_51468_1783039949771_1783039950019_986225c84c3411f10ae399d9084cae21",
+      "provider": "openai-oauth",
+      "model": "gpt-5.5",
+      "wall_ms": 134065,
+      "turns": 9,
+      "tool_calls": 8,
+      "tools_per_turn": 0.9,
+      "total_tool_ms": 497,
+      "llm_stream_ms": 133367,
+      "cache_ratio": 0.8301802744876187,
+      "cached_tokens": 186368,
+      "prompt_tokens": 224491,
+      "uncached_tokens": 38123,
+      "output_tokens": 13835,
+      "cache_weighted_input_10": 56759.8,
+      "cache_weighted_input_25": 84715,
+      "prompt_growth": 35644,
+      "antipatterns": 1,
+      "issues": 15,
+      "issue_types": [
+        "cache",
+        "cache_break",
+        "readonly_stall",
+        "broad_tool_result",
+        "service_tier",
+        "missed_parallelism",
+        "tool_churn_cluster"
+      ],
+      "read_fragmentation_paths": [],
+      "grep_sweep_count": 0
+    },
+    {
+      "session": "sess_51564_1783039949781_1783039950027_596c0770872549478e285dccd2390597",
+      "provider": "openai-oauth",
+      "model": "gpt-5.5",
+      "wall_ms": 80356,
+      "turns": 11,
+      "tool_calls": 16,
+      "tools_per_turn": 1.5,
+      "total_tool_ms": 3261,
+      "llm_stream_ms": 76951,
+      "cache_ratio": 0.7513836864762118,
+      "cached_tokens": 161280,
+      "prompt_tokens": 214644,
+      "uncached_tokens": 53364,
+      "output_tokens": 5943,
+      "cache_weighted_input_10": 69492,
+      "cache_weighted_input_25": 93684,
+      "prompt_growth": 23910,
+      "antipatterns": 1,
+      "issues": 15,
+      "issue_types": [
+        "cache",
+        "cache_break",
+        "tool_error",
+        "service_tier",
+        "missed_parallelism",
+        "tool_churn_cluster"
+      ],
+      "read_fragmentation_paths": [],
+      "grep_sweep_count": 0
+    },
+    {
+      "session": "sess_19712_1783039949794_1783039950038_d4b96a9272d27d7f810e51341f8c58b7",
+      "provider": "openai-oauth",
+      "model": "gpt-5.5",
+      "wall_ms": 63903,
+      "turns": 8,
+      "tool_calls": 12,
+      "tools_per_turn": 1.5,
+      "total_tool_ms": 1254,
+      "llm_stream_ms": 62907,
+      "cache_ratio": 0.5888853058153284,
+      "cached_tokens": 109568,
+      "prompt_tokens": 186060,
+      "uncached_tokens": 76492,
+      "output_tokens": 4420,
+      "cache_weighted_input_10": 87448.8,
+      "cache_weighted_input_25": 103884,
+      "prompt_growth": 36532,
+      "antipatterns": 0,
+      "issues": 10,
+      "issue_types": [
+        "cache",
+        "cache_break",
+        "service_tier"
+      ],
+      "read_fragmentation_paths": [],
+      "grep_sweep_count": 0
+    }
+  ]
+}

package/scripts/bench-run.mjs

@@ -17,7 +17,7 @@
 // This is INTERNAL A/B only (mixdog vs its own prior round). Cross-CLI compare
 // vs codex/claude (tree-total vs solo) is a separate future mode; codex/claude
 // runners are left as slots.
-import { execFileSync } from 'node:child_process';
+import { execFile, execFileSync, spawn } from 'node:child_process';
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { fileURLToPath, pathToFileURL } from 'node:url';
 import { dirname, resolve } from 'node:path';
@@ -57,7 +57,105 @@ function extractSessionId(text) {
 }
 
 const RUNNERS = {
-  mixdog(task, opts) {
+  // Full Lead session (workflow routing + agent tool fan-out). This is the
+  // same path the REPL drives: createMixdogSessionRuntime().ask(). Use it to
+  // measure the delegation/cost-routing design (explorer/worker on cheap
+  // models, lead/reviewer on frontier) instead of a single-role headless run.
+  async lead(task, opts) {
+    const RUNTIME_URL = pathToFileURL(resolve(__dir, '../src/mixdog-session-runtime.mjs')).href;
+    const driver = [
+      `import { createMixdogSessionRuntime } from ${JSON.stringify(RUNTIME_URL)};`,
+      `process.chdir(${JSON.stringify(task.cwd ? resolve(task.cwd) : process.cwd())});`,
+      `const rt = await createMixdogSessionRuntime({`,
+      `  provider: ${JSON.stringify(opts.provider || null)} || undefined,`,
+      `  model: ${JSON.stringify(opts.model || null)} || undefined,`,
+      `});`,
+      `let text = '';`,
+      `// Mirror the TUI auto-resume EXACTLY (engine/agent-job-feed.mjs):`,
+      `// runtime notifications enqueue the model-visible completion into the`,
+      `// session's pending queue; the TUI then kicks an EMPTY pending-resume`,
+      `// ask (no injected user text) via queueMicrotask — the pre-send drain`,
+      `// pulls the pending messages. No polling, no synthetic 'continue', no`,
+      `// verdict heuristic: resume immediately per kick, finish when no kick`,
+      `// arrived during the last turn.`,
+      `let busy = false;`,
+      `let kickDeferred = false;`,
+      `let wake = null;`,
+      `rt.onNotification(() => {`,
+      `  // Defer like scheduleExecutionPendingResumeKick: the model-visible`,
+      `  // body is enqueued after onNotification returns.`,
+      `  queueMicrotask(() => {`,
+      `    if (busy) { kickDeferred = true; return; }`,
+      `    kickDeferred = true;`,
+      `    if (wake) wake();`,
+      `  });`,
+      `  return false;`,
+      `});`,
+      `const askOnce = async (msg) => {`,
+      `  busy = true;`,
+      `  try {`,
+      `    let t = '';`,
+      `    const { result } = await rt.ask(msg, { onTextDelta: (c) => { t += c; } });`,
+      `    return String(result?.text ?? t ?? '');`,
+      `  } finally { busy = false; }`,
+      `};`,
+      `text = await askOnce(${JSON.stringify(task.prompt || '')});`,
+      `const deadline = Date.now() + 30 * 60_000;`,
+      `// Session is "settled" only when no delegated agent task is still`,
+      `// running/pending — the live TUI stays open while agents run, so the`,
+      `// bench must too. Idle grace only applies once the agent board is clear.`,
+      `const agentsBusy = () => {`,
+      `  try {`,
+      `    const st = rt.agentStatus?.() || {};`,
+      `    const jobs = Array.isArray(st.agentJobs) ? st.agentJobs : [];`,
+      `    return jobs.some((j) => /running|pending|spawn|queued/i.test(String(j?.status || j?.state || '')));`,
+      `  } catch { return false; }`,
+      `};`,
+      `while (Date.now() < deadline) {`,
+      `  if (!kickDeferred) {`,
+      `    // Wait for the next kick. While agent tasks are still running the`,
+      `    // wait is unbounded (like a live TUI left open); once the board is`,
+      `    // clear a short grace catches stragglers, then the session settles.`,
+      `    for (;;) {`,
+      `      const kicked = await new Promise((r) => {`,
+      `        wake = () => r(true);`,
+      `        setTimeout(() => r(false), 10_000);`,
+      `      });`,
+      `      wake = null;`,
+      `      if (kicked || kickDeferred) break;`,
+      `      if (!agentsBusy()) break;`,
+      `      if (Date.now() >= deadline) break;`,
+      `    }`,
+      `    if (!kickDeferred && !agentsBusy()) break;`,
+      `  }`,
+      `  kickDeferred = false;`,
+      `  const t = await askOnce('');`,
+      `  if (String(t || '').trim()) text = t;`,
+      `}`,
+      `const sid = rt.sessionId || rt.session?.id || '';`,
+      `process.stdout.write('sessionId: ' + sid + '\\n');`,
+      `process.stdout.write(text);`,
+      `await rt.close('bench-exit');`,
+      `process.exit(0);`,
+    ].join('\n');
+    const started = Date.now();
+    const { raw, ok } = await new Promise((resolveRun) => {
+      execFile('node', ['--input-type=module', '-e', driver], {
+        encoding: 'utf8',
+        maxBuffer: 64 * 1024 * 1024,
+        env: {
+          ...process.env,
+          ...(opts.effort ? { MIXDOG_AGENT_EFFORT: opts.effort } : {}),
+          ...(opts.env || {}),
+        },
+      }, (err, stdout, stderrOut) => {
+        if (err) resolveRun({ raw: String(stdout || '') + String(stderrOut || ''), ok: false });
+        else resolveRun({ raw: String(stdout || ''), ok: true });
+      });
+    });
+    return { sessionId: extractSessionId(raw), ok, ms: Date.now() - started, raw };
+  },
+  async mixdog(task, opts) {
     const driver = [
       `import { runHeadlessRole } from ${JSON.stringify(HEADLESS)};`,
       `const out = [];`,
@@ -74,10 +172,8 @@ const RUNNERS = {
       `process.exit(code);`,
     ].join('\n');
     const started = Date.now();
-    let raw = '';
-    let ok = false;
-    try {
-      raw = execFileSync('node', ['--input-type=module', '-e', driver], {
+    const { raw, ok } = await new Promise((resolveRun) => {
+      execFile('node', ['--input-type=module', '-e', driver], {
         encoding: 'utf8',
         maxBuffer: 64 * 1024 * 1024,
         env: {
@@ -86,16 +182,67 @@ const RUNNERS = {
           ...(opts.fast ? { MIXDOG_AGENT_FAST: '1' } : {}),
           ...(opts.env || {}),
         },
+      }, (err, stdout, stderrOut) => {
+        if (err) resolveRun({ raw: String(stdout || '') + String(stderrOut || ''), ok: false });
+        else resolveRun({ raw: String(stdout || ''), ok: true });
       });
-      ok = true;
-    } catch (e) {
-      raw = String(e.stdout || '') + String(e.stderr || '');
-      ok = false;
-    }
+    });
     return { sessionId: extractSessionId(raw), ok, ms: Date.now() - started, raw };
   },
-  codex() {
-    throw new Error('runner "codex" not implemented (slot: codex exec --json). Cross-CLI compare is a separate mode.');
+  // Cross-CLI runner: codex exec --json (JSONL events). Prompt goes via stdin
+  // ("-") so no shell-quoting issues on Windows (.cmd shim needs shell:true).
+  // Sandbox is read-only: review tasks must not mutate the repo. Usage comes
+  // from turn.completed events; the final agent_message is kept for judging.
+  async codex(task, opts) {
+    const args = ['exec', '--json', '--skip-git-repo-check', '-s', 'read-only',
+      '-C', task.cwd ? resolve(task.cwd) : process.cwd()];
+    if (opts.model) args.push('-m', opts.model);
+    if (opts.effort) args.push('-c', `model_reasoning_effort="${opts.effort}"`);
+    args.push('-');
+    const started = Date.now();
+    const { raw, ok } = await new Promise((resolveRun) => {
+      const child = spawn('codex', args, { shell: true, env: { ...process.env, ...(opts.env || {}) } });
+      let out = '';
+      let errOut = '';
+      child.stdout.on('data', (d) => { out += d; });
+      child.stderr.on('data', (d) => { errOut += d; });
+      child.on('error', (e) => resolveRun({ raw: `${out}\n${errOut}\n${e.message}`, ok: false }));
+      child.on('close', (code) => resolveRun({ raw: code === 0 ? out : `${out}\n${errOut}`, ok: code === 0 }));
+      child.stdin.write(String(task.prompt || ''));
+      child.stdin.end();
+    });
+    const ms = Date.now() - started;
+    // Parse JSONL: thread id, summed usage, final agent message.
+    let threadId = null;
+    const usage = { uncached_in: 0, cached_in: 0, out_tokens: 0 };
+    let finalMessage = '';
+    for (const line of raw.split(/\r?\n/)) {
+      if (!line.trim().startsWith('{')) continue;
+      let ev = null;
+      try { ev = JSON.parse(line); } catch { continue; }
+      if (ev?.thread_id && !threadId) threadId = String(ev.thread_id);
+      const u = ev?.usage || ev?.turn?.usage || null;
+      if (u) {
+        usage.uncached_in += num(u.input_tokens) - num(u.cached_input_tokens);
+        usage.cached_in += num(u.cached_input_tokens);
+        usage.out_tokens += num(u.output_tokens);
+      }
+      const item = ev?.item;
+      if (item && (item.type === 'agent_message' || item.item_type === 'agent_message') && item.text) {
+        finalMessage = String(item.text);
+      }
+    }
+    const totalIn = usage.uncached_in + usage.cached_in;
+    const card = {
+      session: threadId || `codex-${started}`,
+      wall_ms: ms,
+      uncached_in: usage.uncached_in,
+      cached_in: usage.cached_in,
+      out_tokens: usage.out_tokens,
+      cache_ratio: totalIn > 0 ? Math.round((usage.cached_in / totalIn) * 1000) / 1000 : 0,
+      completed: ok ? 1 : 0,
+    };
+    return { sessionId: card.session, ok, ms, raw, card, finalMessage };
   },
   claude() {
     throw new Error('runner "claude" not implemented (slot: claude -p --output-format json). Cross-CLI compare is a separate mode.');
@@ -106,6 +253,7 @@ function scoreSessions(ids) {
   if (!ids.length) return null;
   const raw = execFileSync('node', [TASK_BENCH, '--session', ids.join(','), '--group', '--json'], {
     encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
+    env: { ...process.env, ...(opts?.env || {}) },
   });
   const s = raw.replace(/^\uFEFF/, '');
   const i = s.indexOf('{');
@@ -116,12 +264,13 @@ function sleepMs(ms) {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, ms));
 }
 
-function scoreSession(sessionId, { attempts = 5 } = {}) {
+function scoreSession(sessionId, { attempts = 5, env = null } = {}) {
   let lastError = null;
   for (let i = 0; i < attempts; i += 1) {
     try {
       const raw = execFileSync('node', [TASK_BENCH, '--session', sessionId, '--json'], {
         encoding: 'utf8', maxBuffer: 64 * 1024 * 1024,
+        env: { ...process.env, ...(env || {}) },
       });
       const s = raw.replace(/^\uFEFF/, '');
       const j = JSON.parse(s.slice(Math.max(0, s.indexOf('{'))));
@@ -189,25 +338,62 @@ const opts = {
 };
 process.stderr.write(`[bench-run] model=${opts.model || '(agent default)'} provider=${opts.provider || '(agent default)'} effort=${opts.effort || '-'} fast=${opts.fast} env=${JSON.stringify(_env)}\n`);
 
-const results = [];
-for (const task of tasks) {
+// --parallel N: run up to N tasks concurrently (default: all). Scoring stays
+// sequential after runs so task-bench reads a settled trace.
+const parallel = Math.max(1, Number.parseInt(argValue('--parallel', String(tasks.length)), 10) || tasks.length);
+// --repeat N: run the whole task set N times (repeat r appended to task id as
+// "id#r") so group averages settle over more samples.
+const repeat = Math.max(1, Number.parseInt(argValue('--repeat', '1'), 10) || 1);
+process.stderr.write(`[bench-run] parallel=${parallel} repeat=${repeat}\n`);
+const runList = [];
+for (let r = 1; r <= repeat; r += 1) {
+  for (const task of tasks) {
+    runList.push(repeat > 1 ? { ...task, id: `${task.id || task.agent}#${r}` } : task);
+  }
+}
+
+async function runTask(task) {
   process.stderr.write(`[bench-run] round=${round} runner=${runnerName} task=${task.id || task.agent} ...\n`);
   let r;
-  try { r = runner(task, opts); }
-  catch (e) { console.error(`[bench-run] runner error: ${e.message}`); process.exit(1); }
-  process.stderr.write(`[bench-run]   -> ${r.ok ? 'ok' : 'FAIL'} ${Math.round(r.ms / 1000)}s session=${r.sessionId || '(none)'}\n`);
-  const result = { id: task.id || null, agent: task.agent || null, ok: r.ok, ms: r.ms, sessionId: r.sessionId };
-  if (r.sessionId) {
-    const scored = scoreSession(r.sessionId);
-    if (scored.ok) {
-      result.card = scored.card;
-      process.stderr.write(`[bench-run]   -> scored ${r.sessionId.slice(0, 22)} turns=${scored.card.turns} tools=${scored.card.tool_calls}\n`);
-    } else {
-      result.scoreError = scored.error;
-      process.stderr.write(`[bench-run]   -> SCORE FAIL ${r.sessionId.slice(0, 22)} ${scored.error}\n`);
+  try { r = await runner(task, opts); }
+  catch (e) {
+    console.error(`[bench-run] runner error (${task.id || task.agent}): ${e.message}`);
+    return { id: task.id || null, agent: task.agent || null, ok: false, ms: 0, sessionId: null };
+  }
+  process.stderr.write(`[bench-run]   -> ${task.id || task.agent}: ${r.ok ? 'ok' : 'FAIL'} ${Math.round(r.ms / 1000)}s session=${r.sessionId || '(none)'}\n`);
+  return {
+    id: task.id || null, agent: task.agent || null, ok: r.ok, ms: r.ms, sessionId: r.sessionId,
+    // External runners (codex/claude) pre-build their scorecard; mixdog scores
+    // from its own trace below. Output text is kept for quality judging.
+    ...(r.card ? { card: r.card } : {}),
+    output: String(r.finalMessage || r.raw || '').slice(-8000),
+  };
+}
+
+const results = new Array(runList.length);
+{
+  let next = 0;
+  async function drain() {
+    for (;;) {
+      const i = next++;
+      if (i >= runList.length) return;
+      results[i] = await runTask(runList[i]);
     }
   }
-  results.push(result);
+  await Promise.all(Array.from({ length: Math.min(parallel, runList.length) }, drain));
+}
+
+for (const result of results) {
+  if (!result.sessionId) continue;
+  if (result.card) continue; // external runner already carries its card
+  const scored = scoreSession(result.sessionId, { env: opts.env });
+  if (scored.ok) {
+    result.card = scored.card;
+    process.stderr.write(`[bench-run]   -> scored ${result.sessionId.slice(0, 22)} turns=${scored.card.turns} tools=${scored.card.tool_calls}\n`);
+  } else {
+    result.scoreError = scored.error;
+    process.stderr.write(`[bench-run]   -> SCORE FAIL ${result.sessionId.slice(0, 22)} ${scored.error}\n`);
+  }
 }
 
 const sessionIds = results.map((r) => r.sessionId).filter(Boolean);