@tangle-network/agent-bench 0.3.7 → 0.3.8

package/src/swe-arena/proposer-fanout.test.mts

@@ -6,8 +6,11 @@ import type { AnalystFinding } from '@tangle-network/agent-eval'
 import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 import {
   defaultGen3Config,
+  defaultGen4Config,
+  defaultGen5Config,
   defaultRound4Config,
   GEN3_IMPROVEMENT_SET,
+  GEN3_PARETO_PARENTS,
   GEN3_SPARE_POOL,
   resolveSmokeInstance,
   type OuterLoopConfig,
@@ -17,9 +20,14 @@ import {
   defaultProposers,
   fanOutLoopsGenerator,
   loadAuthorProfile,
+  materializeParetoParents,
+  mergeAuthorPrompt,
+  parentsPromptSection,
   PROFILES_DIR,
   proposerBuildPrompt,
+  resolveAuthorProfile,
   sliceFindings,
+  type ParetoParentContext,
   type ProposerSpec,
   type SmokeRunner,
 } from './proposer-fanout.mts'
@@ -77,6 +85,134 @@ describe('proposerBuildPrompt', () => {
   })
 })
 
+// ---------------------------------------------------------------------------
+// GEN-4: pinned models, Pareto-parent seeding, and the merge seat.
+// ---------------------------------------------------------------------------
+
+const PARENTS: ParetoParentContext[] = [
+  {
+    commit: 'aaaa111122223333aaaa111122223333aaaa1111',
+    label: 'parent-a',
+    resolvedInstances: ['pydata__xarray-4687', 'sphinx-doc__sphinx-9658'],
+    note: 'mechanical risk scan',
+    diff: 'diff --git a/src/worker-evidence.ts b/src/worker-evidence.ts\n+scan',
+  },
+  {
+    commit: 'bbbb111122223333bbbb111122223333bbbb1111',
+    label: 'parent-b',
+    resolvedInstances: ['django__django-11532'],
+    diff: 'diff --git a/extensions/pi/prompts/supervisor-system.md b/...\n+hidden suite',
+  },
+]
+
+describe('resolveAuthorProfile (pinned models)', () => {
+  it('merges the spec model into the loaded profile as model.default', () => {
+    const spec: ProposerSpec = {
+      name: 'x',
+      profile: 'default-author.profile.json',
+      harness: 'claude',
+      model: 'claude-fable-5',
+    }
+    const profile = resolveAuthorProfile(spec)
+    expect(profile?.name).toBe('swe-arena-default-author')
+    expect(profile?.model?.default).toBe('claude-fable-5')
+  })
+
+  it('synthesizes a minimal named profile for a pinned model without a profile path', () => {
+    const profile = resolveAuthorProfile({ name: 'glm-author', harness: 'opencode', model: 'zai-coding-plan/glm-5.2' })
+    expect(profile?.name).toBe('glm-author-pinned')
+    expect(profile?.model?.default).toBe('zai-coding-plan/glm-5.2')
+  })
+
+  it('is byte-identical to loadAuthorProfile without a pin (gen-3 seats unchanged)', () => {
+    const spec: ProposerSpec = { name: 'x', profile: 'default-author.profile.json', harness: 'claude' }
+    expect(resolveAuthorProfile(spec)).toEqual(loadAuthorProfile(spec))
+    expect(resolveAuthorProfile({ name: 'bare', harness: 'claude' })).toBeUndefined()
+  })
+})
+
+describe('proposerBuildPrompt with pareto parents', () => {
+  it('appends the parents section (evidence + diffs) after the protocol prompt and lens', () => {
+    const spec: ProposerSpec = { name: 'x', harness: 'claude', lens: 'Prefer code-path fixes.' }
+    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS)
+    expect(prompt).toContain('DECLARED CHANGE-SPACE')
+    expect(prompt).toContain('PARETO PARENTS')
+    expect(prompt.indexOf('YOUR AUTHORING LENS')).toBeLessThan(prompt.indexOf('PARETO PARENTS'))
+    expect(prompt).toContain('parent-a (aaaa111122) — resolved: pydata__xarray-4687, sphinx-doc__sphinx-9658')
+    expect(prompt).toContain('mechanical risk scan')
+    expect(prompt).toContain('+scan')
+    expect(prompt).toContain('+hidden suite')
+  })
+
+  it('leaves the prompt untouched when no parents are seeded (gen-3 behavior)', () => {
+    const spec: ProposerSpec = { name: 'x', harness: 'claude' }
+    expect(proposerBuildPrompt({ report: undefined, findings: [] }, spec)).not.toContain('PARETO PARENTS')
+    expect(parentsPromptSection(PARENTS)).toContain('measured evidence')
+  })
+})
+
+describe('mergeAuthorPrompt', () => {
+  const spec: ProposerSpec = { name: 'merge-author', harness: 'claude', merge: true }
+
+  it('keeps the change-space contract and presents BOTH parent diffs with the coherent-union task', () => {
+    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS)
+    expect(prompt).toContain('DECLARED CHANGE-SPACE')
+    expect(prompt).toContain('MERGE SEAT')
+    expect(prompt).toContain('UNION of the')
+    expect(prompt).toContain('KEEPING BOTH BEHAVIORS')
+    expect(prompt).toContain('=== parent parent-a (aaaa111122) ===')
+    expect(prompt).toContain('=== parent parent-b (bbbb111122) ===')
+    expect(prompt).toContain('+scan')
+    expect(prompt).toContain('+hidden suite')
+    // Per-instance evidence rides along for both parents.
+    expect(prompt).toContain('resolved: django__django-11532')
+  })
+
+  it('fails loud with fewer than two parents', () => {
+    expect(() => mergeAuthorPrompt({ report: undefined, findings: [] }, spec, [PARENTS[0]!])).toThrow(/>=2/)
+    expect(() => proposerBuildPrompt({ report: undefined, findings: [] }, spec, [])).toThrow(/>=2/)
+  })
+})
+
+describe('defaultGen4Config', () => {
+  const config = defaultGen4Config()
+
+  it('seats four proposers (one candidate slot each) with the pinned glm model and the merge seat', () => {
+    expect(config.proposers).toHaveLength(4)
+    expect(config.populationSize).toBe(4)
+    const byName = Object.fromEntries(config.proposers!.map((p) => [p.name, p]))
+    expect(byName['claude-author']).toMatchObject({ harness: 'claude', profile: 'default-author.profile.json' })
+    expect(byName['claude-author']!.model).toBeUndefined()
+    expect(byName['glm-author']).toMatchObject({ harness: 'opencode', model: 'zai-coding-plan/glm-5.2' })
+    expect(byName['codex-author']).toMatchObject({ harness: 'codex' })
+    expect(byName['merge-author']).toMatchObject({ harness: 'claude', merge: true })
+  })
+
+  it('drops the codex seat (and shrinks the population) when includeCodex is false', () => {
+    const noCodex = defaultGen4Config(undefined, { includeCodex: false })
+    expect(noCodex.proposers!.map((p) => p.name)).toEqual(['claude-author', 'glm-author', 'merge-author'])
+    expect(noCodex.populationSize).toBe(3)
+  })
+
+  it('seeds the gen-3 frontier as pareto parents and points at a fresh gen4 outDir + baseline artifact', () => {
+    expect(config.paretoParents).toEqual(GEN3_PARETO_PARENTS)
+    expect(config.paretoParents!.map((p) => p.commit.slice(0, 10))).toEqual(['cc0d95584c', 'a7a2a982e5'])
+    expect(config.outDir).toContain('gen4')
+    expect(config.premeasuredBaselinePath).toContain('gen4')
+    expect(config.premeasuredBaselinePath).not.toBe(defaultGen3Config().premeasuredBaselinePath)
+  })
+
+  it('keeps the gen-3 protocol frame: 6-instance set, 2 reps, 1 generation, prefilter, 2-rep holdout', () => {
+    expect(config.instances).toEqual([...GEN3_IMPROVEMENT_SET])
+    expect(config.repsPerInstance).toBe(2)
+    expect(config.generations).toBe(1)
+    expect(config.prefilter).toEqual({ enabled: true, smokeInstance: 'cheapest-of-set', requireResolved: false })
+    expect(config.holdoutRepsPerInstance).toBe(2)
+    expect(config.holdoutBaseline).toBe('measure')
+    for (const iid of config.instances) expect(config.holdoutInstances).not.toContain(iid)
+  })
+})
+
 describe('loadAuthorProfile', () => {
   it('loads the committed default-author profile (bare: no prompt, no model)', () => {
     const profile = loadAuthorProfile({ name: 'a', profile: 'default-author.profile.json', harness: 'claude' })
@@ -360,6 +496,40 @@ describe('fanOutLoopsGenerator', () => {
     await expect(gen.generate(generatorArgs(1))).rejects.toThrow(/populationSize must equal/)
   })
 
+  it('materializes pareto parents from real commits (full diff, truncation, missing-commit fail-loud)', async () => {
+    await writeFile(join(loopsRepo, 'src.ts'), 'base\nchanged\n')
+    await git(['add', '-A'], loopsRepo)
+    await git(['commit', '-q', '-m', 'parent change'], loopsRepo)
+    const commit = await git(['rev-parse', 'HEAD'], loopsRepo)
+    const seed = { commit, label: 'p1', resolvedInstances: ['inst-a'] }
+    const [parent] = await materializeParetoParents(loopsRepo, [seed])
+    expect(parent).toMatchObject(seed)
+    expect(parent!.diff).toContain('parent change')
+    expect(parent!.diff).toContain('+changed')
+    const [truncated] = await materializeParetoParents(loopsRepo, [seed], 40)
+    expect(truncated!.diff).toContain('[... diff truncated at 40 chars ...]')
+    await expect(
+      materializeParetoParents(loopsRepo, [{ commit: 'deadbeef'.repeat(5), label: 'ghost', resolvedInstances: [] }]),
+    ).rejects.toThrow(/not found/)
+  })
+
+  it('refuses a merge seat without >=2 materialized parents', () => {
+    const config = baseConfig([{ name: 'merge-author', harness: 'claude', merge: true }])
+    expect(() => fanOutLoopsGenerator(config, { author: async () => ({ applied: false, summary: '' }) })).toThrow(
+      /merge proposer/,
+    )
+    expect(() =>
+      fanOutLoopsGenerator(config, {
+        author: async () => ({ applied: false, summary: '' }),
+        parents: [PARENTS[0]!],
+      }),
+    ).toThrow(/needs >=2|merge proposer/)
+    // With two parents the generator constructs.
+    expect(() =>
+      fanOutLoopsGenerator(config, { author: async () => ({ applied: false, summary: '' }), parents: PARENTS }),
+    ).not.toThrow()
+  })
+
   it('rejects duplicate proposer names and empty proposer lists', () => {
     expect(() =>
       fanOutLoopsGenerator(baseConfig([
@@ -370,3 +540,80 @@ describe('fanOutLoopsGenerator', () => {
     expect(() => fanOutLoopsGenerator({ ...defaultRound4Config(), loopsRepo, outDir })).toThrow(/empty/)
   })
 })
+
+// ---------------------------------------------------------------------------
+// GEN-5: config bundle + prompt sections (MAP+TOOLBOX briefing, activation
+// contract) on every seat, merge included.
+// ---------------------------------------------------------------------------
+
+describe('defaultGen5Config', () => {
+  const config = defaultGen5Config()
+
+  it('keeps the full gen-4 shape (seats, parents, 2 reps, deferred holdout) under a fresh gen5 outDir', () => {
+    expect(config.proposers!.map((p) => p.name)).toEqual(
+      defaultGen4Config().proposers!.map((p) => p.name),
+    )
+    expect(config.populationSize).toBe(config.proposers!.length)
+    expect(config.paretoParents).toEqual(GEN3_PARETO_PARENTS)
+    expect(config.repsPerInstance).toBe(2)
+    expect(config.generations).toBe(1)
+    expect(config.holdoutRepsPerInstance).toBe(2)
+    expect(config.outDir).toContain('gen5')
+    expect(config.premeasuredBaselinePath).toContain('gen5')
+  })
+
+  it('turns on the five gen-5 mechanisms as a unit', () => {
+    expect(config.scoreSplit).toEqual({ publicCount: 4 })
+    expect(config.briefing).toBe('map-toolbox-v1')
+    expect(config.activationGate).toBe(true)
+    expect(config.rolloutLedger).toEqual({ enabled: true })
+    expect(config.lineage).toBe(true)
+    expect(config.priorEvidenceDirs!.some((d) => d.includes('gen4'))).toBe(true)
+  })
+
+  it('drops the codex seat when includeCodex is false (population follows)', () => {
+    const noCodex = defaultGen5Config(undefined, { includeCodex: false })
+    expect(noCodex.proposers!.map((p) => p.name)).toEqual(['claude-author', 'glm-author', 'merge-author'])
+    expect(noCodex.populationSize).toBe(3)
+  })
+})
+
+describe('gen-5 prompt sections', () => {
+  const briefing = {
+    indexPath: '/tmp/out/evidence-index.md',
+    briefingText: 'TOOLBOX: traces CLI, analysts, AxLLM. PERMISSION: subagents.',
+    briefingSource: 'default' as const,
+  }
+
+  it('appends EVIDENCE MAP + briefing + activation contract after the protocol prompt', () => {
+    const spec: ProposerSpec = { name: 'x', harness: 'claude' }
+    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, [], {
+      briefing,
+      activationGate: true,
+    })
+    expect(prompt).toContain('DECLARED CHANGE-SPACE')
+    expect(prompt).toContain('EVIDENCE MAP: /tmp/out/evidence-index.md')
+    expect(prompt).toContain('TOOLBOX')
+    expect(prompt).toContain('ACTIVATION PREDICATE')
+    expect(prompt.indexOf('DECLARED CHANGE-SPACE')).toBeLessThan(prompt.indexOf('EVIDENCE MAP'))
+  })
+
+  it('the merge seat gets the gen-5 sections too', () => {
+    const spec: ProposerSpec = { name: 'merge-author', harness: 'claude', merge: true }
+    const prompt = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS, {
+      briefing,
+      activationGate: true,
+    })
+    expect(prompt).toContain('MERGE SEAT')
+    expect(prompt).toContain('EVIDENCE MAP')
+    expect(prompt).toContain('ACTIVATION PREDICATE')
+  })
+
+  it('leaves gen-3/gen-4 prompts byte-identical when no extras are passed', () => {
+    const spec: ProposerSpec = { name: 'x', harness: 'claude' }
+    const legacy = proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS)
+    expect(legacy).not.toContain('EVIDENCE MAP')
+    expect(legacy).not.toContain('ACTIVATION PREDICATE')
+    expect(proposerBuildPrompt({ report: undefined, findings: [] }, spec, PARENTS, {})).toBe(legacy)
+  })
+})

package/src/swe-arena/proposer-provenance.mts

@@ -0,0 +1,177 @@
+/**
+ * GEN-4 proposer-model provenance — pin each author seat's MODEL IDENTITY in
+ * the run record at t=0, before any author shot fires.
+ *
+ * Why: a proposer spec may pin a model explicitly (`spec.model` → the harness
+ * CLI's `-m` flag via the author profile's `model.default`), but the claude
+ * seat deliberately does NOT pass `-m` — the CLI runs on its logged-in
+ * account, whose resolved model comes from its own settings. Verified against
+ * claude CLI 2.1.217: `--model` IS supported headless (`-p`), but the run's
+ * provenance must not depend on a flag we chose not to send. So the capture
+ * records what is VERIFIABLE at launch for every configured harness:
+ *
+ *   - `<harness> --version` output (the exact CLI build that authored),
+ *   - the claude settings default model (`~/.claude/settings.json` `model`),
+ *   - `codex login status` (the codex seat must be authed or the run refuses
+ *     at t=0 — a mid-run auth failure would silently kill one candidate slot),
+ *   - the spec's pinned model id (null when the seat rides the CLI default).
+ *
+ * The capture FAILS LOUD on a missing/broken harness binary: populationSize
+ * must equal proposers.length, so a dead seat cannot be skipped at runtime —
+ * the config decides membership, this guard proves it at launch.
+ */
+
+import { readFileSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
+import {
+  DEFAULT_GEPA_PYTHON,
+  isGepaSeat,
+  loadGepaMethodFactory,
+  probeGepaRuntime,
+  type CampaignModuleImport,
+} from './gepa-seat.mts'
+import { run } from './proc.ts'
+import type { ProposerSpec } from './proposer-fanout.mts'
+
+export interface ProposerModelProvenance {
+  name: string
+  /** Absent on a GEN-6 engine seat (see `engine`). */
+  harness: ProposerSpec['harness']
+  /** Explicit model pin from the spec (threaded as `-m`), or null when the
+   *  seat runs the CLI's own resolved default. */
+  pinnedModel: string | null
+  /** `<harness> --version` stdout (trimmed). For a GEN-6 gepa seat this is
+   *  the bridge python's `--version` output — the runtime that authors. */
+  harnessVersion: string
+  /** claude seats only: the settings default model the logged-in CLI resolves
+   *  when no `-m` is passed. Null when unreadable (recorded, never fatal —
+   *  the version capture is the hard gate). */
+  settingsModel: string | null
+  /** codex seats only: `codex login status` stdout (trimmed). */
+  authStatus: string | null
+  merge: boolean
+  /** GEN-6 gepa seat: the engine name from the spec. */
+  engine?: 'gepa' | 'omni'
+  /** GEN-6 gepa seat: the ONE change-space file GEPA optimizes. */
+  surface?: string
+  /** GEN-6 gepa seat: installed gepa version ('source' for a source pin). */
+  gepaVersion?: string
+  /** GEN-6 gepa seat: the Python bridge module the seat runs. */
+  bridge?: string
+}
+
+export interface ProvenanceCaptureRecord {
+  schema: 'swe-arena.proposer-provenance.v1'
+  capturedAt: string
+  proposers: ProposerModelProvenance[]
+}
+
+/** Exec seam — test-injectable. Mirrors `run` from proc.ts. */
+export type VersionExec = (
+  command: string,
+  args: string[],
+) => Promise<{ code: number | null; stdout: string; stderr: string }>
+
+const defaultExec: VersionExec = async (command, args) => {
+  const res = await run(command, args, { timeoutMs: 30_000 })
+  return { code: res.code, stdout: res.stdout, stderr: res.stderr }
+}
+
+/** Read the claude CLI's settings default model. Pure over injected reader. */
+export function claudeSettingsModel(
+  readFile: (path: string) => string = (p) => readFileSync(p, 'utf8'),
+  settingsPath = join(homedir(), '.claude', 'settings.json'),
+): string | null {
+  try {
+    const parsed = JSON.parse(readFile(settingsPath)) as { model?: unknown }
+    return typeof parsed.model === 'string' && parsed.model.length > 0 ? parsed.model : null
+  } catch {
+    return null
+  }
+}
+
+/** Capture per-proposer model provenance. Throws when any configured harness
+ *  binary is missing/broken, when a codex seat is not logged in, or — GEN-6 —
+ *  when a gepa seat's runtime is incomplete: the installed agent-eval must
+ *  export `gepaOptimizationMethod` and the Python bridge + GEPA engine must
+ *  import (probeGepaRuntime carries the exact install instructions). A dead
+ *  seat fails the launch at t=0, never a mid-run candidate slot. */
+export async function captureProposerProvenance(
+  proposers: ProposerSpec[],
+  deps: {
+    exec?: VersionExec
+    readSettingsModel?: () => string | null
+    importCampaign?: CampaignModuleImport
+  } = {},
+): Promise<ProvenanceCaptureRecord> {
+  const exec = deps.exec ?? defaultExec
+  const readSettingsModel = deps.readSettingsModel ?? (() => claudeSettingsModel())
+  const versionByHarness = new Map<string, string>()
+  const authByHarness = new Map<string, string>()
+  const gepaBySeat = new Map<string, { pythonVersion: string; gepaVersion: string }>()
+  const gepaSeats = proposers.filter(isGepaSeat)
+  if (gepaSeats.length > 0) {
+    // Node side first: the adapter export (fails loud with upgrade hint).
+    await loadGepaMethodFactory(...(deps.importCampaign ? [deps.importCampaign] : []))
+    for (const seat of gepaSeats) {
+      gepaBySeat.set(seat.name, await probeGepaRuntime(seat.python ?? DEFAULT_GEPA_PYTHON, exec, seat.name))
+    }
+  }
+  const harnesses = [...new Set(proposers.map((p) => p.harness))].filter(
+    (h): h is NonNullable<ProposerSpec['harness']> => h !== undefined,
+  )
+  for (const harness of harnesses) {
+    const res = await exec(harness, ['--version'])
+    if (res.code !== 0) {
+      throw new Error(
+        `proposer provenance: '${harness} --version' failed (rc=${res.code}) — the ${harness} seat cannot author. ` +
+          `stderr: ${res.stderr.slice(0, 300)}`,
+      )
+    }
+    versionByHarness.set(harness, res.stdout.trim())
+    if (harness === 'codex') {
+      const auth = await exec('codex', ['login', 'status'])
+      const authOut = auth.stdout + auth.stderr
+      const authed = /logged in/i.test(authOut) && !/not logged in/i.test(authOut)
+      if (auth.code !== 0 || !authed) {
+        throw new Error(
+          `proposer provenance: codex seat configured but 'codex login status' says not authed ` +
+            `(rc=${auth.code}, out=${(auth.stdout + auth.stderr).trim().slice(0, 200)})`,
+        )
+      }
+      authByHarness.set('codex', (auth.stdout + auth.stderr).trim())
+    }
+  }
+  return {
+    schema: 'swe-arena.proposer-provenance.v1',
+    capturedAt: new Date().toISOString(),
+    proposers: proposers.map((spec): ProposerModelProvenance => {
+      if (isGepaSeat(spec)) {
+        const probe = gepaBySeat.get(spec.name)!
+        return {
+          name: spec.name,
+          harness: undefined,
+          pinnedModel: null,
+          harnessVersion: probe.pythonVersion,
+          settingsModel: null,
+          authStatus: null,
+          merge: false,
+          engine: spec.engine,
+          surface: spec.surface,
+          gepaVersion: probe.gepaVersion,
+          bridge: 'agent_eval_rpc.gepa_bridge',
+        }
+      }
+      return {
+        name: spec.name,
+        harness: spec.harness,
+        pinnedModel: spec.model ?? null,
+        harnessVersion: versionByHarness.get(spec.harness!)!,
+        settingsModel: spec.harness === 'claude' && !spec.model ? readSettingsModel() : null,
+        authStatus: (spec.harness !== undefined ? authByHarness.get(spec.harness) : undefined) ?? null,
+        merge: spec.merge === true,
+      }
+    }),
+  }
+}

package/src/swe-arena/proposer-provenance.test.mts

@@ -0,0 +1,106 @@
+import { describe, expect, it } from 'vitest'
+import {
+  captureProposerProvenance,
+  claudeSettingsModel,
+  type VersionExec,
+} from './proposer-provenance.mts'
+import type { ProposerSpec } from './proposer-fanout.mts'
+
+const ok = (stdout: string) => ({ code: 0, stdout, stderr: '' })
+
+const gen4ish: ProposerSpec[] = [
+  { name: 'claude-author', profile: 'default-author.profile.json', harness: 'claude' },
+  { name: 'glm-author', harness: 'opencode', model: 'zai-coding-plan/glm-5.2' },
+  { name: 'codex-author', harness: 'codex' },
+  { name: 'merge-author', profile: 'default-author.profile.json', harness: 'claude', merge: true },
+]
+
+describe('captureProposerProvenance', () => {
+  const exec: VersionExec = async (command, args) => {
+    if (args[0] === '--version') return ok(`${command}-version 9.9.9`)
+    if (command === 'codex' && args[0] === 'login') return ok('Logged in using ChatGPT')
+    throw new Error(`unexpected exec ${command} ${args.join(' ')}`)
+  }
+
+  it('records pinned model, harness version, settings model, and codex auth per seat', async () => {
+    const record = await captureProposerProvenance(gen4ish, {
+      exec,
+      readSettingsModel: () => 'claude-fable-5',
+    })
+    expect(record.schema).toBe('swe-arena.proposer-provenance.v1')
+    const byName = Object.fromEntries(record.proposers.map((p) => [p.name, p]))
+    // Claude seat: no pin — the CLI's resolved settings model is the record.
+    expect(byName['claude-author']).toMatchObject({
+      harness: 'claude',
+      pinnedModel: null,
+      settingsModel: 'claude-fable-5',
+      harnessVersion: 'claude-version 9.9.9',
+      authStatus: null,
+      merge: false,
+    })
+    // Pinned opencode seat: explicit model id, settings model not consulted.
+    expect(byName['glm-author']).toMatchObject({
+      harness: 'opencode',
+      pinnedModel: 'zai-coding-plan/glm-5.2',
+      settingsModel: null,
+      harnessVersion: 'opencode-version 9.9.9',
+    })
+    // Codex seat: version + auth status captured.
+    expect(byName['codex-author']).toMatchObject({
+      harness: 'codex',
+      pinnedModel: null,
+      authStatus: 'Logged in using ChatGPT',
+    })
+    expect(byName['merge-author']!.merge).toBe(true)
+  })
+
+  it('fails loud when a configured harness binary is missing', async () => {
+    const broken: VersionExec = async (command, args) =>
+      command === 'codex' ? { code: 127, stdout: '', stderr: 'not found' } : exec(command, args)
+    await expect(
+      captureProposerProvenance(gen4ish, { exec: broken, readSettingsModel: () => null }),
+    ).rejects.toThrow(/'codex --version' failed/)
+  })
+
+  it('fails loud when the codex seat is not logged in', async () => {
+    const loggedOut: VersionExec = async (command, args) => {
+      if (args[0] === '--version') return ok(`${command} 1.0.0`)
+      return ok('Not logged in')
+    }
+    await expect(
+      captureProposerProvenance([{ name: 'codex-author', harness: 'codex' }], {
+        exec: loggedOut,
+        readSettingsModel: () => null,
+      }),
+    ).rejects.toThrow(/not authed/)
+  })
+
+  it('runs one version probe per harness, not per proposer', async () => {
+    const calls: string[] = []
+    const counting: VersionExec = async (command, args) => {
+      calls.push(`${command} ${args.join(' ')}`)
+      if (args[0] === '--version') return ok(`${command} 1`)
+      return ok('Logged in using ChatGPT')
+    }
+    await captureProposerProvenance(gen4ish, { exec: counting, readSettingsModel: () => null })
+    expect(calls.filter((c) => c === 'claude --version')).toHaveLength(1)
+    expect(calls.filter((c) => c === 'codex --version')).toHaveLength(1)
+    expect(calls.filter((c) => c === 'codex login status')).toHaveLength(1)
+  })
+})
+
+describe('claudeSettingsModel', () => {
+  it('reads the settings model field', () => {
+    expect(claudeSettingsModel(() => JSON.stringify({ model: 'claude-fable-5' }), '/x')).toBe('claude-fable-5')
+  })
+
+  it('returns null on unreadable/missing/blank settings', () => {
+    expect(
+      claudeSettingsModel(() => {
+        throw new Error('ENOENT')
+      }, '/x'),
+    ).toBeNull()
+    expect(claudeSettingsModel(() => JSON.stringify({}), '/x')).toBeNull()
+    expect(claudeSettingsModel(() => JSON.stringify({ model: '' }), '/x')).toBeNull()
+  })
+})