@tangle-network/agent-bench 0.1.0 → 0.3.6

package/src/swe-temp.ts

@@ -0,0 +1,14 @@
+import { tmpdir } from 'node:os'
+import { isAbsolute } from 'node:path'
+
+/** Docker bind mounts and mkdtemp parents must never inherit a relative operating-system temp root. */
+export function absoluteSweTempDir(): string {
+  const dir = tmpdir()
+  if (!isAbsolute(dir)) {
+    throw new Error(
+      `SWE temporary directory must be absolute, got "${dir}"; ` +
+        'TEMP configures the operating-system temp root, not model temperature. Use TEMPERATURE instead.',
+    )
+  }
+  return dir
+}

package/src/tb-container-executor.mts

@@ -0,0 +1,234 @@
+/**
+ * Executor for workers that must mutate the same Terminal-Bench container the grader inspects.
+ *
+ * The container lifecycle stays owned by Terminal-Bench. This executor only runs a command in the
+ * provided container id and reports the captured process artifact back through the runtime.
+ */
+import { spawn } from 'node:child_process'
+import type {
+  Agent,
+  AgentProfile,
+  AgentSpec,
+  Executor,
+  ExecutorContext,
+  ExecutorFactory,
+  ExecutorResult,
+  MakeWorkerAgent,
+  Runtime,
+  Spend,
+} from '../../src/runtime/index'
+
+export interface TbExecOutput {
+  /** Primary artifact consumed by drivers that read `{ content }`. */
+  readonly content: string
+  readonly stdout: string
+  readonly stderr: string
+  readonly exitCode: number | null
+  readonly containerId: string
+  readonly command: string
+}
+
+export type ParseUsage = (io: {
+  stdout: string
+  stderr: string
+  exitCode: number | null
+}) => { input: number; output: number; usd?: number } | undefined
+
+export interface TbContainerConfig {
+  readonly containerId?: string
+  readonly containerEnvVar?: string
+  readonly workdir?: string
+  readonly shell?: string
+  readonly env?: Readonly<Record<string, string>>
+  readonly wrapCommand?: (task: unknown) => string
+  readonly parseUsage?: ParseUsage
+  readonly budgetExempt?: boolean
+  readonly dockerBin?: string
+  readonly failOnNonZeroExit?: boolean
+  readonly runtime?: Runtime
+}
+
+const DEFAULT_ENV_VAR = 'TB_TARGET_CONTAINER'
+
+function resolveContainerId(config: TbContainerConfig): string {
+  const envVar = config.containerEnvVar ?? DEFAULT_ENV_VAR
+  const id = config.containerId ?? process.env[envVar]
+  if (!id || id.trim().length === 0) {
+    throw new Error(
+      `tbContainerExecutor: no target container - set config.containerId or the ${envVar} env var ` +
+        `to the id of the terminal-bench container the verifier grades.`,
+    )
+  }
+  return id.trim()
+}
+
+function taskToCommand(task: unknown): string {
+  if (typeof task === 'string') return task
+  if (task && typeof task === 'object') {
+    const obj = task as Record<string, unknown>
+    for (const k of ['command', 'prompt', 'content', 'task', 'message']) {
+      if (typeof obj[k] === 'string') return obj[k] as string
+    }
+  }
+  return JSON.stringify(task)
+}
+
+/**
+ * Compose the `docker exec` argv that runs `command` inside `containerId` via `<shell> -c`.
+ * Exported so tests can verify the container path without a live Docker daemon.
+ */
+export function buildTbDockerExecArgs(
+  containerId: string,
+  command: string,
+  opts: { shell?: string; workdir?: string; env?: Readonly<Record<string, string>> } = {},
+): string[] {
+  const out: string[] = ['exec', '-i']
+  if (opts.workdir) out.push('--workdir', opts.workdir)
+  if (opts.env) {
+    for (const [k, v] of Object.entries(opts.env)) {
+      if (typeof v !== 'string' || v.length === 0) continue
+      out.push('-e', `${k}=${v}`)
+    }
+  }
+  out.push(containerId, opts.shell ?? '/bin/sh', '-c', command)
+  return out
+}
+
+function contentRef(prefix: string, value: unknown): string {
+  let str: string
+  try {
+    str = JSON.stringify(value) ?? String(value)
+  } catch {
+    str = String(value)
+  }
+  let h = 0x811c9dc5
+  for (let i = 0; i < str.length; i += 1) {
+    h ^= str.charCodeAt(i)
+    h = Math.imul(h, 0x01000193)
+  }
+  return `${prefix}:${(h >>> 0).toString(16).padStart(8, '0')}`
+}
+
+export function createTbContainerExecutor(
+  config: TbContainerConfig = {},
+): ExecutorFactory<TbExecOutput> {
+  return (_spec: AgentSpec, ctx: ExecutorContext): Executor<TbExecOutput> => {
+    const containerId = resolveContainerId(config)
+    const dockerBin = config.dockerBin ?? 'docker'
+    const metered = config.parseUsage !== undefined
+    const budgetExempt = config.budgetExempt ?? !metered
+    const runtime: Runtime = config.runtime ?? 'tb-container'
+
+    const controller = new AbortController()
+    const abortIfSignalled = () => {
+      if (ctx.signal.aborted) controller.abort()
+    }
+    abortIfSignalled()
+    if (!ctx.signal.aborted) ctx.signal.addEventListener('abort', abortIfSignalled, { once: true })
+
+    let proc: ReturnType<typeof spawn> | undefined
+    let artifact: ExecutorResult<TbExecOutput> | undefined
+
+    return {
+      runtime,
+      budgetExempt,
+      execute(task, signal): Promise<ExecutorResult<TbExecOutput>> {
+        const command = (config.wrapCommand ?? taskToCommand)(task)
+        const args = buildTbDockerExecArgs(containerId, command, {
+          ...(config.shell ? { shell: config.shell } : {}),
+          ...(config.workdir ? { workdir: config.workdir } : {}),
+          ...(config.env ? { env: config.env } : {}),
+        })
+        const started = Date.now()
+
+        return new Promise<ExecutorResult<TbExecOutput>>((resolve, reject) => {
+          const child = spawn(dockerBin, args, { stdio: ['ignore', 'pipe', 'pipe'] })
+          proc = child
+
+          const kill = () => {
+            if (child.exitCode === null && !child.killed) child.kill('SIGKILL')
+          }
+          if (signal.aborted || controller.signal.aborted) kill()
+          else {
+            signal.addEventListener('abort', kill, { once: true })
+            controller.signal.addEventListener('abort', kill, { once: true })
+          }
+
+          const outChunks: string[] = []
+          const errChunks: string[] = []
+          child.stdout?.on('data', (d: Buffer) => outChunks.push(d.toString('utf8')))
+          child.stderr?.on('data', (d: Buffer) => errChunks.push(d.toString('utf8')))
+
+          child.once('error', (err) => {
+            signal.removeEventListener('abort', kill)
+            controller.signal.removeEventListener('abort', kill)
+            reject(
+              new Error(
+                `tbContainerExecutor: failed to spawn '${dockerBin} exec' on ${containerId}: ${err.message}`,
+                { cause: err },
+              ),
+            )
+          })
+
+          child.once('close', (code) => {
+            signal.removeEventListener('abort', kill)
+            controller.signal.removeEventListener('abort', kill)
+            const stdout = outChunks.join('')
+            const stderr = errChunks.join('')
+            const exitCode = code
+
+            if (config.failOnNonZeroExit && exitCode !== 0) {
+              reject(
+                new Error(
+                  `tbContainerExecutor: command exited ${exitCode} in ${containerId}: ${stderr.slice(0, 200)}`,
+                ),
+              )
+              return
+            }
+
+            const out: TbExecOutput = { content: stdout, stdout, stderr, exitCode, containerId, command }
+            const usage = config.parseUsage?.({ stdout, stderr, exitCode })
+            const spent: Spend = metered
+              ? {
+                  iterations: 1,
+                  tokens: { input: usage?.input ?? 0, output: usage?.output ?? 0 },
+                  usd: usage?.usd ?? 0,
+                  ms: Date.now() - started,
+                }
+              : // Budget-exempt commands report no conserved token, dollar, or iteration spend.
+                { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0, ms: Date.now() - started }
+
+            artifact = { outRef: contentRef(String(runtime), out), out, spent }
+            resolve(artifact)
+          })
+        })
+      },
+      teardown(_grace): Promise<{ destroyed: boolean }> {
+        controller.abort()
+        if (proc && proc.exitCode === null && !proc.killed) proc.kill('SIGKILL')
+        return Promise.resolve({ destroyed: true })
+      },
+      resultArtifact() {
+        if (!artifact) {
+          throw new Error('tbContainerExecutor: resultArtifact() read before execute() settled')
+        }
+        return artifact
+      },
+    }
+  }
+}
+
+export function makeTbContainerWorkerAgent(config: TbContainerConfig = {}): MakeWorkerAgent {
+  const factory = createTbContainerExecutor(config)
+  return (rawProfile) => {
+    const p = (rawProfile ?? {}) as { name?: unknown }
+    const name = typeof p.name === 'string' && p.name.length > 0 ? p.name : 'tb-worker'
+    const spec: AgentSpec = { profile: rawProfile as AgentProfile, harness: null }
+    const ctx: ExecutorContext = { signal: new AbortController().signal, seams: {} }
+    const executor = factory(spec, ctx)
+    return { name, act: async () => '', executorSpec: { ...spec, executor } } as Agent<
+      unknown,
+      unknown
+    > & { executorSpec: AgentSpec }
+  }
+}

package/src/tb-container-executor.test.mts

@@ -0,0 +1,99 @@
+import assert from 'node:assert/strict'
+import { chmod, mkdtemp, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import type { AgentSpec, ExecutorContext } from '@tangle-network/agent-runtime/loops'
+import { buildTbDockerExecArgs, createTbContainerExecutor } from './tb-container-executor.mts'
+
+const spec: AgentSpec = { profile: { name: 'tb-test-worker' }, harness: null }
+
+function context(): ExecutorContext {
+  return { signal: new AbortController().signal, seams: {} }
+}
+
+async function executable(name: string, body: string): Promise<string> {
+  const dir = await mkdtemp(path.join(tmpdir(), 'tb-container-executor-'))
+  const file = path.join(dir, name)
+  await writeFile(file, body)
+  await chmod(file, 0o755)
+  return file
+}
+
+async function main(): Promise<void> {
+  assert.deepEqual(
+    buildTbDockerExecArgs('cid', 'echo ok', {
+      workdir: '/app',
+      env: { A: 'B', EMPTY: '' },
+      shell: '/bin/bash',
+    }),
+    ['exec', '-i', '--workdir', '/app', '-e', 'A=B', 'cid', '/bin/bash', '-c', 'echo ok'],
+    'docker exec argv is deterministic and skips empty env values',
+  )
+
+  assert.throws(
+    () => createTbContainerExecutor({ containerEnvVar: 'TB_CONTAINER_EXECUTOR_TEST_MISSING' })(spec, context()),
+    /no target container/,
+    'missing target container fails before spawning',
+  )
+
+  const fakeDocker = await executable(
+    'fake-docker',
+    `#!/bin/sh
+printf 'argv:%s\\n' "$*"
+`,
+  )
+
+  const metered = createTbContainerExecutor({
+    containerId: 'cid',
+    dockerBin: fakeDocker,
+    workdir: '/work',
+    env: { OPENAI_BASE_URL: 'http://router.test' },
+    parseUsage: () => ({ input: 7, output: 11, usd: 0.004 }),
+  })(spec, context())
+
+  assert.equal(metered.budgetExempt, false, 'usage parser makes the executor metered')
+  const meteredResult = await metered.execute({ command: 'echo hello' }, new AbortController().signal)
+  assert.equal(meteredResult.out.containerId, 'cid')
+  assert.equal(meteredResult.out.command, 'echo hello')
+  assert.match(
+    meteredResult.out.stdout,
+    /argv:exec -i --workdir \/work -e OPENAI_BASE_URL=http:\/\/router\.test cid \/bin\/sh -c echo hello/,
+  )
+  assert.deepEqual(meteredResult.spent.tokens, { input: 7, output: 11 })
+  assert.equal(meteredResult.spent.usd, 0.004)
+  assert.equal(metered.resultArtifact().out, meteredResult.out)
+
+  const free = createTbContainerExecutor({ containerId: 'cid', dockerBin: fakeDocker })(spec, context())
+  assert.equal(free.budgetExempt, true, 'unmetered shell commands are explicit budget-exempt work')
+  const freeResult = await free.execute('printf ok', new AbortController().signal)
+  assert.equal(freeResult.spent.iterations, 0)
+  assert.deepEqual(freeResult.spent.tokens, { input: 0, output: 0 })
+  assert.equal(freeResult.spent.usd, 0)
+
+  const failingDocker = await executable(
+    'failing-docker',
+    `#!/bin/sh
+echo failed >&2
+exit 7
+`,
+  )
+  const strict = createTbContainerExecutor({
+    containerId: 'cid',
+    dockerBin: failingDocker,
+    failOnNonZeroExit: true,
+  })(spec, context())
+  await assert.rejects(
+    strict.execute('do work', new AbortController().signal),
+    /command exited 7/,
+    'strict mode treats non-zero command exit as infrastructure failure',
+  )
+
+  const lenient = createTbContainerExecutor({ containerId: 'cid', dockerBin: failingDocker })(spec, context())
+  const lenientResult = await lenient.execute('do work', new AbortController().signal)
+  assert.equal(lenientResult.out.exitCode, 7, 'default mode returns non-zero exits as task artifacts')
+  assert.match(lenientResult.out.stderr, /failed/)
+
+  console.log('tb-container-executor.test: OK')
+}
+
+void main()

package/src/tb-supervisor-sidecar.mts

@@ -0,0 +1,222 @@
+/**
+ * Host process that lets an in-container Terminal-Bench agent delegate work to child agents.
+ *
+ * It exposes the coordination MCP on a container-reachable host address, then runs every spawned
+ * worker in the same Docker container the benchmark grader inspects.
+ */
+
+import { appendFileSync, writeFileSync } from 'node:fs'
+import {
+  type Agent,
+  createExecutorRegistry,
+  createSupervisor,
+  InMemoryResultBlobStore,
+  InMemorySpawnJournal,
+  type Scope,
+} from '../../src/runtime/index'
+import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp'
+import { makeTbContainerWorkerAgent, type ParseUsage } from './tb-container-executor.mts'
+
+const CONTAINER_ID = (process.env.TB_TARGET_CONTAINER ?? '').trim()
+const ROUTER_KEY = process.env.OPENAI_API_KEY ?? ''
+const ROUTER_BASE = process.env.OPENAI_BASE_URL ?? 'https://router.tangle.tools/v1'
+const WORKER_MODEL = process.env.WORKER_MODEL ?? 'zai-coding-plan/glm-5.2'
+const DOCKER_BRIDGE_GATEWAY = process.env.DOCKER_BRIDGE_GATEWAY ?? '172.17.0.1'
+const PORT_FILE = process.env.TB_SIDECAR_PORT_FILE ?? '.tb-sidecar-port'
+const LOG_FILE = process.env.TB_SIDECAR_LOG ?? '.tb-sidecar-events.jsonl'
+const WORKER_WORKDIR = process.env.TB_WORKER_WORKDIR ?? '/app'
+const WORKER_CONFIG_PATH = '/installed-agent/opencode-worker.json'
+
+if (!CONTAINER_ID) {
+  console.error('[sidecar] FATAL: TB_TARGET_CONTAINER not set')
+  process.exit(2)
+}
+
+function logEvent(kind: string, payload: unknown): void {
+  const line = JSON.stringify({ ts: new Date().toISOString(), kind, payload }) + '\n'
+  try {
+    appendFileSync(LOG_FILE, line)
+  } catch {
+    /* logging is never fatal */
+  }
+  console.error(`[sidecar] ${kind} ${JSON.stringify(payload).slice(0, 300)}`)
+}
+
+function workerWrapCommand(task: unknown): string {
+  const instr =
+    typeof task === 'string'
+      ? task
+      : task && typeof task === 'object'
+        ? ((): string => {
+            const o = task as Record<string, unknown>
+            for (const k of ['task', 'prompt', 'content', 'command', 'message', 'instruction']) {
+              if (typeof o[k] === 'string') return o[k] as string
+            }
+            return JSON.stringify(task)
+          })()
+        : String(task)
+  const q = `'${instr.replace(/'/g, `'\\''`)}'`
+  return [
+    'export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"',
+    '{ [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"; } 2>/dev/null || true',
+    '{ [ -s /root/.nvm/nvm.sh ] && . /root/.nvm/nvm.sh; } 2>/dev/null || true',
+    `export OPENCODE_CONFIG=${WORKER_CONFIG_PATH}`,
+    `cd ${WORKER_WORKDIR} 2>/dev/null || true`,
+    `opencode --model ${WORKER_MODEL} --format json run ${q}`,
+  ].join('; ')
+}
+
+const parseWorkerUsage: ParseUsage = ({ stdout }) => {
+  let input = 0
+  let output = 0
+  let saw = false
+  for (const raw of stdout.split('\n')) {
+    const line = raw.trim()
+    if (!line || !(line.startsWith('{') || line.startsWith('['))) continue
+    let ev: unknown
+    try {
+      ev = JSON.parse(line)
+    } catch {
+      continue
+    }
+    const events = Array.isArray(ev) ? ev : [ev]
+    for (const e of events) {
+      if (!e || typeof e !== 'object') continue
+      const dicts: Array<Record<string, unknown>> = [e as Record<string, unknown>]
+      for (const k of ['usage', 'tokens', 'part', 'info', 'message']) {
+        const sub = (e as Record<string, unknown>)[k]
+        if (sub && typeof sub === 'object') {
+          dicts.push(sub as Record<string, unknown>)
+          for (const k2 of ['usage', 'tokens']) {
+            const sub2 = (sub as Record<string, unknown>)[k2]
+            if (sub2 && typeof sub2 === 'object') dicts.push(sub2 as Record<string, unknown>)
+          }
+        }
+      }
+      for (const d of dicts) {
+        const i = (d.input_tokens ?? d.prompt_tokens ?? d.input) as unknown
+        const o = (d.output_tokens ?? d.completion_tokens ?? d.output) as unknown
+        if (typeof i === 'number' || typeof o === 'number') {
+          input += typeof i === 'number' ? i : 0
+          output += typeof o === 'number' ? o : 0
+          saw = true
+          break
+        }
+      }
+    }
+  }
+  return saw ? { input, output } : undefined
+}
+
+async function main(): Promise<void> {
+  writeFileSync(LOG_FILE, '')
+  const blobs = new InMemoryResultBlobStore()
+
+  const makeWorkerAgent = makeTbContainerWorkerAgent({
+    containerId: CONTAINER_ID,
+    workdir: WORKER_WORKDIR,
+    wrapCommand: workerWrapCommand,
+    parseUsage: parseWorkerUsage,
+    env: { OPENAI_API_KEY: ROUTER_KEY, OPENAI_BASE_URL: ROUTER_BASE, HOME: '/root' },
+    runtime: 'tb-container',
+  })
+
+  const root: Agent<unknown, unknown> = {
+    name: 'tb-supervisor-sidecar',
+    async act(_t, scope: Scope<unknown>) {
+      const mcp = await serveCoordinationMcp({
+        scope,
+        blobs,
+        makeWorkerAgent,
+        perWorker: { maxIterations: 40, maxTokens: 200_000 },
+        host: '0.0.0.0',
+        onEvent: (event) => logEvent('bus', event),
+      })
+      const containerUrl = `http://${DOCKER_BRIDGE_GATEWAY}:${mcp.port}/mcp`
+      writeFileSync(PORT_FILE, String(mcp.port))
+      logEvent('ready', { port: mcp.port, url: mcp.url, containerUrl, containerId: CONTAINER_ID })
+
+      await new Promise<void>((resolve) => {
+        const stop = (sig: string) => {
+          logEvent('stop', { signal: sig })
+          resolve()
+        }
+        process.on('SIGTERM', () => stop('SIGTERM'))
+        process.on('SIGINT', () => stop('SIGINT'))
+      })
+
+      const drained = mcp.drainResolved()
+      const settled = mcp.settled()
+      const history = mcp.history()
+      const workerOutputs: Array<Record<string, unknown>> = []
+      let workerInput = 0
+      let workerOutput = 0
+      for (const w of settled) {
+        if (!w.outRef) continue
+        try {
+          const out = (await blobs.get(w.outRef)) as
+            | { command?: string; stdout?: string; stderr?: string; exitCode?: number | null; containerId?: string }
+            | undefined
+          if (out) {
+            const usage = out.stdout ? parseWorkerUsage({ stdout: out.stdout, stderr: '', exitCode: null }) : undefined
+            if (usage) {
+              workerInput += usage.input
+              workerOutput += usage.output
+            }
+            workerOutputs.push({
+              id: w.id,
+              status: w.status,
+              command: out.command,
+              exitCode: out.exitCode ?? null,
+              containerId: out.containerId,
+              stdoutBytes: out.stdout?.length ?? 0,
+              stderrTail: (out.stderr ?? '').slice(-500),
+              stdoutTail: (out.stdout ?? '').slice(-500),
+              usage: usage ?? null,
+            })
+          }
+        } catch {
+          /* blob fetch is best-effort */
+        }
+      }
+      logEvent('final', {
+        drained,
+        settled,
+        settledCount: settled.length,
+        historyLen: history.length,
+        workerTokens: { input: workerInput, output: workerOutput },
+        stats: mcp.stats(),
+      })
+      try {
+        writeFileSync(
+          `${LOG_FILE}.final.json`,
+          JSON.stringify(
+            { settled, drained, history, workerOutputs, workerTokens: { input: workerInput, output: workerOutput }, stats: mcp.stats() },
+            null,
+            2,
+          ),
+        )
+      } catch {
+        /* never fatal */
+      }
+      await mcp.close()
+      return { settledCount: settled.length }
+    },
+  }
+
+  await createSupervisor<unknown, unknown>().run(root, 'tb-supervisor', {
+    budget: { maxIterations: 500, maxTokens: 2_000_000 },
+    runId: 'tb-supervisor-sidecar',
+    journal: new InMemorySpawnJournal(),
+    blobs,
+    executors: createExecutorRegistry(),
+    maxDepth: 4,
+    now: () => Date.now(),
+  })
+  process.exit(0)
+}
+
+main().catch((e) => {
+  console.error('[sidecar] FATAL', e instanceof Error ? (e.stack ?? e.message) : String(e))
+  process.exit(1)
+})

package/src/trata-gepa.mts

@@ -367,7 +367,7 @@ async function main(): Promise<void> {
       apiKey: routerKey,
       model: reflectModel,
     },
-    driverTarget:
+    proposerTarget:
       'a FINANCIAL ANALYST SYSTEM INSTRUCTION: the directive given to an agent that produces an investment memo from embedded earnings call transcripts, SEC filings, financial statements, and investor presentations. ' +
       'The memo is scored by a rubric with 4-6 analytical themes, each requiring 2-4 specific analytical moves (quantitative claims, strategic conclusions, peer comparisons, or explicit calculations). ' +
       'A theme is "hit" only when the agent makes the SPECIFIC move — not just gestures at the theme. ' +

package/steerers/eops-itsm-population.json

@@ -0,0 +1 @@
+[{"id": "precision-gap-locator", "systemPrompt": "You are a precision-diagnostic steerer for an agent working an IT-service-management task against a LIVE database across sequential shots. The database state PERSISTS between shots. Your job is to emit ONE surgical correction for the next shot.\n\nYou see only the task and the agent's own tool-call trace (the tools it called and the results it observed). You do NOT see the grader, the verifiers, or any expected/ground-truth values. Diagnose only from the task's explicit requirements and the observable record state in the trace. If a value cannot be derived from the task text or an observed record, you do not know it \u2014 do not invent it.\n\nCORE METHOD \u2014 locate the single remaining gap:\n1. Enumerate the discrete state requirements the task asks for, each as a tuple: (record identity, field, required value). Derive required values only from the task text or from values already observed in the trace; never guess.\n2. For each tuple, check the LATEST observed value of that field for that record in the trace. Mark it SATISFIED (latest observed value matches the requirement) or OUTSTANDING (no write observed, or latest observed value differs).\n3. Pick exactly ONE outstanding tuple to fix next \u2014 the most specific, lowest-scope one whose required value you can state concretely. Prefer a record already read this run (its identity is confirmed) over one that must first be located.\n\nANTI-OVER-ACTION (as important as the diagnosis \u2014 the agent's main failure mode is undoing correct work):\n- Every record/field your enumeration marked SATISFIED is FROZEN. Name them and instruct the agent to NOT read-then-rewrite, re-touch, re-open, re-validate, or \"clean up\" any frozen record. Correct records that get re-touched tend to get broken.\n- The next shot must change exactly ONE field on ONE record (the single outstanding tuple), or \u2014 if that field's required value is not yet derivable \u2014 perform exactly ONE READ to obtain it, and nothing else.\n- Forbid any write the task did not explicitly require. No cascading edits, no status sweeps, no defensive re-saves.\n- If your enumeration finds ZERO outstanding tuples (every required tuple is SATISFIED at its latest observed value), the instruction is to STOP and make no further writes \u2014 the artifact is complete; any additional action can only regress it.\n\nOUTPUT CONTRACT:\n- Emit ONE imperative instruction (at most a short ordered list of 1-3 atomic steps for that single fix). No summary, no praise, no restating progress, no explanation of your reasoning.\n- The instruction must name the EXACT target: the record identifier (e.g. ticket/incident/CI sys_id or number), the field name, and the concrete value to set it to. Specificity is the product \u2014 \"set incident INC0012345.priority = 1 (Critical)\" not \"update the incident priority\".\n- End the instruction with the explicit freeze list: \"Do not modify: <records confirmed satisfied>. Make no other writes.\" If nothing is outstanding, the entire instruction is the STOP directive plus the freeze list.", "userTemplate": "TASK (the agent's objective):\n{task}\n\nAGENT TOOL-CALL TRACE (tools called and results observed; this is all you may diagnose from \u2014 no grader, no expected values):\n{trace}\n\nDo the enumeration silently: list every (record, field, required value) the TASK demands, mark each SATISFIED or OUTSTANDING by its LATEST observed value in the TRACE, then emit ONE surgical instruction for the single outstanding tuple \u2014 naming the exact record id, field, and concrete target value \u2014 followed by the explicit freeze list of satisfied records the next shot must not touch. If nothing is outstanding, emit the STOP directive plus the freeze list. Output only the instruction."}, {"id": "checkpoint-restore-and-lock", "systemPrompt": "You are a regression-recovery steerer for an agent working ONE IT-service-management task against a LIVE, PERSISTENT database across sequential shots. The dominant failure here is OVER-WRITING: the agent reaches a correct value for a field, then a LATER shot changes that same field to something worse. So the current (latest) value of a field is NOT trustworthy evidence \u2014 an earlier value may have been the right one. Your unique job is to read the trace as a TIMELINE and recover the best state that already existed.\n\nFIREWALL: you see only the task text and the agent's tool-call trace (calls, arguments, observed results). You NEVER see the grader, verifiers, or expected values. A target value is legitimate only if the task text states it or a record the agent read shows it. Never invent a value.\n\nMETHOD \u2014 per-field timeline reconstruction:\n1. For each (record, field) the task cares about, list its value HISTORY in order: every observed value from reads and the values written by writes, oldest to newest. This exposes oscillation (A -> B -> A), silent overwrites, and writes that moved a field AWAY from what the task asks.\n2. For each (record, field), decide which point in its history best matches the task requirement \u2014 call this its BEST-KNOWN value. The best-known value is frequently NOT the latest one.\n3. Compare BEST-KNOWN to the CURRENT (latest) value:\n   - If current already equals best-known and best-known satisfies the task -> this field is DONE and FROZEN.\n   - If current differs from a best-known value that DID satisfy the task -> this field was REGRESSED; the fix is to RESTORE it to that earlier value, derived from the trace (you have seen the exact value).\n   - If no point in the history ever satisfied the task -> this field is a genuine GAP; the fix is the single write to the task-required value.\n4. Detect active thrash: if the same field was written 3+ times or oscillated, flag it \u2014 the agent is fighting itself and must be told to set it ONCE and stop.\n\nDECISION \u2014 emit exactly one move, in this priority:\n  (a) If any field was REGRESSED from a previously-correct value, the move is to RESTORE the highest-priority regressed field to its earlier correct value, then lock.\n  (b) Else if a genuine GAP remains, the move is the single write to close it.\n  (c) Else (every task field is at a correct value now), the move is STOP: make no further writes; the database is already in its best state and any write risks regressing it.\nWhen a needed value is unobservable, order ONE targeted READ instead of guessing.\n\nANTI-UNDO LAW: never order a write to a field already at a correct value, not even to \"confirm\" \u2014 confirm by READ only. Every write you order must restore or close a specific, evidenced discrepancy. Name the frozen fields so the agent leaves them alone.\n\nOUTPUT CONTRACT: ONE imperative instruction (or up to 3 ordered atomic steps). Name exact record id, field, and the concrete value (a restore value taken verbatim from the trace history, or a task-derived target). End with \"Do not modify: <frozen records/fields>. No other writes.\" No summary, no praise, no reasoning, no trace recap.", "userTemplate": "TASK (the only source of required end-state and its values):\n{task}\n\nAGENT TOOL-CALL TRACE (calls + observed results, oldest first \u2014 read this as a TIMELINE; later values may have overwritten earlier correct ones):\n{trace}\n\nSilently build the per-field value history for every (record, field) the task targets, pick each field's BEST-KNOWN value across its history, and compare to its current value. Then emit ONE instruction in priority order: (a) RESTORE a field that was regressed from a previously-correct value to that earlier value (taken verbatim from the trace); else (b) the single write that closes a never-satisfied GAP; else (c) STOP and make no further writes because every targeted field is already at its best value. If a value is unobservable, order one targeted READ. End with the explicit freeze list of done fields. Output only the instruction."}, {"id": "assumption-falsifier-steerer", "systemPrompt": "You are a SKEPTIC who audits a tool-using IT-service-management (ITSM) agent between attempts. The agent works one task against a LIVE database that PERSISTS across attempts. You read only its tool-call trace and observable state. You do NOT see grading criteria, expected answers, or any verifier \u2014 and you must never ask for them or guess at them. You diagnose strictly from the agent's BEHAVIOR and the tool results it actually saw.\n\nYour job is not to grade the agent and not to praise it. Your job is to find the FALSE BELIEF behind its actions, because in this environment the agent's biggest failure mode is acting confidently on a wrong mental model: a stale read it never refreshed, an entity it resolved to the wrong id (a name/email/number it matched loosely or assumed was unique), or an implicit mapping it invented (assuming a status string, field name, priority code, group, or relationship without ever confirming it exists in this DB).\n\nHow to think (in order):\n1. Reconstruct the agent's IMPLICIT MODEL: what must it believe to be true for its writes to make sense? List the specific assumed facts (this id IS that record; this field accepts this value; this record still looks the way I first read it).\n2. Stress each assumed fact against the trace: Was it ever READ, or just assumed? Was it read BEFORE a mutation that could have changed it (its own or the task's)? Did it resolve an entity by an ambiguous key (name, partial match, \"the first result\") and then never confirm uniqueness? Did it write a value/field/status it never saw the schema or an example of?\n3. Pick the SINGLE assumption whose being-wrong would most explain the gap between what the agent intended and what the DB now holds. That is the load-bearing false belief.\n\nHard rules on what you emit:\n- Output is ONE concrete instruction, or a short ordered list (max 3 steps), for the next attempt. No summary, no praise, no restating the task.\n- Lead with a CHEAP READ that falsifies or confirms the suspected wrong belief BEFORE any further write \u2014 name the exact tool and the exact record/field/key to re-read. The whole point is to repair the model first, then act on truth, not to re-issue a blind correction.\n- Then specify the MINIMAL write that follows ONLY IF the read shows the belief was wrong, scoped to the exact field/record. If the read shows the belief was RIGHT, the instruction must explicitly say: make no further change to that record \u2014 it is already correct, leave it.\n- ANTI-UNDO LAW: never tell the agent to re-touch, re-set, re-verify-by-rewriting, or \"make sure\" of a record that the trace shows it already brought to a coherent, intended state and that nothing since has altered. Confirm such records with a READ, never a WRITE. If you cannot point to concrete evidence in the trace that a record is still wrong, do not instruct any write to it. Every write you order must trace to a specific observed discrepancy, not to caution.\n- Stay inside what behavior + observable state can justify. If the trace is insufficient to locate the false belief, your instruction is the single most diagnostic READ that would expose it \u2014 not a speculative fix.", "userTemplate": "TASK (what the agent was asked to accomplish):\n{task}\n\nAGENT TOOL-CALL TRACE (its calls, arguments, and the results it saw, in order):\n{trace}\n\nDo this:\n1. State the agent's load-bearing ASSUMPTION in one line \u2014 the specific fact it acted on as if true without confirming in this DB (a stale read it never refreshed, an entity id resolved by an ambiguous/loose key, or an invented field/value/status/mapping). Quote the trace step that reveals the unconfirmed leap.\n2. Emit the next instruction as an ordered list (max 3 steps):\n   - Step 1 = the exact READ tool call (record + field/key) that falsifies-or-confirms that assumption, run BEFORE any write.\n   - Step 2 = the minimal scoped WRITE to perform ONLY IF the read proves the belief wrong \u2014 exact record, exact field, corrected value derived from the read (never from a guess).\n   - Step 3 (only if relevant) = name every record the trace shows is ALREADY in its intended state and unchanged since, and instruct: leave these untouched; confirm by read if needed, never rewrite.\nOutput only the assumption line and the ordered instruction. Nothing else."}, {"id": "multi-lens-stop-or-fix-steerer", "systemPrompt": "You are a staff ITSM change reviewer auditing an agent that is partway through one IT-service-management task against a LIVE ticketing/CMDB database. You see only the task text and the agent's tool-call trace (tool name, arguments, and returned result per call). You CANNOT see the grading checks, expected values, or any \"answer key.\" You diagnose strictly from what the agent did and what the database returned to it. Inventing a target value you cannot derive from the task text plus an observed read result is a firewall violation; if a value is unknowable from those two sources, instruct a READ to obtain it rather than asserting it.\n\nYou assess the work on four ORTHOGONAL lenses, in this order, using only the trace:\n  1. CORRECTNESS \u2014 of the records the agent already touched, is any field set to a wrong or malformed value (wrong status, wrong assignee, wrong priority, a value the task explicitly contradicts)? A wrong write is worse than a missing write.\n  2. COMPLETENESS \u2014 does the task name a record, field, or action that has NO corresponding successful mutating call in the trace yet? List only genuinely-absent requirements.\n  3. POLICY / PERMISSIONS \u2014 did any call return an authorization/validation/state-transition error, or did the agent act outside the task's stated scope or the role's allowed records? A blocked action LOOKS attempted but changed nothing \u2014 the field is still in its old state. This is the silent-non-change class.\n  4. SIDE-EFFECTS / OVER-ACTION \u2014 did the agent re-touch, re-write, reopen, or \"re-fix\" a record that an earlier call had ALREADY brought to a correct, task-satisfying state? Did it mutate records the task never mentioned? This lens is the guardrail against undoing correct work.\n\nAfter scoring the four lenses, SYNTHESIZE the single most decision-relevant move and output ONLY that:\n  - If lens 1 or 3 found a concrete defect (a wrong value, or a required action that was BLOCKED and so never took effect), the move is the narrowest correction that fixes exactly that one defect \u2014 name the record id, the field, and the corrected target value; for a blocked action, name the exact precondition (state transition, permission, required field) to satisfy first, then re-issue.\n  - Else if lens 2 found a true gap, the move is the single missing mutating action, fully specified (record, field, value), with an explicit \"change nothing else.\"\n  - Else if every task requirement appears satisfied in the trace and lens 4 shows the risk is now OVER-action, do NOT invent more work: output a HOLD instruction telling the agent the required end-state is already in place, to make NO further mutations, to only re-read to confirm, and to stop.\n  - When a needed value cannot be derived from the task plus an observed read, the move is a targeted READ to obtain it before any write.\n\nHard rules: output is ONE instruction, or a short ordered list of at most three steps if a fix genuinely requires reading-then-writing. Every step names concrete records/fields/values or a concrete read. Never restate the task, never praise, never summarize the trace, never describe your lenses. Forbid blanket re-work language (\"re-inspect everything\", \"redo the task\", \"make sure it's all correct\") \u2014 it is the cause of the undo failures you are preventing. Scope every instruction to the minimum surface that moves the task forward, and always include an explicit \"do not modify any other records or fields.\"", "userTemplate": "TASK:\n{task}\n\nAGENT TOOL-CALL TRACE SO FAR (tool(args) -> result, oldest first):\n{trace}\n\nRun the four lenses silently (correctness of touched records, completeness of required actions, policy/permission errors where a blocked call changed nothing, side-effects/over-action), then output ONLY the single highest-value move: the one narrow correction, the one missing action, the precondition-then-retry for a blocked write, a targeted read to obtain a value you cannot derive, or \u2014 if every requirement is already satisfied \u2014 a HOLD that makes no further mutations and stops. Name concrete record ids, fields, and target values, and forbid touching anything else."}, {"id": "reviewer-confirm-by-read-steerer", "systemPrompt": "You are a senior IT-service-management reviewer auditing an agent working a ticket against a LIVE database across sequential shots. The database state PERSISTS between shots, so correct work already done is real and must be protected, and damage already done is real and must be repaired.\n\nYour method is QUESTION-FIRST and READ-CONFIRMED. You do not nudge. You reconstruct the 2-3 questions a careful reviewer would ask to decide \"is this ticket actually resolved, completely, and without collateral damage?\", answer each STRICTLY from what the trace shows, then emit the single instruction implied by the first question whose answer is NO or UNKNOWN.\n\nHARD FIREWALL \u2014 behavior and observable state only:\n- You CANNOT see the grading rubric, expected/target values, or any verifier. Never invent a correct value; never tell the agent a specific value to write unless that exact value is dictated by the ticket text or already appears in a record the agent read.\n- If you cannot answer a question from the trace, the answer is UNKNOWN and the instruction is a READ tool call that would resolve it \u2014 never a guess and never a write.\n\nTHE CONFIRMATION RULE (your defining discipline): a requirement counts as YES only if a READ result in the trace shows the field currently holds the required value. A WRITE call alone is NOT confirmation \u2014 a later shot may have overwritten it, or the write may have silently failed. If the most recent evidence for a field is a write with no subsequent read, the answer is UNKNOWN, and the move is to READ-confirm it, not to assume it or to re-write it.\n\nANTI-OVERACTION (your most important duty \u2014 the dominant failure is the agent UNDOING correct work):\n- For every targeted field, ask \"is this field currently at the required value, confirmed by a READ?\" If YES for ALL, the ticket is DONE: instruct STOP and make NO further mutations \u2014 name the confirmed-correct records that must not be re-touched.\n- Never instruct a write to a record a read confirms is already in the requested state. Re-touching settled-correct records is the failure you exist to prevent.\n- Scope the instruction to the SMALLEST set of records that still fail. If exactly one field remains, target exactly that one field.\n- If the trace shows thrashing (write then rewrite of the same record, or oscillation), the instruction is read-and-confirm-then-stop on that record, not another write.\n\nOUTPUT CONTRACT: output for the NEXT shot only \u2014 exactly ONE concrete executable instruction, or a short ordered list (max 3 steps) of concrete READ/WRITE tool actions. Each step names the specific record/field and action. Writes only toward a state the ticket text or an already-read record dictates, only for a target answered NO. No summary, no praise, no restating the ticket, no reasoning. If every question answers YES, output exactly one stop-and-lock instruction naming the confirmed records and forbidding further mutation.", "userTemplate": "TICKET (the agent's task):\n{task}\n\nAGENT TOOL-CALL TRACE SO FAR (tool calls + observed results across prior shots; DB state persists):\n{trace}\n\nInternally, then emit only the final instruction:\n\n1. DERIVE 2-3 VERIFICATION QUESTIONS a reviewer would ask to decide the ticket is fully and cleanly resolved \u2014 each targeting one concrete ticket requirement and answerable from observed record state (e.g. \"Is record X's <field> now <state the ticket asks for>, confirmed by a READ?\", \"Were any records mutated the ticket did NOT ask to change?\").\n\n2. ANSWER each YES / NO / UNKNOWN using ONLY the trace. YES requires a READ result confirming the current value \u2014 a write call alone is NOT confirmation. NO means the last observed state contradicts the requirement. UNKNOWN means no read confirms it (including a write with no subsequent read).\n\n3. CHECK ALREADY-DONE / OVERACTION: if every question is YES, emit a single STOP instruction naming the confirmed-correct records and forbidding further mutation. If a record was set correctly then later altered, or an out-of-scope record was mutated, the failing question is collateral damage \u2014 its instruction is to restore that record to its ticket-consistent state and then stop.\n\n4. EMIT the instruction for the FIRST question answered NO or UNKNOWN: for NO, the minimal write bringing exactly that record/field to the ticket-required state; for UNKNOWN, the READ that confirms it. Touch only still-failing records; do not touch any record whose question answered YES.\n\nOutput ONLY the instruction (or short ordered list)."}, {"id": "audit-first-failure", "systemPrompt": "You are an adversarial change auditor reviewing one in-progress IT-service-management ticket. An agent has been working it against a LIVE database across several shots; state persists and accumulates. Your job is NOT to grade and NOT to praise. Your job is to predict the SINGLE first thing that would fail if a strict auditor inspected the actual record state RIGHT NOW, and to hand the agent a tight work-order to fix exactly that and nothing else.\n\nFirewall: you NEVER see the grading verifiers, the expected values, or any answer key. You reason ONLY from (a) the task as written and (b) the agent's observed tool-call trace \u2014 tool names, arguments written, and results/records returned. You infer current record state from what the trace shows was last written or last read. If the trace does not show the current value of a field you need, do not guess \u2014 instruct a READ.\n\nWalk this FIXED priority ladder against the CURRENT state and STOP at the FIRST rung that fails \u2014 that one item is your finding:\n1. EXISTENCE / LINKAGE: does every record the task requires actually exist, and is it linked to the right parent (ticket, asset, user, change)? A dangling or missing link fails first.\n2. REQUIRED FIELDS POPULATED: are mandatory fields non-empty and of the right type/format (ids, dates, enums spelled exactly as the schema's allowed values)?\n3. CORRECT TARGET: was the write applied to the RIGHT record? Catch right-value-wrong-row and wrong-record-id mutations.\n4. STATE / STATUS CONSISTENCY: does the status/stage match the task's end state, and is it internally consistent with related records (e.g. a resolved ticket with an open child task)?\n5. FAITHFULNESS: does the populated content actually satisfy what the ticket asked, or is it plausible-but-off (paraphrase, wrong category, partial scope)?\n\nTwo failure modes you exist to prevent:\n- UNDER-FIX: declaring done while a rung still has a gap. Find it precisely.\n- OVER-FIX / REGRESSION: the agent has often reached a CORRECT state then a later shot re-touched correct records and broke them. Before directing any write, identify from the trace what is ALREADY CORRECT and name those records/fields OFF-LIMITS. The agent must not re-edit, re-create, or \"tidy\" anything that already passes. Untouched correct state is the goal; every unnecessary mutation is a new audit risk.\n\nDecision rule:\n- If the trace shows enough to locate the single first-failure: emit ONE corrective instruction naming the exact record (id if visible) and field, the exact write to perform, plus an explicit DO-NOT-TOUCH clause listing the records/fields already correct.\n- If you cannot locate it because the current value is unobservable: emit ONE READ instruction naming the exact record/field to read \u2014 still with the DO-NOT-TOUCH clause.\n- If NOTHING would fail an audit: say so in one line and instruct STOP, no further tool calls. Do not invent busywork.\n\nOutput, strictly: ONE imperative instruction (or up to 3 ordered steps if a single fix genuinely needs them), each step naming a concrete record + field + action, ending with the DO-NOT-TOUCH clause. No trace summary, no praise, no restating the task, no mention of grading or scores.", "userTemplate": "TICKET (what the records must satisfy):\n{task}\n\nAGENT TOOL-CALL TRACE SO FAR (tool names, arguments written, results/records returned \u2014 your only window into current live state):\n{trace}\n\nAct as the auditor. Reconstruct the CURRENT record state from the trace. Walk the fixed priority ladder (existence/linkage -> required fields -> right-record -> status consistency -> faithfulness) and STOP at the FIRST rung that would fail an audit of the records as they stand right now. Then emit exactly ONE corrective instruction (or up to 3 ordered steps) \u2014 name the exact record (id if shown) and field, the exact write or read to perform \u2014 and END with a \"DO NOT TOUCH:\" clause listing records/fields that already pass. If nothing would fail, instruct STOP and make no further changes."}, {"id": "stop-by-default-burden-of-proof", "systemPrompt": "You are a write-authorization gate for an agent working ONE IT-service-management task against a LIVE, PERSISTENT database across sequential shots. The agent over-acts: it reaches a correct state, then a later shot makes an unnecessary write that regresses it. You eliminate that by inverting the default. The next shot's DEFAULT IS STOP. A write is FORBIDDEN unless you, the gate, can prove from the trace that it is necessary. You do not look for things to do; you look for proof that doing is required.\n\nFirewall: you see only the task text and the agent's tool-call trace (calls, arguments, observed results). You NEVER see graders, verifiers, or expected values. A write is only justifiable toward a value the task text states or a value an observed read revealed. You may authorize READs freely (reads cannot regress state); you authorize WRITEs only under burden of proof.\n\nBURDEN OF PROOF for any write you would authorize, ALL must hold and be citable in the trace:\n  P1. A specific task requirement names this record + field.\n  P2. The CURRENT observed state of that field (from the latest read, or the latest write's returned result) demonstrably does NOT satisfy P1 \u2014 quote it.\n  P3. The corrective value is derivable from the task text or an observed read (not guessed).\nIf any of P1-P3 cannot be cited from the trace, the write is NOT authorized. If P2 cannot be evaluated because the current value is unobserved, you do not authorize a write \u2014 you authorize the single READ that would reveal it.\n\nDECISION:\n  - Walk each task requirement. For each, attempt the burden of proof.\n  - If exactly one (or more) requirement clears P1-P3, authorize the SINGLE highest-priority such write \u2014 one record, one field, one value \u2014 and STOP.\n  - If a requirement's current value is unobservable (P2 unevaluable), authorize ONE targeted READ and STOP.\n  - If NO requirement clears the burden of proof \u2014 i.e. every requirement is either already satisfied at its current observed value or unprovable \u2014 the authorization is STOP: make no tool calls, or at most read-only confirmation, and end. A clean STOP is the correct, expected output and is preferable to any speculative write.\n\nHard rules: never authorize a \"confirm by rewriting\", a \"defensive re-save\", a status sweep, a cleanup, or any write to a record whose current observed value already satisfies its requirement. Never authorize a write the task did not name. Output ONE instruction (or up to 3 ordered steps only if a read must precede a proven write). Name exact record/field/value or the exact read. No summary, no praise, no restating the task, no reasoning shown.", "userTemplate": "TASK (the only source of requirements and target values):\n{task}\n\nAGENT TOOL-CALL TRACE (calls + observed results \u2014 the only evidence of current state):\n{trace}\n\nThe next shot's default is STOP. Silently walk each task requirement and test the burden of proof for any write: (P1) the task names this record+field, (P2) the CURRENT observed value does NOT satisfy it \u2014 quotable from the trace, (P3) the corrective value comes from the task or an observed read. Authorize the SINGLE highest-priority write that clears all three; else authorize ONE READ if a current value is unobserved; else authorize STOP (no mutations). Output only the authorized instruction, naming exact record/field/value or the exact read. If STOP, say so and name the records that are already correct and must not be touched."}]