dsh-llm-verifier 0.1.13 → 0.1.14

package/src/auto.test.ts

@@ -1,86 +1,86 @@
-import { describe, expect, it } from 'vitest'
-import { Session } from '@deepseek-ai/dsh-session'
-import { createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
-import { AutoVerificationBudget, analyzeAutoTask, automaticFeedback } from './auto.ts'
-
-function taskSession() {
-  const session = Session.create('session-00000000-0000-4000-8000-000000000009' as never)
-  session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Implement and test the feature' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
-  return session
-}
-
-function call(session: ReturnType<typeof taskSession>, name: string, id: string) {
-  session.append('tool/call', { turn: 1, step: 1, callId: id as never, name, arguments: '{}' })
-  session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: id as never, content: [{ type: 'text', text: 'ok' }], isError: false }) }, { surfaceOp: 'append' })
-}
-
-const smart = { mode: 'smart' as const, minToolCalls: 3, maxPerTask: 2, maxPerSession: 8 }
-
-describe('automatic verification policy', () => {
-  it('requires consequential work and enough evidence in smart mode', () => {
-    const session = taskSession()
-    call(session, 'read', 'one')
-    call(session, 'grep', 'two')
-    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'no-consequential-work' })
-    call(session, 'edit', 'three')
-    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: true, toolCalls: 3, consequentialToolCalls: 1 })
-  })
-
-  it('does not count a failed consequential call paired with an unrelated success', () => {
-    const session = taskSession()
-    session.append('tool/call', { turn: 1, step: 1, callId: 'bad' as never, name: 'edit', arguments: '{}' })
-    session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'bad' as never, content: [{ type: 'text', text: 'failed' }], isError: true }), error: { name: 'Error', code: 'FAILED' } }, { surfaceOp: 'append' })
-    call(session, 'read', 'good')
-    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ consequentialToolCalls: 0, eligible: false, reason: 'no-consequential-work' })
-  })
-
-  it('strict mode accepts one completed consequential call', () => {
-    const session = taskSession()
-    call(session, 'ssh_exec', 'one')
-    expect(analyzeAutoTask(session.events, { ...smart, mode: 'strict' })).toMatchObject({ eligible: true, reason: 'strict-eligible' })
-  })
-
-  it('manual verification suppresses the automatic gate', () => {
-    const session = taskSession()
-    call(session, 'edit', 'one')
-    call(session, 'pwsh', 'two')
-    call(session, 'verifier_current_session', 'three')
-    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'already-verified', hasManualSessionVerification: true })
-  })
-
-  it('resets per-task attempts for a new direct user message and enforces the session cap', () => {
-    const session = taskSession()
-    call(session, 'edit', 'one')
-    call(session, 'read', 'two')
-    call(session, 'pwsh', 'three')
-    const agent = { id: session.id, session } as never
-    const budget = new AutoVerificationBudget()
-    const policy = { ...smart, maxPerTask: 1, maxPerSession: 2 }
-    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(true)
-    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(false)
-    session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Do another task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
-    call(session, 'write', 'four')
-    call(session, 'read', 'five')
-    call(session, 'pwsh', 'six')
-    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(true)
-    session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Third task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
-    call(session, 'edit', 'seven')
-    call(session, 'read', 'eight')
-    call(session, 'pwsh', 'nine')
-    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(false)
-  })
-
-  it('builds actionable low-score feedback', () => {
-    expect(automaticFeedback(0.42, 0.31, 'A', 0.65)).toContain('42.0%')
-    expect(automaticFeedback(0.42, 0.31, 'A', 0.65)).toContain('verification command')
-  })
-
-  it('recognizes PTC mode tool/code-dispatch events for eligibility', () => {
-    const session = taskSession()
-    session.append('tool/code-dispatch', { subCallId: 'c1' as never, name: 'read', arguments: '{}', isError: false, content: [{ type: 'text', text: 'data' }] })
-    session.append('tool/code-dispatch', { subCallId: 'c2' as never, name: 'grep', arguments: '{}', isError: false, content: [{ type: 'text', text: 'match' }] })
-    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'no-consequential-work' })
-    session.append('tool/code-dispatch', { subCallId: 'c3' as never, name: 'edit', arguments: '{}', isError: false, content: [{ type: 'text', text: 'done' }] })
-    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: true, toolCalls: 3, consequentialToolCalls: 1 })
-  })
-})
+import { describe, expect, it } from 'vitest'
+import { Session } from '@deepseek-ai/dsh-session'
+import { createToolResultMessage, createUserMessage } from '@deepseek-ai/dsh-llm'
+import { AutoVerificationBudget, analyzeAutoTask, automaticFeedback } from './auto.ts'
+
+function taskSession() {
+  const session = Session.create('session-00000000-0000-4000-8000-000000000009' as never)
+  session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Implement and test the feature' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
+  return session
+}
+
+function call(session: ReturnType<typeof taskSession>, name: string, id: string) {
+  session.append('tool/call', { turn: 1, step: 1, callId: id as never, name, arguments: '{}' })
+  session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: id as never, content: [{ type: 'text', text: 'ok' }], isError: false }) }, { surfaceOp: 'append' })
+}
+
+const smart = { mode: 'smart' as const, minToolCalls: 3, maxPerTask: 2, maxPerSession: 8 }
+
+describe('automatic verification policy', () => {
+  it('requires consequential work and enough evidence in smart mode', () => {
+    const session = taskSession()
+    call(session, 'read', 'one')
+    call(session, 'grep', 'two')
+    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'no-consequential-work' })
+    call(session, 'edit', 'three')
+    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: true, toolCalls: 3, consequentialToolCalls: 1 })
+  })
+
+  it('does not count a failed consequential call paired with an unrelated success', () => {
+    const session = taskSession()
+    session.append('tool/call', { turn: 1, step: 1, callId: 'bad' as never, name: 'edit', arguments: '{}' })
+    session.append('tool/result', { turn: 1, step: 1, message: createToolResultMessage({ callId: 'bad' as never, content: [{ type: 'text', text: 'failed' }], isError: true }), error: { name: 'Error', code: 'FAILED' } }, { surfaceOp: 'append' })
+    call(session, 'read', 'good')
+    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ consequentialToolCalls: 0, eligible: false, reason: 'no-consequential-work' })
+  })
+
+  it('strict mode accepts one completed consequential call', () => {
+    const session = taskSession()
+    call(session, 'ssh_exec', 'one')
+    expect(analyzeAutoTask(session.events, { ...smart, mode: 'strict' })).toMatchObject({ eligible: true, reason: 'strict-eligible' })
+  })
+
+  it('manual verification suppresses the automatic gate', () => {
+    const session = taskSession()
+    call(session, 'edit', 'one')
+    call(session, 'pwsh', 'two')
+    call(session, 'verifier_current_session', 'three')
+    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'already-verified', hasManualSessionVerification: true })
+  })
+
+  it('resets per-task attempts for a new direct user message and enforces the session cap', () => {
+    const session = taskSession()
+    call(session, 'edit', 'one')
+    call(session, 'read', 'two')
+    call(session, 'pwsh', 'three')
+    const agent = { id: session.id, session } as never
+    const budget = new AutoVerificationBudget()
+    const policy = { ...smart, maxPerTask: 1, maxPerSession: 2 }
+    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(true)
+    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(false)
+    session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Do another task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
+    call(session, 'write', 'four')
+    call(session, 'read', 'five')
+    call(session, 'pwsh', 'six')
+    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(true)
+    session.append('user/message', createUserMessage({ content: [{ type: 'text', text: 'Third task' }], source: { kind: 'user' } }), { surfaceOp: 'append' })
+    call(session, 'edit', 'seven')
+    call(session, 'read', 'eight')
+    call(session, 'pwsh', 'nine')
+    expect(budget.claim(agent, analyzeAutoTask(session.events, policy), policy)).toBe(false)
+  })
+
+  it('builds actionable low-score feedback', () => {
+    expect(automaticFeedback(0.42, 0.31, 'A', 0.65)).toContain('42.0%')
+    expect(automaticFeedback(0.42, 0.31, 'A', 0.65)).toContain('verification command')
+  })
+
+  it('recognizes PTC mode tool/code-dispatch events for eligibility', () => {
+    const session = taskSession()
+    session.append('tool/code-dispatch', { subCallId: 'c1' as never, name: 'read', arguments: '{}', isError: false, content: [{ type: 'text', text: 'data' }] })
+    session.append('tool/code-dispatch', { subCallId: 'c2' as never, name: 'grep', arguments: '{}', isError: false, content: [{ type: 'text', text: 'match' }] })
+    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: false, reason: 'no-consequential-work' })
+    session.append('tool/code-dispatch', { subCallId: 'c3' as never, name: 'edit', arguments: '{}', isError: false, content: [{ type: 'text', text: 'done' }] })
+    expect(analyzeAutoTask(session.events, smart)).toMatchObject({ eligible: true, toolCalls: 3, consequentialToolCalls: 1 })
+  })
+})

package/src/auto.ts

@@ -1,133 +1,133 @@
-import type { Agent } from '@deepseek-ai/dsh-agent'
-import type { SessionEvent } from '@deepseek-ai/dsh-session'
-
-export type AutoVerifyMode = 'manual' | 'smart' | 'strict'
-
-export interface AutoVerifyPolicy {
-  mode: AutoVerifyMode
-  minToolCalls: number
-  maxPerTask: number
-  maxPerSession: number
-}
-
-export interface AutoTaskEvidence {
-  taskStartSeq: number
-  toolCalls: number
-  completedToolResults: number
-  consequentialToolCalls: number
-  hasManualSessionVerification: boolean
-  eligible: boolean
-  reason: string
-}
-
-const PASSIVE_TOOLS = new Set([
-  'read', 'read_image', 'glob', 'grep', 'web_search', 'ssh_list', 'job_list',
-  'job_output', 'list_agents', 'get_goal', 'skill', 'mcp__codegraph__codegraph_explore',
-])
-const VERIFIER_TOOLS = new Set([
-  'verifier_compare', 'verifier_select', 'verifier_track', 'verifier_current_session',
-])
-const CONSEQUENTIAL_TOOLS = new Set([
-  'edit', 'write', 'pwsh', 'bash', 'run_code', 'codex_image_generate',
-  'ssh_exec', 'ssh_upload', 'ssh_download', 'ssh_tunnel', 'ssh_cluster',
-  'job_kill', 'workbench_session_delete', 'create_goal', 'update_goal',
-])
-
-function latestDirectUserSeq(events: readonly SessionEvent[]): number | undefined {
-  for (let index = events.length - 1; index >= 0; index -= 1) {
-    const event = events[index]
-    if (event?.type === 'user/message' && event.data.source.kind === 'user') return event.seq
-  }
-  return undefined
-}
-
-function isConsequential(name: string): boolean {
-  if (CONSEQUENTIAL_TOOLS.has(name)) return true
-  if (PASSIVE_TOOLS.has(name) || VERIFIER_TOOLS.has(name)) return false
-  return /(?:edit|write|patch|apply|deploy|upload|delete|remove|kill|exec|shell|command|migration|database|tunnel|cluster)/iu.test(name)
-}
-
-interface CodeDispatchData {
-  subCallId?: string
-  name: string
-  arguments?: unknown
-  isError?: boolean
-  content?: readonly { isError?: boolean }[]
-}
-
-function isSuccessfulCodeDispatch(data: CodeDispatchData): boolean {
-  if (data.isError === true) return false
-  if (Array.isArray(data.content) && data.content.some(b => b.isError === true)) return false
-  return true
-}
-
-export function analyzeAutoTask(events: readonly SessionEvent[], policy: AutoVerifyPolicy): AutoTaskEvidence {
-  const taskStartSeq = latestDirectUserSeq(events)
-  if (taskStartSeq === undefined) return { taskStartSeq: 0, toolCalls: 0, completedToolResults: 0, consequentialToolCalls: 0, hasManualSessionVerification: false, eligible: false, reason: 'no-direct-user-task' }
-
-  const relevant = events.filter(event => event.seq >= taskStartSeq)
-  const calls = relevant.filter((event): event is SessionEvent<'tool/call'> => event.type === 'tool/call')
-  const successfulResults = new Set(relevant.filter((event): event is SessionEvent<'tool/result'> => event.type === 'tool/result' && event.data.error === undefined && event.data.message.content.every(block => block.isError !== true)).map(event => String(event.data.message.source.callId)))
-  const pairedCalls = calls.filter(event => successfulResults.has(String(event.data.callId)))
-
-  const codeDispatches = relevant.filter(event => event.type === 'tool/code-dispatch').map(event => event.data as CodeDispatchData)
-  const successfulDispatches = codeDispatches.filter(isSuccessfulCodeDispatch)
-
-  const toolCalls = calls.filter(event => !VERIFIER_TOOLS.has(event.data.name)).length + codeDispatches.filter(d => !VERIFIER_TOOLS.has(d.name)).length
-  const completedToolResults = pairedCalls.length + successfulDispatches.length
-  const consequentialToolCalls = pairedCalls.filter(event => isConsequential(event.data.name)).length + successfulDispatches.filter(d => isConsequential(d.name)).length
-  const hasManualSessionVerification = pairedCalls.some(event => event.data.name === 'verifier_current_session') || successfulDispatches.some(d => d.name === 'verifier_current_session')
-
-  if (policy.mode === 'manual') return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'manual-mode' }
-  if (hasManualSessionVerification) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'already-verified' }
-  if (consequentialToolCalls === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'no-consequential-work' }
-  if (completedToolResults === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'no-completed-evidence' }
-  if (policy.mode === 'smart' && toolCalls < policy.minToolCalls) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'insufficient-tool-evidence' }
-  return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: true, reason: policy.mode + '-eligible' }
-}
-
-interface SessionBudget {
-  taskStartSeq: number
-  taskAttempts: number
-  sessionAttempts: number
-  lastEvaluatedSeq: number
-}
-
-export class AutoVerificationBudget {
-  private readonly states = new Map<string, SessionBudget>()
-
-  claim(agent: Agent, evidence: AutoTaskEvidence, policy: AutoVerifyPolicy): boolean {
-    if (!evidence.eligible) return false
-    const id = String(agent.id)
-    const previous = this.states.get(id)
-    const state: SessionBudget = previous ?? { taskStartSeq: evidence.taskStartSeq, taskAttempts: 0, sessionAttempts: 0, lastEvaluatedSeq: -1 }
-    if (state.taskStartSeq !== evidence.taskStartSeq) {
-      state.taskStartSeq = evidence.taskStartSeq
-      state.taskAttempts = 0
-      state.lastEvaluatedSeq = -1
-    }
-    const lastSeq = agent.session.events.at(-1)?.seq ?? -1
-    if (lastSeq <= state.lastEvaluatedSeq || state.taskAttempts >= policy.maxPerTask || state.sessionAttempts >= policy.maxPerSession) {
-      this.states.set(id, state)
-      return false
-    }
-    state.taskAttempts += 1
-    state.sessionAttempts += 1
-    state.lastEvaluatedSeq = lastSeq
-    this.states.set(id, state)
-    return true
-  }
-
-  release(agent: Agent): void {
-    this.states.delete(String(agent.id))
-  }
-}
-
-export function automaticFeedback(score: number, baselineScore: number, winner: 'A' | 'B' | 'tie', threshold: number): string {
-  const percent = (value: number) => (value * 100).toFixed(1) + '%'
-  return [
-    '[Automatic verifier gate]',
-    `The independent verifier did not clear this task for completion: evidence score ${percent(score)}, baseline ${percent(baselineScore)}, verdict ${winner}, required ${percent(threshold)}.`,
-    'Re-open the task requirements, inspect the actual tool outputs for unresolved errors or missing proof, make any necessary corrections, and run a directly relevant verification command before concluding. Do not merely restate that the task is complete.',
-  ].join('\n')
-}
+import type { Agent } from '@deepseek-ai/dsh-agent'
+import type { SessionEvent } from '@deepseek-ai/dsh-session'
+
+export type AutoVerifyMode = 'manual' | 'smart' | 'strict'
+
+export interface AutoVerifyPolicy {
+  mode: AutoVerifyMode
+  minToolCalls: number
+  maxPerTask: number
+  maxPerSession: number
+}
+
+export interface AutoTaskEvidence {
+  taskStartSeq: number
+  toolCalls: number
+  completedToolResults: number
+  consequentialToolCalls: number
+  hasManualSessionVerification: boolean
+  eligible: boolean
+  reason: string
+}
+
+const PASSIVE_TOOLS = new Set([
+  'read', 'read_image', 'glob', 'grep', 'web_search', 'ssh_list', 'job_list',
+  'job_output', 'list_agents', 'get_goal', 'skill', 'mcp__codegraph__codegraph_explore',
+])
+const VERIFIER_TOOLS = new Set([
+  'verifier_compare', 'verifier_select', 'verifier_track', 'verifier_current_session',
+])
+const CONSEQUENTIAL_TOOLS = new Set([
+  'edit', 'write', 'pwsh', 'bash', 'run_code', 'codex_image_generate',
+  'ssh_exec', 'ssh_upload', 'ssh_download', 'ssh_tunnel', 'ssh_cluster',
+  'job_kill', 'workbench_session_delete', 'create_goal', 'update_goal',
+])
+
+function latestDirectUserSeq(events: readonly SessionEvent[]): number | undefined {
+  for (let index = events.length - 1; index >= 0; index -= 1) {
+    const event = events[index]
+    if (event?.type === 'user/message' && event.data.source.kind === 'user') return event.seq
+  }
+  return undefined
+}
+
+function isConsequential(name: string): boolean {
+  if (CONSEQUENTIAL_TOOLS.has(name)) return true
+  if (PASSIVE_TOOLS.has(name) || VERIFIER_TOOLS.has(name)) return false
+  return /(?:edit|write|patch|apply|deploy|upload|delete|remove|kill|exec|shell|command|migration|database|tunnel|cluster)/iu.test(name)
+}
+
+interface CodeDispatchData {
+  subCallId?: string
+  name: string
+  arguments?: unknown
+  isError?: boolean
+  content?: readonly { isError?: boolean }[]
+}
+
+function isSuccessfulCodeDispatch(data: CodeDispatchData): boolean {
+  if (data.isError === true) return false
+  if (Array.isArray(data.content) && data.content.some(b => b.isError === true)) return false
+  return true
+}
+
+export function analyzeAutoTask(events: readonly SessionEvent[], policy: AutoVerifyPolicy): AutoTaskEvidence {
+  const taskStartSeq = latestDirectUserSeq(events)
+  if (taskStartSeq === undefined) return { taskStartSeq: 0, toolCalls: 0, completedToolResults: 0, consequentialToolCalls: 0, hasManualSessionVerification: false, eligible: false, reason: 'no-direct-user-task' }
+
+  const relevant = events.filter(event => event.seq >= taskStartSeq)
+  const calls = relevant.filter((event): event is SessionEvent<'tool/call'> => event.type === 'tool/call')
+  const successfulResults = new Set(relevant.filter((event): event is SessionEvent<'tool/result'> => event.type === 'tool/result' && event.data.error === undefined && event.data.message.content.every(block => block.isError !== true)).map(event => String(event.data.message.source.callId)))
+  const pairedCalls = calls.filter(event => successfulResults.has(String(event.data.callId)))
+
+  const codeDispatches = relevant.filter(event => event.type === 'tool/code-dispatch').map(event => event.data as CodeDispatchData)
+  const successfulDispatches = codeDispatches.filter(isSuccessfulCodeDispatch)
+
+  const toolCalls = calls.filter(event => !VERIFIER_TOOLS.has(event.data.name)).length + codeDispatches.filter(d => !VERIFIER_TOOLS.has(d.name)).length
+  const completedToolResults = pairedCalls.length + successfulDispatches.length
+  const consequentialToolCalls = pairedCalls.filter(event => isConsequential(event.data.name)).length + successfulDispatches.filter(d => isConsequential(d.name)).length
+  const hasManualSessionVerification = pairedCalls.some(event => event.data.name === 'verifier_current_session') || successfulDispatches.some(d => d.name === 'verifier_current_session')
+
+  if (policy.mode === 'manual') return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'manual-mode' }
+  if (hasManualSessionVerification) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'already-verified' }
+  if (consequentialToolCalls === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'no-consequential-work' }
+  if (completedToolResults === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'no-completed-evidence' }
+  if (policy.mode === 'smart' && toolCalls < policy.minToolCalls) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: false, reason: 'insufficient-tool-evidence' }
+  return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, eligible: true, reason: policy.mode + '-eligible' }
+}
+
+interface SessionBudget {
+  taskStartSeq: number
+  taskAttempts: number
+  sessionAttempts: number
+  lastEvaluatedSeq: number
+}
+
+export class AutoVerificationBudget {
+  private readonly states = new Map<string, SessionBudget>()
+
+  claim(agent: Agent, evidence: AutoTaskEvidence, policy: AutoVerifyPolicy): boolean {
+    if (!evidence.eligible) return false
+    const id = String(agent.id)
+    const previous = this.states.get(id)
+    const state: SessionBudget = previous ?? { taskStartSeq: evidence.taskStartSeq, taskAttempts: 0, sessionAttempts: 0, lastEvaluatedSeq: -1 }
+    if (state.taskStartSeq !== evidence.taskStartSeq) {
+      state.taskStartSeq = evidence.taskStartSeq
+      state.taskAttempts = 0
+      state.lastEvaluatedSeq = -1
+    }
+    const lastSeq = agent.session.events.at(-1)?.seq ?? -1
+    if (lastSeq <= state.lastEvaluatedSeq || state.taskAttempts >= policy.maxPerTask || state.sessionAttempts >= policy.maxPerSession) {
+      this.states.set(id, state)
+      return false
+    }
+    state.taskAttempts += 1
+    state.sessionAttempts += 1
+    state.lastEvaluatedSeq = lastSeq
+    this.states.set(id, state)
+    return true
+  }
+
+  release(agent: Agent): void {
+    this.states.delete(String(agent.id))
+  }
+}
+
+export function automaticFeedback(score: number, baselineScore: number, winner: 'A' | 'B' | 'tie', threshold: number): string {
+  const percent = (value: number) => (value * 100).toFixed(1) + '%'
+  return [
+    '[Automatic verifier gate]',
+    `The independent verifier did not clear this task for completion: evidence score ${percent(score)}, baseline ${percent(baselineScore)}, verdict ${winner}, required ${percent(threshold)}.`,
+    'Re-open the task requirements, inspect the actual tool outputs for unresolved errors or missing proof, make any necessary corrections, and run a directly relevant verification command before concluding. Do not merely restate that the task is complete.',
+  ].join('\n')
+}