@linxin666/dsh-client-ui-task-board 0.4.5 → 0.4.6

package/src/host/verification-runner.ts

@@ -10,9 +10,18 @@
 import { createUserMessage, type FinishReason, type LlmRuntime } from '@deepseek-ai/dsh-llm'
 import { openOneShotStream } from './llm-dispatch.ts'
 import { createHash } from 'node:crypto'
+import { DEFAULT_VERIFICATION_BUDGET_MS } from '../core/verification-budget.ts'
 import {
   CODING_CRITERIA,
   EMPTY_WORK_BASELINE,
+  workSlotOfRound,
+  VERIFICATION_DETAIL_MAX_FINDINGS,
+  VERIFICATION_DETAIL_MAX_PROBLEM_CHARS,
+  VERIFICATION_DETAIL_MAX_QUOTE_CHARS,
+  assessAcceptanceValidity,
+  type AcceptanceEvidenceText,
+  type VerificationCriterionFinding,
+  type VerificationInvalidReason,
   VERIFICATION_ROUNDS,
   VERIFICATION_THRESHOLD,
   acceptancePassed,
@@ -38,11 +47,21 @@ export const VERIFICATION_MAX_TRACE_CHARS = 80_000
 export const VERIFICATION_MAX_ENTRY_CHARS = 4_000
 /** Largest per-finding text kept from one judge answer. */
 export const VERIFICATION_MAX_FINDING_CHARS = 400
-/** Findings kept per judge answer and per acceptance. */
+/** Largest verbatim citation kept from one finding. */
+export const VERIFICATION_MAX_QUOTE_CHARS = VERIFICATION_DETAIL_MAX_QUOTE_CHARS
+/** Largest requirement/observation/gap statement kept from one finding. */
+export const VERIFICATION_MAX_PROBLEM_CHARS = VERIFICATION_DETAIL_MAX_PROBLEM_CHARS
+/** Structured findings kept per judge answer and per acceptance. */
 export const VERIFICATION_MAX_FINDINGS_PER_CALL = 3
-export const VERIFICATION_MAX_FINDINGS = 6
-/** One judge request's ceiling and its retry budget. */
-export const VERIFICATION_CALL_TIMEOUT_MS = 120_000
+export const VERIFICATION_MAX_FINDINGS = VERIFICATION_DETAIL_MAX_FINDINGS
+/**
+ * Default ceiling of ONE judge request. The effective ceiling is a row setting
+ * (`goalVerificationCallTimeoutSeconds`), read live by the gate, because 120s was
+ * tight for a reasoning model reading an 80,000 character trace with a 16,384
+ * token output cap: every timeout it produced was charged as an acceptance
+ * anomaly against a budget that then failed the card.
+ */
+export const VERIFICATION_CALL_TIMEOUT_MS = 150_000
 export const VERIFICATION_MAX_TOKENS = 16_384
 export const VERIFICATION_TEMPERATURE = 0.2
 export const VERIFICATION_REQUEST_ATTEMPTS = 3
@@ -54,7 +73,25 @@ export const VERIFICATION_REDACT_PATTERNS: readonly string[] = [
 ]
 
 /** Why one acceptance could not produce a verdict. */
-export type VerificationErrorKind = 'route-unavailable' | 'timeout' | 'auth' | 'parse' | 'request' | 'aborted'
+export type VerificationErrorKind = 'route-unavailable' | 'timeout' | 'auth' | 'parse' | 'request' | 'aborted' | 'budget'
+
+/**
+ * The time one acceptance attempt may still spend.
+ *
+ * `deadline` is the instant the whole attempt runs out; `callTimeoutMs` is the
+ * ceiling of a single judge call. The pair is what ties the retry budget to the
+ * acceptance budget: a call is not started unless its own ceiling still fits
+ * inside what is left, because starting one the outer budget will abort anyway
+ * only burns wall time and turns a slow machine into an anomaly.
+ */
+export interface AcceptanceBudget {
+  /** Absolute instant (ms epoch) the acceptance attempt runs out at. */
+  deadline: number
+  /** Ceiling of one judge call in milliseconds. */
+  callTimeoutMs: number
+  /** Clock the caller runs on, injected so tests drive it deterministically. */
+  now: () => number
+}
 
 /** An acceptance anomaly: never a quality verdict, always recorded as one. */
 export class VerificationError extends Error {
@@ -239,16 +276,54 @@ interface JudgeAnswer {
   usage: VerificationUsage
 }
 
-/** One finding the judge located, before it is rendered. */
+/**
+ * One finding the judge reported, before its citation is resolved against the
+ * evidence the judge actually saw.
+ */
 interface RawFinding {
-  criterion?: string
-  evidence: string
+  /** The criterion id this finding is attached to (always the caller's id). */
+  criterionId: string
+  /** The criterion name as the prompt offered it. */
+  criterionName: string
+  /** Which evidence block the citation claims to come from. */
+  location: VerificationCriterionFinding['location']
+  quote: string
+  requirement: string
+  observation: string
+  gap: string
   body: string
   action?: string
 }
 
-/** Parse the optional `<finding ...>` lines out of one judge answer. */
-function parseFindings(text: string, criterion: VerificationCriterion): RawFinding[] {
+/** How one cited slot maps onto the caller's own evidence under a swapped round. */
+function locationOfSlot(slot: 'A' | 'B', round: number): VerificationCriterionFinding['location'] {
+  // The odd round swaps the two slots: citing the slot the work occupies is a
+  // citation of the work's own trajectory, and citing the other one is a
+  // citation of the empty-work baseline, which can never veto the work.
+  return slot === workSlotOfRound(round) ? 'trajectory' : 'baseline'
+}
+
+/** Resolve one `location` (or legacy `evidence`) attribute into an evidence location. */
+function readLocation(raw: string, round: number): VerificationCriterionFinding['location'] | undefined {
+  const value = raw.trim().toUpperCase()
+  if (value === 'TASK') return 'task'
+  if (value === 'CONTEXT' || value === 'WORKSPACE') return 'workspace'
+  if (value === 'TRAJECTORY_A' || value === 'A') return locationOfSlot('A', round)
+  if (value === 'TRAJECTORY_B' || value === 'B') return locationOfSlot('B', round)
+  return undefined
+}
+
+/**
+ * Parse the `<finding ...>` elements out of one judge answer.
+ *
+ * A finding must name the criterion this prompt offered and cite an evidence
+ * location the prompt actually rendered; a finding for another criterion, a
+ * citation of a location that was never offered, and an empty body are dropped
+ * here rather than echoed back as if the judge had located something. The
+ * quote is carried verbatim: whether it really occurs in the evidence it names
+ * is decided by the caller, which is the only side holding the evidence text.
+ */
+function parseFindings(text: string, criterion: VerificationCriterion, round: number): RawFinding[] {
   const pattern = /<finding\s+([^>]*)>([\s\S]*?)<\/finding>/giu
   const attributes = /([A-Za-z_][A-Za-z0-9_]*)\s*=\s*"([^"]*)"/gu
   const found: RawFinding[] = []
@@ -256,45 +331,77 @@ function parseFindings(text: string, criterion: VerificationCriterion): RawFindi
   for (let match = pattern.exec(text); match !== null && found.length < VERIFICATION_MAX_FINDINGS_PER_CALL; match = pattern.exec(text)) {
     const attrs = new Map<string, string>()
     for (let attribute = attributes.exec(match[1] ?? ''); attribute !== null; attribute = attributes.exec(match[1] ?? '')) {
-      if (['criterion', 'evidence', 'action'].includes(attribute[1] ?? '')) attrs.set(attribute[1] ?? '', attribute[2] ?? '')
+      if (['criterion', 'evidence', 'location', 'requirement', 'observation', 'gap', 'quote', 'action'].includes(attribute[1] ?? '')) {
+        attrs.set(attribute[1] ?? '', attribute[2] ?? '')
+      }
     }
     const named = attrs.get('criterion') ?? ''
-    // A finding must name the criterion this prompt offered and cite an evidence
-    // token the prompt actually rendered; anything else is dropped, never echoed.
+    // A finding must name the criterion this prompt offered; anything else is
+    // dropped, never echoed.
     if (named !== '' && named.toLowerCase() !== criterion.name.toLowerCase() && named.toLowerCase() !== criterion.id.toLowerCase()) continue
-    const evidence = (attrs.get('evidence') ?? '').trim().toUpperCase()
-    if (!['TASK', 'A', 'B'].includes(evidence)) continue
+    const location = readLocation(attrs.get('location') ?? attrs.get('evidence') ?? '', round)
+    if (location === undefined) continue
+    const quote = (attrs.get('quote') ?? '').trim().slice(0, VERIFICATION_MAX_QUOTE_CHARS)
+    const requirement = (attrs.get('requirement') ?? '').trim().slice(0, VERIFICATION_MAX_PROBLEM_CHARS)
+    const observation = (attrs.get('observation') ?? '').trim().slice(0, VERIFICATION_MAX_PROBLEM_CHARS)
+    const gap = (attrs.get('gap') ?? '').trim().slice(0, VERIFICATION_MAX_PROBLEM_CHARS)
     const body = (match[2] ?? '').trim().slice(0, VERIFICATION_MAX_FINDING_CHARS)
-    if (body === '') continue
+    if (body === '' && quote === '') continue
     const action = (attrs.get('action') ?? '').trim().slice(0, 300)
-    const key = named + '|' + evidence + '|' + body
+    const key = criterion.id + '|' + location + '|' + quote + '|' + body
     if (seen.has(key)) continue
     seen.add(key)
-    found.push({ criterion: criterion.name, evidence, body, ...(action === '' ? {} : { action }) })
+    found.push({
+      criterionId: criterion.id,
+      criterionName: criterion.name,
+      location,
+      quote,
+      requirement,
+      observation,
+      gap,
+      body,
+      ...(action === '' ? {} : { action }),
+    })
   }
   return found
 }
 
 /** Render one finding as the single feedback line the agent reads. */
 function renderFinding(finding: RawFinding): string {
-  const locator = '[' + (finding.criterion ?? 'finding') + ' evidence=' + finding.evidence + ']'
+  const locator = '[' + finding.criterionName + ' ' + finding.location + ']'
   const action = finding.action === undefined ? '' : ' → 建议核实：' + finding.action
-  return locator + ' ' + finding.body + action
+  const detail = finding.body !== '' ? finding.body : finding.quote
+  return locator + ' ' + detail + action
 }
 
-/** One judge request's outcome. */
+/**
+ * One judge request's outcome.
+ *
+ * Every attempt is admitted against the acceptance budget first: a call whose
+ * own ceiling no longer fits in the time left is refused before it is opened,
+ * so a slow route cannot spend the whole acceptance budget on calls the outer
+ * abort is going to kill anyway.
+ */
 async function judgeOnce(
   llm: LlmRuntime,
   route: VerificationRoute,
   prompt: string,
   signal: AbortSignal,
+  budget: AcceptanceBudget,
 ): Promise<{ text: string; usage: VerificationUsage }> {
   const usage = emptyUsage()
   let lastError: unknown
+  /** The outer abort is the acceptance budget running out, not a cancellation. */
+  const stopped = (): VerificationError => budget.now() >= budget.deadline
+    ? new VerificationError('budget', 'the acceptance budget ran out before the judge answered', usage)
+    : new VerificationError('aborted', 'the acceptance was cancelled', usage)
   for (let attempt = 0; attempt < VERIFICATION_REQUEST_ATTEMPTS; attempt++) {
-    if (signal.aborted) throw new VerificationError('aborted', 'the acceptance was cancelled before the judge answer', usage)
+    if (signal.aborted) throw stopped()
+    if (budget.deadline - budget.now() <= budget.callTimeoutMs) {
+      throw new VerificationError('budget', 'the acceptance budget cannot fit another judge call', usage)
+    }
     const timeout = new AbortController()
-    const timer = setTimeout(() => { timeout.abort(new Error('task-board verification: the judge request timed out')) }, VERIFICATION_CALL_TIMEOUT_MS)
+    const timer = setTimeout(() => { timeout.abort(new Error('task-board verification: the judge request timed out')) }, budget.callTimeoutMs)
     const forward = (): void => { timeout.abort(signal.reason) }
     signal.addEventListener('abort', forward, { once: true })
     try {
@@ -333,7 +440,7 @@ async function judgeOnce(
           finishReason = chunk.type === 'finish' ? chunk.reason : undefined
         }
       }
-      if (signal.aborted) throw new VerificationError('aborted', 'the acceptance was cancelled', usage)
+      if (signal.aborted) throw stopped()
       if (timeout.signal.aborted) throw new VerificationError('timeout', 'the judge request timed out', usage)
       if (finishReason?.kind === 'error' || finishReason?.kind === 'aborted') {
         const failure = finishReason.failure
@@ -358,9 +465,9 @@ async function judgeOnce(
     } catch (error) {
       lastError = error
       if (error instanceof VerificationError && (error.kind === 'aborted' || error.kind === 'auth')) throw error
-      if (signal.aborted) throw new VerificationError('aborted', 'the acceptance was cancelled', usage)
+      if (signal.aborted) throw stopped()
       if (timeout.signal.aborted) {
-        lastError = new VerificationError('timeout', 'the judge request timed out after ' + Math.round(VERIFICATION_CALL_TIMEOUT_MS / 1000) + 's', usage)
+        lastError = new VerificationError('timeout', 'the judge request timed out after ' + Math.round(budget.callTimeoutMs / 1000) + 's', usage)
       }
       const parseFailure = lastError instanceof VerificationError && lastError.kind === 'parse'
       if (attempt + 1 >= VERIFICATION_REQUEST_ATTEMPTS || (!parseFailure && !retryable(lastError))) break
@@ -387,8 +494,8 @@ export interface AcceptanceResult {
  * Run one acceptance: every criterion is judged twice, the odd round with the
  * A/B slots swapped, and the per-round scores are mapped back and averaged per
  * criterion.
- * @param input - the judge route, evidence, threshold and cancellation.
- * @returns the recorded attempt (quality verdict or anomaly).
+ * @param input - the judge route, evidence, threshold, time budget and cancellation.
+ * @returns the recorded attempt (quality verdict, anomaly, or budget stop).
  */
 export async function runAcceptance(input: {
   llm: LlmRuntime
@@ -397,6 +504,10 @@ export async function runAcceptance(input: {
   threshold: number
   index: number
   signal: AbortSignal
+  /** Instant (ms epoch) the attempt runs out at; defaults to now plus the default budget. */
+  deadline?: number
+  /** Ceiling of one judge call; defaults to {@link VERIFICATION_CALL_TIMEOUT_MS}. */
+  callTimeoutMs?: number
   /** Host-observed workspace changes, rendered as the prompt's reference context. */
   context?: string
   /** How many changed files that context shows. */
@@ -405,6 +516,11 @@ export async function runAcceptance(input: {
 }): Promise<AcceptanceResult> {
   const now = input.now ?? Date.now
   const startedAt = now()
+  const budget: AcceptanceBudget = {
+    deadline: input.deadline ?? startedAt + DEFAULT_VERIFICATION_BUDGET_MS,
+    callTimeoutMs: input.callTimeoutMs ?? VERIFICATION_CALL_TIMEOUT_MS,
+    now,
+  }
   const usage = emptyUsage()
   const evidenceSummary: VerificationEvidenceSummary = {
     chars: input.evidence.trace.length,
@@ -417,6 +533,15 @@ export async function runAcceptance(input: {
   }
   const findings: string[] = []
   const seenFindings = new Set<string>()
+  const structured: VerificationCriterionFinding[] = []
+  const seenStructured = new Set<string>()
+  // The exact evidence texts this acceptance handed to the judge, retained so
+  // the validity check can verify every citation against what was really shown.
+  const evidenceText: AcceptanceEvidenceText = {
+    task: input.evidence.problem,
+    trajectory: input.evidence.trace,
+    workspace: input.context ?? '',
+  }
   try {
     const perCriterion = new Map<string, { criterion: VerificationCriterion; work: number[]; baseline: number[] }>()
     for (const criterion of CODING_CRITERIA) perCriterion.set(criterion.id, { criterion, work: [], baseline: [] })
@@ -425,8 +550,8 @@ export async function runAcceptance(input: {
         const swapped = repeat % 2 === 1
         const candidateA = swapped ? EMPTY_WORK_BASELINE : input.evidence.trace
         const candidateB = swapped ? input.evidence.trace : EMPTY_WORK_BASELINE
-        const prompt = buildAcceptancePrompt(input.evidence.problem, candidateA, candidateB, criterion, undefined, input.context)
-        const answer = await judgeOnce(input.llm, input.route, prompt, input.signal)
+        const prompt = buildAcceptancePrompt(input.evidence.problem, candidateA, candidateB, criterion, undefined, input.context, workSlotOfRound(repeat))
+        const answer = await judgeOnce(input.llm, input.route, prompt, input.signal, budget)
         const scoreA = extractScore(answer.text, 'score_A')
         const scoreB = extractScore(answer.text, 'score_B')
         if (answer.usage.usageIncomplete === true) usage.usageIncomplete = true
@@ -437,13 +562,23 @@ export async function runAcceptance(input: {
         const bucket = perCriterion.get(criterion.id)!
         bucket.work.push(swapped ? scoreB : scoreA)
         bucket.baseline.push(swapped ? scoreA : scoreB)
-        for (const finding of parseFindings(answer.text, criterion)) {
-          // The prompt rendered the swapped slots; the finding must cite the
-          // caller's slots, or it would point the agent at the wrong object.
-          const located: RawFinding = swapped && (finding.evidence === 'A' || finding.evidence === 'B')
-            ? { ...finding, evidence: finding.evidence === 'A' ? 'B' : 'A' }
-            : finding
-          const line = renderFinding(located)
+        for (const finding of parseFindings(answer.text, criterion, repeat)) {
+          if (structured.length < VERIFICATION_MAX_FINDINGS) {
+            const key = finding.criterionId + '|' + finding.location + '|' + finding.quote + '|' + finding.body
+            if (!seenStructured.has(key)) {
+              seenStructured.add(key)
+              structured.push({
+                criterionId: finding.criterionId,
+                requirement: finding.requirement !== '' ? finding.requirement : finding.criterionName,
+                observation: finding.observation !== '' ? finding.observation : finding.body,
+                gap: finding.gap !== '' ? finding.gap : finding.body,
+                quote: finding.quote,
+                location: finding.location,
+                ...(finding.action === undefined ? {} : { action: finding.action }),
+              })
+            }
+          }
+          const line = renderFinding(finding)
           if (seenFindings.has(line)) continue
           seenFindings.add(line)
           if (findings.length < VERIFICATION_MAX_FINDINGS) findings.push(line)
@@ -459,12 +594,23 @@ export async function runAcceptance(input: {
     const score = average(criteria.map(criterion => criterion.score))
     const baseline = average(criteria.map(criterion => criterion.baseline))
     const passed = acceptancePassed(score, baseline, criteria, input.threshold)
+    // A veto is only bookable when its evidence holds up; a rejecting attempt
+    // that cannot substantiate the rejection is recorded as INVALID so it can
+    // never consume the quality budget or fail the card.
+    const validity = passed
+      ? { valid: true as const }
+      : assessAcceptanceValidity({
+        criteria,
+        findings: structured,
+        evidence: evidenceText,
+        omittedCharacters: input.evidence.omittedCharacters,
+      })
     return {
       attempt: {
         index: input.index,
         at: startedAt,
-        stage: 'quality',
-        passed,
+        stage: validity.valid ? 'quality' : 'invalid',
+        passed: validity.valid ? passed : false,
         score,
         baseline,
         criteria,
@@ -474,6 +620,11 @@ export async function runAcceptance(input: {
         route: input.route,
         channel: 'explicit-tag',
         rounds: VERIFICATION_ROUNDS,
+        ...(structured.length === 0 ? {} : { criterionFindings: structured }),
+        ...(validity.valid ? {} : {
+          invalidReason: (validity as { reason: VerificationInvalidReason }).reason,
+          error: classifyInvalid(validity as { reason: VerificationInvalidReason; detail: string }),
+        }),
       },
     }
   } catch (error) {
@@ -484,7 +635,10 @@ export async function runAcceptance(input: {
       attempt: {
         index: input.index,
         at: startedAt,
-        stage: 'exception',
+        // A budget stop is its own stage: the time ran out before any verdict,
+        // which is neither a judgement of the work nor an anomaly of the
+        // judge route, so it is charged to no budget.
+        stage: kind === 'budget' ? 'budget' : 'exception',
         passed: false,
         score: 0,
         baseline: 0,
@@ -518,6 +672,18 @@ function mergeUsage(target: VerificationUsage, source: VerificationUsage): Verif
   }
 }
 
+/** Prefix one invalid-acceptance message with its class so the report stays actionable. */
+function classifyInvalid(validity: { reason: VerificationInvalidReason; detail: string }): string {
+  const labels: Record<VerificationInvalidReason, string> = {
+    'missing-finding': '验收无效（未达标判据没有对应的结构化问题反馈）',
+    'unlocatable-quote': '验收无效（问题反馈引用不存在于本次裁判所见的证据中）',
+    'baseline-finding': '验收无效（问题反馈指向空工作基线而非本次工作）',
+    'vacuous-finding': '验收无效（问题反馈空泛，未说明实际观测与差异）',
+    'insufficient-evidence': '验收无效（裁判所见证据不足，无法支撑否决）',
+  }
+  return labels[validity.reason] + ': ' + validity.detail
+}
+
 /** Prefix one anomaly message with its class so the report stays actionable. */
 function classifyMessage(kind: VerificationErrorKind, message: string): string {
   const labels: Record<VerificationErrorKind, string> = {
@@ -527,6 +693,7 @@ function classifyMessage(kind: VerificationErrorKind, message: string): string {
     parse: '验收异常（裁判回答无法解析）',
     request: '验收异常（裁判请求失败）',
     aborted: '验收异常（验收被取消）',
+    budget: '验收未完成（时间预算耗尽，未产生任何判定，不计入异常额度）',
   }
   return labels[kind] + ': ' + message
 }