dsh-llm-verifier 0.4.0 → 0.6.0

package/src/caller.ts

@@ -1,15 +1,39 @@
 import { BlockAssembler, ReasoningEffortId, createUserMessage, type ContentBlock, type FinishReason, type LlmRuntime } from '@deepseek-ai/dsh-llm'
 import * as LlmModule from '@deepseek-ai/dsh-llm'
 
+/**
+ * Deep-freeze a call-options graph, leaving any live AbortSignal mutable.
+ *
+ * The host re-exports its own `deepFreeze` from `@deepseek-ai/dsh-llm` on the 0.1.1 line; the
+ * 0.1.5 line moved the helper to `@deepseek-ai/dsh-util-values` and no longer re-exports it, so
+ * the fallback below is what actually runs there. Both implementations must carry the same rule
+ * ("leaving live AbortSignal objects mutable" — the host's own wording), because freezing a
+ * signal breaks the transport twice over: Node >= 26.5 initializes the signal's internal event
+ * map lazily, so the first `addEventListener` throws "Cannot assign to read only property
+ * 'Symbol(kEvents)'", and on every Node version the attempt timeout's `controller.abort()` throws
+ * the same way on 'Symbol(kAborted)'. The signal belongs to this call's retry loop and is still
+ * unsubscribed when the options are frozen, which is what makes the ordering fatal.
+ */
 function deepFreeze<T>(value: T): T {
   const mod = LlmModule as unknown as Record<string, unknown>
   if (typeof mod.deepFreeze === 'function') {
     return (mod.deepFreeze as Function)(value)
   }
+  return fallbackDeepFreeze(value)
+}
+
+/**
+ * The graph walk used when the host exports no `deepFreeze`; exported for tests.
+ * @param value - value to freeze.
+ * @returns The same value, with every reachable enumerable child frozen except live AbortSignals.
+ */
+export function fallbackDeepFreeze<T>(value: T): T {
+  // See deepFreeze(): a frozen signal fails on the transport's first subscription.
+  if (value instanceof AbortSignal) return value
   if (value && typeof value === 'object' && !Object.isFrozen(value)) {
     Object.freeze(value)
     for (const key of Object.keys(value)) {
-      deepFreeze((value as Record<string, unknown>)[key])
+      fallbackDeepFreeze((value as Record<string, unknown>)[key])
     }
   }
   return value
@@ -50,13 +74,92 @@ export interface UsageStats {
   cachedInputTokens: number
   outputTokens: number
   reasoningTokens: number
+  /**
+   * At least one attempt's usage is UNKNOWN (an earlier attempt failed and was retried, or a
+   * response came back unusable). The token counts are a floor, never a confident total.
+   */
+  usageIncomplete?: boolean
 }
 
 export type ScoringMode = 'top-logprobs' | 'explicit-tag'
-export interface VerifierCompletion extends CompletionLogprobs { usage: UsageStats; scoringMode: ScoringMode }
+export interface VerifierCompletion extends CompletionLogprobs {
+  usage: UsageStats
+  scoringMode: ScoringMode
+  /**
+   * The direct logprob transport was attempted and the provider rejected it, so this
+   * answer came from the explicit-tag fallback. Diagnostics only: it never changes the
+   * score, and it is recorded so a silent downgrade is visible in the statistics.
+   */
+  channelFallback?: boolean
+}
+
+/**
+ * Where a finally-failed request records how many attempts it already spent.
+ *
+ * A failure that never returned usage leaves the tokens UNKNOWN, not zero. The attempt
+ * count is the one fact the transport does know, so it is carried on the error itself
+ * (the engine catches per-judge errors) instead of being discarded.
+ */
+const REQUEST_ATTEMPTS = Symbol('llm-verifier.requestAttempts')
+
+/** Attempts one failed verifier request already spent; 0 when the error carries none. */
+export function requestAttempts(error: unknown): number {
+  if (typeof error !== 'object' || error === null) return 0
+  const value = (error as { [REQUEST_ATTEMPTS]?: unknown })[REQUEST_ATTEMPTS]
+  return typeof value === 'number' && Number.isSafeInteger(value) && value > 0 ? value : 0
+}
+
+/**
+ * Usage the transport DID return for a response that then turned out unusable.
+ *
+ * A parse failure or a truncated judge answer happened after the request was billed, so the
+ * engine must not fold it into a zero-cost failure. The usage rides on the error next to the
+ * attempt count; one error can carry both (a retried call whose final answer also failed).
+ */
+const PARTIAL_USAGE = Symbol('llm-verifier.partialUsage')
+
+/** Usage a completed-but-unusable response already cost; undefined when the error carries none. */
+export function partialUsage<T = UsageStats>(error: unknown): T | undefined {
+  if (typeof error !== 'object' || error === null) return undefined
+  const value = (error as { [PARTIAL_USAGE]?: unknown })[PARTIAL_USAGE]
+  return typeof value === 'object' && value !== null ? value as T : undefined
+}
+
+/** Attach the usage a completed-but-unusable response already cost to its error. */
+export function attachUsage(error: unknown, usage: UsageStats): void {
+  if (typeof error === 'object' && error !== null) (error as { [PARTIAL_USAGE]?: UsageStats })[PARTIAL_USAGE] = usage
+}
+
+/**
+ * An error for a response that came back and was billed but produced no usable verdict.
+ *
+ * The usage is attached so the failure row reports known requests and tokens instead of zero.
+ */
+function unusable(reason: string, attempt: number, raw: { inputTokens?: number; outputTokens?: number; cacheReadTokens?: number; cacheWriteTokens?: number; reasoningTokens?: number } | undefined): Error {
+  const error = new Error('llm-verifier: ' + reason)
+  attachUsage(error, usage(attempt, raw ?? {}))
+  return error
+}
 
+/**
+ * Describe why a finished stream is unusable.
+ *
+ * `FinishReason.failure` is the host's SERIALIZABLE facts record (`{message, code, status?,
+ * providerRetryAfterMs?, requestId?}`), not the error the adapter threw: `normalizeLlmFailure`
+ * detaches the facts at the adapter boundary, so the stack and the cause are already gone before
+ * this plugin sees them. The machine code and the provider status ARE carried, and dropping them
+ * left the board with one line and no way to tell an auth failure from a transport one.
+ */
 function failureMessage(finish: FinishReason): string | undefined {
-  if (finish.kind === 'error' || finish.kind === 'aborted') return finish.failure.message
+  if (finish.kind === 'error' || finish.kind === 'aborted') {
+    const failure = finish.failure
+    const facts = [
+      failure.code,
+      failure.status === undefined ? undefined : 'HTTP ' + failure.status,
+      failure.requestId === undefined ? undefined : 'request ' + failure.requestId,
+    ].filter((fact): fact is string => typeof fact === 'string' && fact.length > 0)
+    return facts.length > 0 ? failure.message + ' [' + facts.join(', ') + ']' : failure.message
+  }
   if (finish.kind === 'max-tokens') return 'verifier response reached max tokens before completing its answer'
   return undefined
 }
@@ -88,10 +191,59 @@ const RETRYABLE_MESSAGE = /rate|quota|timeout|timed out|temporar|network|fetch|s
  * @param run - one attempt; receives its own deadline signal and 1-based attempt number.
  * @returns The first successful completion.
  */
+function attachAttempts(error: unknown, attempt: number): void {
+  if (typeof error === 'object' && error !== null) (error as { [REQUEST_ATTEMPTS]?: number })[REQUEST_ATTEMPTS] = attempt
+}
+
+/**
+ * A fresh error for one aborted call.
+ *
+ * Never annotate and rethrow the shared `signal.reason`: several concurrent requests share it,
+ * and the engine attaches its accumulator to the thrown error, so one object carries several
+ * accumulators and they get merged repeatedly. The wrapper preserves the message and name.
+ */
+function abortFailure(reason: unknown, attempt: number): Error {
+  const source = reason instanceof Error ? reason : new Error(reason === undefined ? 'llm-verifier: request aborted' : String(reason))
+  const error = new Error(source.message)
+  error.name = source.name
+  attachAttempts(error, attempt)
+  return error
+}
+
+/** Token-only addition: call/attempt counters already describe the logical request. */
+function addTokens(target: Pick<UsageStats, 'inputTokens' | 'cachedInputTokens' | 'outputTokens' | 'reasoningTokens'>, source: Pick<UsageStats, 'inputTokens' | 'cachedInputTokens' | 'outputTokens' | 'reasoningTokens'>): void {
+  target.inputTokens += source.inputTokens
+  target.cachedInputTokens += source.cachedInputTokens
+  target.outputTokens += source.outputTokens
+  target.reasoningTokens += source.reasoningTokens
+}
+
 async function retrying<T>(config: VerifierClientConfig, signal: AbortSignal | undefined, run: (signal: AbortSignal, attempt: number) => Promise<T>): Promise<T> {
   let attempt = 0
+  // Tokens an earlier attempt already paid for but never delivered a usable result. They are
+  // real cost, so they follow the chain into whichever exit happens (success, final failure or
+  // cancellation) instead of being discarded when no attempt ultimately succeeds.
+  const carried = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningTokens: 0 }
+  let unknownFailure = false
+  // The usage the chain must be able to report: everything already spent plus this attempt's own
+  // billed response. `unknown` (some attempt's tokens never came back) is what makes it a floor.
+  const spent = (billed: UsageStats | undefined, unknown: boolean): UsageStats => ({
+    calls: billed === undefined ? 0 : billed.calls,
+    attempts: attempt,
+    retries: Math.max(0, attempt - 1),
+    inputTokens: carried.inputTokens + (billed?.inputTokens ?? 0),
+    cachedInputTokens: carried.cachedInputTokens + (billed?.cachedInputTokens ?? 0),
+    outputTokens: carried.outputTokens + (billed?.outputTokens ?? 0),
+    reasoningTokens: carried.reasoningTokens + (billed?.reasoningTokens ?? 0),
+    ...(unknown ? { usageIncomplete: true } : {}),
+  })
+  const cancel = (reason: unknown, billed: UsageStats | undefined, unknown: boolean): never => {
+    const error = abortFailure(reason, attempt)
+    attachUsage(error, spent(billed, unknown))
+    throw error
+  }
   while (true) {
-    if (signal?.aborted) throw signal.reason
+    if (signal?.aborted) cancel(signal.reason, undefined, false)
     attempt += 1
     const controller = new AbortController()
     let timedOut = false
@@ -99,14 +251,39 @@ async function retrying<T>(config: VerifierClientConfig, signal: AbortSignal | u
     const abort = () => controller.abort(signal?.reason)
     signal?.addEventListener('abort', abort, { once: true })
     try {
-      return await run(controller.signal, attempt)
+      const value = await run(controller.signal, attempt)
+      if (typeof value === 'object' && value !== null) {
+        const usage = (value as { usage?: UsageStats }).usage
+        if (usage !== undefined) {
+          addTokens(usage, carried)
+          // Only a failed attempt whose tokens never came back makes the total a floor.
+          if (unknownFailure) usage.usageIncomplete = true
+        }
+      }
+      return value
     } catch (error) {
-      if (signal?.aborted) throw signal.reason
+      const billed = partialUsage<UsageStats>(error)
+      if (signal?.aborted) cancel(signal.reason ?? error, billed, unknownFailure || billed === undefined)
       // A deadline abort is retryable even when the adapter wraps the reason in
       // its own error type with an unrelated message.
       const retryable = timedOut || (error instanceof Error && RETRYABLE_MESSAGE.test(error.message))
-      if (attempt > config.maxRetries || !retryable) throw error
-      await delay(Math.min(30000, config.retryBaseDelayMs * 2 ** (attempt - 1) * (0.8 + Math.random() * 0.4)), signal)
+      if (attempt > config.maxRetries || !retryable) {
+        // Every attempt of the chain goes onto the thrown error: the last response alone would
+        // report a fraction of the requests and tokens the invocation really spent.
+        if (billed === undefined) unknownFailure = true
+        const failure = typeof error === 'object' && error !== null ? error : new Error(String(error))
+        attachAttempts(failure, attempt)
+        attachUsage(failure, spent(billed, unknownFailure || billed === undefined))
+        throw failure
+      }
+      if (billed === undefined) unknownFailure = true
+      else addTokens(carried, billed)
+      try {
+        await delay(Math.min(30000, config.retryBaseDelayMs * 2 ** (attempt - 1) * (0.8 + Math.random() * 0.4)), signal)
+      } catch (waitError) {
+        // Cancelled while waiting to retry: the attempts and the tokens already spent are real.
+        cancel(signal?.reason ?? waitError, undefined, unknownFailure)
+      }
     } finally {
       clearTimeout(timeout)
       signal?.removeEventListener('abort', abort)
@@ -165,14 +342,14 @@ async function callTextCompletion(config: VerifierClientConfig, prompt: string,
   for await (const chunk of config.llm.stream(options)) assembler.push(chunk)
   let truncated = false
   if (assembler.finish.kind === 'max-tokens') {
-    if (!tolerateTruncation) throw new Error('llm-verifier: model call failed: ' + failureMessage(assembler.finish))
+    if (!tolerateTruncation) throw unusable('model call failed: ' + failureMessage(assembler.finish), attempt, assembler.usage)
     truncated = true
   } else {
     const failed = failureMessage(assembler.finish)
-    if (failed !== undefined) throw new Error('llm-verifier: model call failed: ' + failed)
+    if (failed !== undefined) throw unusable('model call failed: ' + failed, attempt, assembler.usage)
   }
   const text = assembler.blocks().filter((block): block is Extract<ContentBlock, { type: 'text' }> => block.type === 'text').map(block => block.text).join('')
-  if (!text.trim()) throw new Error('llm-verifier: selected DSH model produced no text')
+  if (!text.trim()) throw unusable('selected DSH model produced no text', attempt, assembler.usage)
   // DSH adapters expose provider-neutral text/usage but not top-logprob candidates.
   // extractScore() therefore uses the model's explicit final A–T tags.
   return { text, tokens: [], positions: [], scoringMode: 'explicit-tag', usage: usage(attempt, assembler.usage), truncated }
@@ -211,6 +388,7 @@ export async function predictScoringChannel(config: VerifierClientConfig): Promi
 
 async function callAutomatic(config: VerifierClientConfig, prompt: string, signal: AbortSignal | undefined, images: readonly VerifierImage[] | undefined, attempt: number): Promise<VerifierCompletion> {
   await config.topLogprobCapabilities.ensureLoaded()
+  let fellBack = false
   if (!config.topLogprobCapabilities.isUnsupported(config.provider, config.model)) {
     const route = await resolveTopLogprobRoute(config.ctx, config.provider)
     if (route !== undefined) {
@@ -221,10 +399,14 @@ async function callAutomatic(config: VerifierClientConfig, prompt: string, signa
         // the whole verification.
         if (!(error instanceof TopLogprobsUnsupportedError)) throw error
         config.topLogprobCapabilities.markUnsupported(config.provider, config.model)
+        // A runtime downgrade AFTER a live attempt, unlike the capability probe that
+        // finds no route at all: only this one is a fallback worth counting.
+        fellBack = true
       }
     } else config.topLogprobCapabilities.markUnsupported(config.provider, config.model)
   }
-  return callExplicitTag(config, prompt, signal, images, attempt)
+  const completion = await callExplicitTag(config, prompt, signal, images, attempt)
+  return fellBack ? { ...completion, channelFallback: true } : completion
 }
 
 export async function callVerifier(config: VerifierClientConfig, prompt: string, signal?: AbortSignal, images?: readonly VerifierImage[]): Promise<VerifierCompletion> {
@@ -330,5 +512,5 @@ export async function callVerifierText(config: VerifierClientConfig, prompt: str
   const invoke = () => retrying(config, signal, (attemptSignal, attempt) => callExplicitTag(config, prompt, attemptSignal, undefined, attempt))
   return config.limiter === undefined ? invoke() : config.limiter.run(invoke, signal)
 }
-export function addUsage(target: UsageStats, source: UsageStats): void { for (const key of ['calls', 'attempts', 'retries', 'inputTokens', 'cachedInputTokens', 'outputTokens', 'reasoningTokens'] as const) target[key] += source[key] }
+export function addUsage(target: UsageStats, source: UsageStats): void { for (const key of ['calls', 'attempts', 'retries', 'inputTokens', 'cachedInputTokens', 'outputTokens', 'reasoningTokens'] as const) target[key] += source[key] ?? 0 }
 export function emptyUsage(): UsageStats { return { calls: 0, attempts: 0, retries: 0, inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, reasoningTokens: 0 } }

package/src/client-fields.test.ts

@@ -0,0 +1,309 @@
+import { describe, expect, it } from 'vitest'
+import { zh, en, computeJudgeCount, computeWorstCaseBudget } from './client-i18n.ts'
+import {
+  CONFIG_DEFAULTS,
+  FIELDS,
+  PROFILES,
+  PROFILE_IDS,
+  SECTIONS,
+  acceptsNumber,
+  activeProfile,
+  applyProfile,
+  fieldsOfSection,
+  isFieldChanged,
+  isFieldVisible,
+  issueMap,
+  issueMessageKey,
+  recommendedBudgets,
+  resetField,
+  renderSections,
+  sectionSummary,
+  textIssue,
+  validateValues,
+  valuesFromView,
+  type FieldSpec,
+  type Values,
+} from './client-fields.ts'
+
+/** A base that passes every validation: the two identity fields have no default. */
+const base: Values = { ...CONFIG_DEFAULTS, provider: 'deepseek-official', model: 'deepseek-flash' }
+const translateFor = (dictionary: Record<string, string>) => (key: string): string | undefined => dictionary[key]
+const zhT = translateFor(zh as unknown as Record<string, string>)
+const enT = translateFor(en as unknown as Record<string, string>)
+const format = (template: string, params?: Record<string, string | number>): string =>
+  (zhT(template) ?? template).replace(/\{(\w+)\}/gu, (_match, name: string) => String(params?.[name] ?? ''))
+
+const field = (key: string): FieldSpec => {
+  const found = FIELDS.find(candidate => candidate.key === key)
+  if (!found) throw new Error('unknown field ' + key)
+  return found
+}
+const codesFor = (values: Values, key: string): string[] =>
+  validateValues(values)
+    .filter(issue => issue.key === key)
+    .map(issue => issue.code)
+
+describe('settings field registry', () => {
+  it('covers every declared field exactly once and only known sections', () => {
+    const keys = FIELDS.map(entry => entry.key)
+    expect(new Set(keys).size).toBe(keys.length)
+    expect(keys.length).toBeGreaterThan(40)
+    const sections = new Set(SECTIONS.map(section => section.id))
+    for (const entry of FIELDS) expect(sections.has(entry.section)).toBe(true)
+    // every section is addressable and non-empty except none: each id must appear
+    for (const section of SECTIONS) expect(FIELDS.some(entry => entry.section === section.id)).toBe(true)
+  })
+
+  it('gives every field a default to fall back to', () => {
+    for (const entry of FIELDS) {
+      const optional = entry.key === 'reasoningEffort' || entry.key === 'label'
+      expect(optional || Object.prototype.hasOwnProperty.call(CONFIG_DEFAULTS, entry.key)).toBe(true)
+    }
+  })
+
+  it('resolves every localized title and help string in both dictionaries', () => {
+    for (const entry of FIELDS) {
+      for (const dictionary of [zhT, enT]) {
+        expect(typeof dictionary(entry.titleKey)).toBe('string')
+        expect(typeof dictionary(entry.helpKey)).toBe('string')
+        if (entry.helpKeyOff) expect(typeof dictionary(entry.helpKeyOff)).toBe('string')
+      }
+    }
+  })
+
+  it('resolves every section title, profile and validation message', () => {
+    const keys = [
+      ...SECTIONS.map(section => section.titleKey),
+      ...PROFILES.map(profile => profile.titleKey),
+      'settings.invalid.required',
+      'settings.invalid.range',
+      'settings.invalid.min',
+      'settings.invalid.max',
+      'settings.invalid.integer',
+      'settings.invalid.cacheDirRelative',
+      'settings.invalid.routeBudget',
+      'settings.summary.line',
+      'settings.summary.mode',
+      'settings.summary.routing',
+      'settings.summary.budgets',
+      'settings.summary.storage',
+      'settings.summary.execution',
+      'settings.summary.on',
+      'settings.summary.off',
+      'settings.summary.manual',
+      'settings.summary.manualShort',
+      'settings.summary.judges',
+      'settings.unit.ms',
+      'settings.unit.chars',
+      'settings.unit.calls',
+      'settings.unit.tokens',
+      'settings.fieldReset',
+      'settings.fieldChanged',
+      'settings.recommend',
+      'settings.profile.title',
+      'settings.profile.hint',
+      'settings.profile.custom',
+      'settings.advancedBadge',
+      'settings.expandAll',
+      'settings.collapseAll',
+      'settings.sectionExpand',
+      'settings.sectionCollapse',
+      'settings.jumpToIssue',
+      'settings.invalid.summary',
+    ]
+    for (const key of keys) {
+      expect(typeof zhT(key), 'zh ' + key).toBe('string')
+      expect(typeof enT(key), 'en ' + key).toBe('string')
+    }
+  })
+})
+
+describe('settings values', () => {
+  it('builds exactly the defaults from an empty namespace view', () => {
+    expect(valuesFromView({})).toEqual(CONFIG_DEFAULTS)
+    expect(valuesFromView(undefined)).toEqual(CONFIG_DEFAULTS)
+  })
+
+  it('keeps the defaults self-consistent, so a fresh form can always be saved', () => {
+    expect(validateValues(base)).toEqual([])
+  })
+
+  it('reads overrides and falls back to the safe member of every enum', () => {
+    const values = valuesFromView({
+      enabled: false,
+      autoVerifyMode: 'strict',
+      criteriaPreset: 'debug',
+      autoRouteSemantic: false,
+      maxRetries: 0,
+      cacheDir: '  nested/cache  ',
+      label: '  judge A ',
+      reasoningEffort: 'high',
+      extraJudges: [{ provider: 'p', model: 'm' }],
+    })
+    expect(values.enabled).toBe(false)
+    expect(values.autoVerifyMode).toBe('strict')
+    expect(values.criteriaPreset).toBe('debug')
+    expect(values.autoRouteSemantic).toBe(false)
+    expect(values.maxRetries).toBe(0)
+    expect(values.cacheDir).toBe('nested/cache')
+    expect(values.label).toBe('judge A')
+    expect(values.reasoningEffort).toBe('high')
+    expect(values.extraJudges).toHaveLength(1)
+    expect(valuesFromView({ autoVerifyMode: 'nonsense', criteriaPreset: 'nonsense' })).toMatchObject({
+      autoVerifyMode: 'smart',
+      criteriaPreset: 'coding',
+    })
+    // a discarded enum member must not leak an invalid value into the draft
+    expect(validateValues(valuesFromView({ provider: 'p', model: 'm', autoVerifyMode: 'nonsense', criteriaPreset: 'nonsense' }))).toEqual([])
+  })
+})
+
+describe('settings validation', () => {
+  it('mirrors the resolveConfig bounds that a text box can violate', () => {
+    expect(codesFor({ ...base, autoVerifyThreshold: 1.5 }, 'autoVerifyThreshold')).toEqual(['range'])
+    expect(codesFor({ ...base, autoRouteMinConfidence: -0.1 }, 'autoRouteMinConfidence')).toEqual(['range'])
+    expect(codesFor({ ...base, autoTrackCompletionThreshold: 2 }, 'autoTrackCompletionThreshold')).toEqual(['range'])
+    expect(codesFor({ ...base, temperature: 2.5 }, 'temperature')).toEqual(['range'])
+    expect(codesFor({ ...base, autoRouteMaxCandidates: 2 }, 'autoRouteMaxCandidates')).toEqual(['range'])
+    expect(codesFor({ ...base, autoRouteMaxCandidates: 17 }, 'autoRouteMaxCandidates')).toEqual(['range'])
+    expect(codesFor({ ...base, autoVerifyMaxChars: 999 }, 'autoVerifyMaxChars')).toEqual(['min'])
+    expect(codesFor({ ...base, maxConcurrency: 0 }, 'maxConcurrency')).toEqual(['min'])
+    expect(codesFor({ ...base, autoMaxModelCallsPerTask: 0 }, 'autoMaxModelCallsPerTask')).toEqual(['min'])
+    expect(codesFor({ ...base, maxRetries: 1.5 }, 'maxRetries')).toEqual(['integer'])
+    expect(codesFor({ ...base, estimatedOutputUsdPerMillion: -1 }, 'estimatedOutputUsdPerMillion')).toEqual(['min'])
+    expect(codesFor({ ...base, timeoutMs: Number.NaN }, 'timeoutMs')).toEqual(['required'])
+  })
+
+  it('keeps the two route budgets ordered', () => {
+    expect(codesFor({ ...base, autoRouteMaxItemChars: 20000, autoRouteMaxInputChars: 30000 }, 'autoRouteMaxInputChars')).toEqual(['routeBudget'])
+    expect(codesFor({ ...base, autoRouteMaxItemChars: 20000, autoRouteMaxInputChars: 40000 }, 'autoRouteMaxInputChars')).toEqual([])
+  })
+
+  it('rejects a cache directory outside the topic, and empty identity fields', () => {
+    expect(codesFor({ ...base, cacheDir: 'C:\\verifier' }, 'cacheDir')).toEqual(['cacheDirRelative'])
+    expect(codesFor({ ...base, cacheDir: 'a/../b' }, 'cacheDir')).toEqual(['cacheDirRelative'])
+    expect(codesFor({ ...base, cacheDir: '/etc/verifier' }, 'cacheDir')).toEqual(['cacheDirRelative'])
+    expect(codesFor({ ...base, cacheDir: ' ' }, 'cacheDir')).toEqual(['required'])
+    expect(codesFor({ ...base, cacheDir: 'nested/verifier' }, 'cacheDir')).toEqual([])
+    expect(codesFor({ ...base, provider: '' }, 'provider')).toEqual(['required'])
+    expect(codesFor({ ...base, model: '  ' }, 'model')).toEqual(['required'])
+  })
+
+  it('never blocks a missing custom rubric file: the resolver falls back and reports why', () => {
+    const values = { ...base, criteriaPreset: 'custom' as const, criteriaFile: '' }
+    expect(validateValues(values)).toEqual([])
+    // ...and the field is only rendered while the custom preset is selected
+    expect(isFieldVisible(field('criteriaFile'), values)).toBe(true)
+    expect(isFieldVisible(field('criteriaFile'), base)).toBe(false)
+    expect(codesFor(base, 'criteriaFile')).toEqual([])
+  })
+
+  it('reports parse-time feedback for a half-typed number', () => {
+    const threshold = field('autoVerifyThreshold')
+    expect(textIssue(threshold, '0.7')).toBeNull()
+    expect(textIssue(threshold, '')).toBeNull()
+    expect(textIssue(threshold, '1.5')?.code).toBe('range')
+    expect(textIssue(threshold, 'abc')?.code).toBe('required')
+    expect(textIssue(field('autoRouteMaxCandidates'), '3.5')?.code).toBe('integer')
+    expect(textIssue(field('maxRetries'), '0')).toBeNull()
+    expect(acceptsNumber(threshold, '0.65')).toBe(true)
+    expect(acceptsNumber(threshold, '1.5')).toBe(false)
+    expect(acceptsNumber(threshold, '')).toBe(false)
+  })
+
+  it('maps issues to the localized message keys and keeps the first issue per field', () => {
+    const issues = validateValues({ ...base, autoVerifyThreshold: 5, temperature: 5 })
+    const map = issueMap(issues)
+    expect(map.get('autoVerifyThreshold')?.code).toBe('range')
+    expect(issueMessageKey({ key: 'autoVerifyThreshold', code: 'range' })).toBe('settings.invalid.range')
+    expect(issues.map(issue => issue.key)).toContain('temperature')
+  })
+})
+
+describe('settings defaults and profiles', () => {
+  it('tracks a changed field and restores it', () => {
+    const threshold = field('autoVerifyThreshold')
+    expect(isFieldChanged(threshold, base)).toBe(false)
+    const changed = { ...base, autoVerifyThreshold: 0.8 }
+    expect(isFieldChanged(threshold, changed)).toBe(true)
+    expect(resetField(changed, 'autoVerifyThreshold').autoVerifyThreshold).toBe(CONFIG_DEFAULTS.autoVerifyThreshold)
+    // identity fields have no meaningful default and must not offer a reset
+    expect(isFieldChanged(field('provider'), { ...base, provider: 'other' })).toBe(false)
+    expect(isFieldChanged(field('model'), { ...base, model: 'other' })).toBe(false)
+    expect(isFieldChanged(field('extraJudges'), base)).toBe(false)
+  })
+
+  it('writes a coherent, warning-free configuration for every profile', () => {
+    const lean = { ...base, provider: 'other', model: 'other-model', criteriaPreset: 'ops' as const, maxConcurrency: 3 }
+    for (const id of PROFILE_IDS) {
+      const next = applyProfile(lean, id)
+      expect(validateValues(next), id).toEqual([])
+      // identity/plumbing settings survive a profile switch
+      expect(next.provider, id).toBe('other')
+      expect(next.model, id).toBe('other-model')
+      expect(next.criteriaPreset, id).toBe('ops')
+      expect(next.maxConcurrency, id).toBe(3)
+      const floor = recommendedBudgets(computeJudgeCount(next.extraJudges.length))
+      expect(next.autoMaxModelCallsPerTask, id).toBeGreaterThanOrEqual(floor.autoMaxModelCallsPerTask)
+      expect(next.autoMaxModelCallsPerSession, id).toBeGreaterThanOrEqual(floor.autoMaxModelCallsPerSession)
+      const worst = computeWorstCaseBudget(computeJudgeCount(next.extraJudges.length))
+      expect(next.autoMaxModelCallsPerTask, id).toBeGreaterThanOrEqual(worst.worstCaseTask)
+      expect(next.autoMaxModelCallsPerSession, id).toBeGreaterThanOrEqual(worst.worstCaseSession)
+    }
+    expect(applyProfile(lean, 'strict').autoVerifyMode).toBe('strict')
+    expect(applyProfile(lean, 'toolsOnly').autoVerifyMode).toBe('manual')
+    expect(applyProfile(lean, 'frugal')).toMatchObject({ autoVerifyMode: 'smart', autoVerifyTeamTasks: false, autoRouteMaxCandidates: 3 })
+    // the recommendation itself still respects the judge count
+    const shape = (budget: { worstCaseTask: number; worstCaseSession: number }) => ({
+      autoMaxModelCallsPerTask: budget.worstCaseTask,
+      autoMaxModelCallsPerSession: budget.worstCaseSession,
+    })
+    expect(recommendedBudgets(2)).toEqual(shape(computeWorstCaseBudget(2)))
+    expect(recommendedBudgets(0)).toEqual(shape(computeWorstCaseBudget(1)))
+  })
+})
+
+describe('active profile', () => {
+  it('reports the profile a draft currently matches, and custom otherwise', () => {
+    expect(activeProfile(base)).toBe('balanced')
+    for (const id of PROFILE_IDS) expect(activeProfile(applyProfile(base, id)), id).toBe(id)
+    expect(activeProfile({ ...base, autoVerifyThreshold: 0.5 })).toBe('custom')
+    expect(activeProfile({ ...base, autoRouteSemantic: false })).toBe('custom')
+    expect(activeProfile(applyProfile(base, 'toolsOnly'))).toBe('toolsOnly')
+  })
+
+  it('ignores the derived budgets when deciding which profile is active', () => {
+    const tuned = { ...applyProfile(base, 'strict'), autoMaxModelCallsPerTask: 321, autoMaxModelCallsPerSession: 654 }
+    expect(activeProfile(tuned)).toBe('strict')
+    expect(validateValues(tuned)).toEqual([])
+  })
+})
+
+describe('settings navigation', () => {
+  const visibleKeys = (values: Values): string[] =>
+    renderSections(values).flatMap(entry => entry.fields.map(entry => String(entry.key)))
+
+  it('renders every section with the fields that are visible for the draft', () => {
+    const all = renderSections(base)
+    expect(all.map(entry => entry.section.id)).toEqual(SECTIONS.map(section => section.id))
+    for (const entry of all) expect(entry.fields.length, entry.section.id).toBeGreaterThan(0)
+    expect(visibleKeys(base)).not.toContain('criteriaFile')
+    const custom = visibleKeys({ ...base, criteriaPreset: 'custom' })
+    expect(custom).toContain('criteriaFile')
+    expect(custom).toHaveLength(visibleKeys(base).length + 1)
+  })
+
+  it('summarizes a collapsed section from the live draft', () => {
+    expect(sectionSummary('tools', { ...base, enabled: false }, zhT, format)).toBe(zhT('settings.summary.off'))
+    expect(sectionSummary('tools', base, zhT, format)).toBe(zhT('settings.summary.on'))
+    expect(sectionSummary('autoVerify', { ...base, autoVerifyMode: 'manual' }, zhT, format)).toBe(zhT('settings.summary.manualShort'))
+    expect(sectionSummary('autoVerify', base, zhT, format)).toContain('0.65')
+    expect(sectionSummary('routing', base, zhT, format)).toContain('2/8')
+    expect(sectionSummary('budgets', base, zhT, format)).toContain('96')
+    expect(sectionSummary('model', base, zhT, format)).toBe('')
+  })
+
+  it('keeps every section reachable through its own fields', () => {
+    for (const section of SECTIONS) expect(fieldsOfSection(section.id).length).toBeGreaterThan(0)
+  })
+})