dsh-llm-verifier 0.8.0 → 0.8.2

package/src/auto.ts

@@ -33,6 +33,8 @@ export interface AutoTaskEvidence {
   /** Whether background subagents started during the task remain in flight. */
   pendingSubagents: boolean
   pendingUserInteraction: boolean
+  /** Plan mode is active: the agent may only research and propose, never be pushed to execute. */
+  planMode: boolean
   eligible: boolean
   reason: string
 }
@@ -448,9 +450,31 @@ export function inspectUserInteractionPause(
   return undefined
 }
 
+/**
+ * Whether the session is currently in plan mode.
+ *
+ * The host logs one `plan/mode` event per committed transition and folds the log as
+ * "empty log → inactive, last event wins" (its own projection does exactly this). Hosts
+ * without plan mode never log the event, so nothing changes there. While planning, the
+ * agent is expected to research and PROPOSE: any automatic route or acceptance verdict at
+ * the turn-stopping boundary can only fail against the missing implementation and steer
+ * "actually implement it" — commanding execution the human has not approved yet. The
+ * `exit_plan_mode` pre-review is the one gate that still runs in this state.
+ * @param events - Session event log (the whole log is folded; the mode may predate the task).
+ * @returns True when the newest `plan/mode` event activated plan mode.
+ */
+export function planModeActive(events: readonly SessionEvent[]): boolean {
+  let active = false
+  for (const event of events) {
+    if ((event.type as string) !== 'plan/mode') continue
+    active = (event as unknown as { data?: { active?: unknown } }).data?.active === true
+  }
+  return active
+}
+
 export function analyzeAutoTask(events: readonly SessionEvent[], policy: AutoVerifyPolicy, sessionId?: string): AutoTaskEvidence {
   const taskStartSeq = latestDirectUserSeq(events)
-  if (taskStartSeq === undefined) return { taskStartSeq: 0, toolCalls: 0, completedToolResults: 0, consequentialToolCalls: 0, hasManualSessionVerification: false, manualVerificationAccepted: false, pendingSubagents: false, pendingUserInteraction: false, eligible: false, reason: 'no-direct-user-task' }
+  if (taskStartSeq === undefined) return { taskStartSeq: 0, toolCalls: 0, completedToolResults: 0, consequentialToolCalls: 0, hasManualSessionVerification: false, manualVerificationAccepted: false, pendingSubagents: false, pendingUserInteraction: false, planMode: planModeActive(events), eligible: false, reason: 'no-direct-user-task' }
 
   const relevant = events.filter(event => event.seq >= taskStartSeq)
   const calls = relevant.filter((event): event is SessionEvent<'tool/call'> => event.type === 'tool/call')
@@ -523,15 +547,20 @@ export function analyzeAutoTask(events: readonly SessionEvent[], policy: AutoVer
   const pendingSubagents = hasPendingSubagents(events, taskStartSeq)
   const userPause = inspectUserInteractionPause(events, taskStartSeq)
   const pendingUserInteraction = userPause !== undefined
-
-  if (policy.mode === 'manual') return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, eligible: false, reason: 'manual-mode' }
-  if (manualVerificationAccepted) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, eligible: false, reason: 'already-verified' }
-  if (pendingSubagents) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents: true, pendingUserInteraction, eligible: false, reason: 'pending-subagents' }
-  if (pendingUserInteraction) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction: true, eligible: false, reason: 'user-interaction-paused' }
-  if (consequentialToolCalls === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, eligible: false, reason: 'no-consequential-work' }
-  if (completedToolResults === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, eligible: false, reason: 'no-completed-evidence' }
-  if (policy.mode === 'smart' && toolCalls < policy.minToolCalls) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, eligible: false, reason: 'insufficient-tool-evidence' }
-  return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, eligible: true, reason: policy.mode + '-eligible' }
+  const planMode = planModeActive(events)
+
+  if (policy.mode === 'manual') return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, planMode, eligible: false, reason: 'manual-mode' }
+  // Planning is the operator's review boundary, checked before every other suppression: an
+  // acceptance verdict here could only fail against the missing implementation and command
+  // execution the human has not approved yet.
+  if (planMode) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, planMode: true, eligible: false, reason: 'plan-mode-active' }
+  if (manualVerificationAccepted) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, planMode, eligible: false, reason: 'already-verified' }
+  if (pendingSubagents) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents: true, pendingUserInteraction, planMode, eligible: false, reason: 'pending-subagents' }
+  if (pendingUserInteraction) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction: true, planMode, eligible: false, reason: 'user-interaction-paused' }
+  if (consequentialToolCalls === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, planMode, eligible: false, reason: 'no-consequential-work' }
+  if (completedToolResults === 0) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, planMode, eligible: false, reason: 'no-completed-evidence' }
+  if (policy.mode === 'smart' && toolCalls < policy.minToolCalls) return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, planMode, eligible: false, reason: 'insufficient-tool-evidence' }
+  return { taskStartSeq, toolCalls, completedToolResults, consequentialToolCalls, hasManualSessionVerification, manualVerificationAccepted, pendingSubagents, pendingUserInteraction, planMode, eligible: true, reason: policy.mode + '-eligible' }
 }
 
 /** One criterion's outcome from a session acceptance (candidate A is the session). */

package/src/index.test.ts

@@ -1306,6 +1306,30 @@ describe('automatic gate lifecycle', () => {
   })
 
 
+  it('skips every automatic route and the final gate while plan mode is active', async () => {
+    // The reported bug: a planning session was judged against the missing implementation and
+    // steered to "actually implement the change" before the human approved the plan.
+    const calls: Array<Record<string, unknown>> = []
+    const { handlers } = assemble(JUDGE, { stream: scriptedStream(1, [], calls), sessions: [{ id: 'topic-plan', createdAt: 1 }] })
+    const steered: unknown[] = []
+    const planningEvents = [
+      user(0, 'Implement it'),
+      call(1, 'e', 'edit'), result(2, 'e', 'file edited'),
+      { seq: 3, type: 'plan/mode', data: { active: true } },
+      call(4, 'r', 'read'), result(5, 'r', 'file content'),
+      call(6, 'g', 'grep'), result(7, 'g', 'matches'),
+    ]
+    await handlers.get('agent/turn-stopping')!({ agent: agent(planningEvents, steered), signal: new AbortController().signal })
+    // No judge call, no routing record, no steering: the turn simply closes for plan review.
+    expect(calls.length).toBe(0)
+    expect(steered.length).toBe(0)
+
+    // Plan approved (plan/mode inactive): the same boundary gates the earlier work again.
+    const approvedEvents = [...planningEvents, { seq: 8, type: 'plan/mode', data: { active: false } }]
+    await handlers.get('agent/turn-stopping')!({ agent: agent(approvedEvents, steered), signal: new AbortController().signal })
+    expect(calls.length).toBeGreaterThan(0)
+  })
+
   it('records a low-confidence classification so "not routed" has a reason', async () => {
     // Diagnostics gap named by the review: a task that was never routed showed nothing on the
     // dashboard. The classification decision itself is now stored with its reason.
@@ -1803,6 +1827,16 @@ describe('early candidate review through agent/pre-step', () => {
     expect(judgeCalls(calls)).toHaveLength(6)
   })
 
+  it('does not inject into a step while plan mode is active', async () => {
+    const calls: Array<Record<string, unknown>> = []
+    const { handlers } = assemble(JUDGE, { stream: reviewStream(calls), sessions: [{ id: 'agent-pre', createdAt: 1 }] })
+    const events = [...workflowEvents([candidate('1', 'candidate one'), candidate('2', 'candidate two')]), { type: 'plan/mode', seq: 3, data: { active: true } }]
+    const base = { kind: 'enter' as const, messages: [] }
+    const decision = await handlers.get('agent/pre-step')!({ agent: agent(events), messages: [], turn: 1, step: 2, signal: signal() }, async () => base)
+    expect(decision).toBe(base)
+    expect(calls).toHaveLength(0)
+  })
+
   it('does not re-buy the same pair at the following stop boundary', async () => {
     const calls: Array<Record<string, unknown>> = []
     const { handlers } = assemble(JUDGE, { stream: reviewStream(calls), sessions: [{ id: 'agent-pre', createdAt: 1 }] })

package/src/index.ts

@@ -15,7 +15,7 @@ import { VerifierEngine, mergeRunStats, normalizeCriteria, partialStats, type Ju
 import { loadVerifierImages } from './images.ts'
 import { extractSession, sanitizeVerifierText, sessionEvents, type SessionExtraction } from './session.ts'
 import { CriteriaResolver, type ResolvedCriteria } from './criteria.ts'
-import { analyzeAutoTask, automaticFeedback, compareRouteFeedbackDetail, failedAcceptanceCriteria, inspectUserInteractionPause, isSubagentSession, selectRouteFeedbackDetail, sessionAccepted, MAX_ROUTE_FEEDBACK_CHARS, type AcceptanceCriterion, type RoutedCandidateRef } from './auto.ts'
+import { analyzeAutoTask, automaticFeedback, compareRouteFeedbackDetail, failedAcceptanceCriteria, inspectUserInteractionPause, isSubagentSession, planModeActive, selectRouteFeedbackDetail, sessionAccepted, MAX_ROUTE_FEEDBACK_CHARS, type AcceptanceCriterion, type RoutedCandidateRef } from './auto.ts'
 import { AutoVerifierRouter, analyzeStructuredRoute, boundDecision, buildSemanticRouteView, estimateRoutedCalls, inspectDeliveryPhase, inspectRecoverySignal, latestDirectUserSeq, nextDiagnosticCycleId, parseSemanticRoute, routedRepeats, semanticDecision, semanticReferencesVisible, semanticRouteHint, type CandidateArtifact, type Reservation, type RouteDecision, type RoutedAgent, type RoutedVerifierKind, type SemanticRouteView } from './router.ts'
 import { VerifierActivities, createActivityObserver, type ActivityView } from './verifier-activity.ts'
 import { ProcessCycleStore, ProcessSelector, resolveProcessFile, type ProcessCycleReport } from './process-selection.ts'
@@ -1337,6 +1337,10 @@ export function apply(ctx: Context, config: Config = {}): void {
     if (!selected.enabled || selected.autoVerifyMode !== 'smart') return decision
     if (payload.signal.aborted) return decision
     if (!selected.autoVerifySubagents && isSubagentSession(payload.agent)) return decision
+    // Planning is the operator's review boundary: injecting candidate feedback or buying a
+    // process-selection cycle here would push the agent to execute a plan the human has not
+    // approved yet.
+    if (planModeActive(sessionEvents(payload.agent.session))) return decision
     // A step carrying a fresh direct or team task belongs to the operator, not to us.
     const carriesTask = payload.messages.some(message => {
       const kind = (message as { source?: { kind?: unknown } }).source?.kind
@@ -1379,6 +1383,17 @@ export function apply(ctx: Context, config: Config = {}): void {
     const userPause = inspectUserInteractionPause(snapshot, evidence.taskStartSeq, turn)
     const userInteractionPending = evidence.pendingUserInteraction || userPause !== undefined
 
+    // Plan mode is the operator's review boundary: the agent is supposed to research and
+    // PROPOSE, so no automatic route, team gate or final acceptance may run here — a verdict
+    // could only fail against the missing implementation and steer "actually implement it",
+    // commanding execution the human has not approved yet. The exit_plan_mode pre-review is
+    // the one gate that still runs while planning, so this boundary resumes once the mode is
+    // exited (plan/mode active → inactive) and real work exists.
+    if (evidence.planMode) {
+      ctx.logger.info?.('llm-verifier automatic session verification skipped: plan mode is active')
+      return
+    }
+
     // A manual `verifier_current_session` that covered the whole task and passed is the
     // strongest acceptance signal available; it discharges the mandatory final gate
     // BEFORE another routed decision is purchased. `analyzeAutoTask` already refused any