@ai-dev-methodologies/rlp-desk 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -69,7 +69,9 @@ CODEX_BIN="" # resolved by check_dependencies when engine=codex
69
69
  # --- Verify Mode ---
70
70
  VERIFY_MODE="${VERIFY_MODE:-per-us}" # per-us|batch
71
71
  VERIFY_CONSENSUS="${VERIFY_CONSENSUS:-0}" # 0|1
72
+ FINAL_CONSENSUS="${FINAL_CONSENSUS:-0}" # 0|1 — consensus for final ALL verify only (independent of VERIFY_CONSENSUS)
72
73
  CONSENSUS_SCOPE="${CONSENSUS_SCOPE:-all}" # all|final-only
74
+ CONSENSUS_FAIL_FAST="${CONSENSUS_FAIL_FAST:-0}" # 0|1 — skip second verifier if first fails
73
75
  CB_THRESHOLD="${CB_THRESHOLD:-3}" # consecutive failures before BLOCKED (default: 3)
74
76
  # Effective CB threshold: doubled when consensus mode active (AC2 auto-double)
75
77
  if [[ "${VERIFY_CONSENSUS:-0}" = "1" ]]; then
@@ -77,6 +79,8 @@ if [[ "${VERIFY_CONSENSUS:-0}" = "1" ]]; then
77
79
  else
78
80
  EFFECTIVE_CB_THRESHOLD=$CB_THRESHOLD
79
81
  fi
82
+ _API_MAX_RETRIES="${_API_MAX_RETRIES:-5}"
83
+ _API_RETRY_INTERVAL_S="${_API_RETRY_INTERVAL_S:-30}"
80
84
 
81
85
  # --- Derived Paths ---
82
86
  DESK="$ROOT/.claude/ralph-desk"
@@ -84,6 +88,14 @@ PROMPTS_DIR="$DESK/prompts"
84
88
  CONTEXT_DIR="$DESK/context"
85
89
  MEMOS_DIR="$DESK/memos"
86
90
  LOGS_DIR="$DESK/logs/$SLUG"
91
+ RUNTIME_DIR="$LOGS_DIR/runtime"
92
+ PRD_FILE="$DESK/plans/prd-$SLUG.md"
93
+ TEST_SPEC_FILE="$DESK/plans/test-spec-$SLUG.md"
94
+ # --- Analytics Directory (user-level, cross-project) ---
95
+ ANALYTICS_SLUG_HASH=$(echo -n "$ROOT" | md5 -q 2>/dev/null || md5sum <<< "$ROOT" | cut -d' ' -f1)
96
+ ANALYTICS_DIR="$HOME/.claude/ralph-desk/analytics/${SLUG}--${ANALYTICS_SLUG_HASH:0:8}"
97
+ CAMPAIGN_JSONL="$ANALYTICS_DIR/campaign.jsonl"
98
+ METADATA_FILE="$ANALYTICS_DIR/metadata.json"
87
99
  WORKER_PROMPT_BASE="$PROMPTS_DIR/${SLUG}.worker.prompt.md"
88
100
  VERIFIER_PROMPT_BASE="$PROMPTS_DIR/${SLUG}.verifier.prompt.md"
89
101
  CONTEXT_FILE="$CONTEXT_DIR/${SLUG}-latest.md"
@@ -93,10 +105,11 @@ DONE_CLAIM_FILE="$MEMOS_DIR/${SLUG}-done-claim.json"
93
105
  VERDICT_FILE="$MEMOS_DIR/${SLUG}-verify-verdict.json"
94
106
  COMPLETE_SENTINEL="$MEMOS_DIR/${SLUG}-complete.md"
95
107
  BLOCKED_SENTINEL="$MEMOS_DIR/${SLUG}-blocked.md"
96
- STATUS_FILE="$LOGS_DIR/status.json"
97
- SESSION_CONFIG="$LOGS_DIR/session-config.json"
98
- WORKER_HEARTBEAT="$LOGS_DIR/worker-heartbeat.json"
99
- VERIFIER_HEARTBEAT="$LOGS_DIR/verifier-heartbeat.json"
108
+ LOCKFILE_PATH="$DESK/logs/.rlp-desk-${SLUG}.lock"
109
+ STATUS_FILE="$RUNTIME_DIR/status.json"
110
+ SESSION_CONFIG="$RUNTIME_DIR/session-config.json"
111
+ WORKER_HEARTBEAT="$RUNTIME_DIR/worker-heartbeat.json"
112
+ VERIFIER_HEARTBEAT="$RUNTIME_DIR/verifier-heartbeat.json"
100
113
  COST_LOG="$LOGS_DIR/cost-log.jsonl"
101
114
 
102
115
  # --- Session Naming ---
@@ -112,43 +125,265 @@ HEARTBEAT_STALE_COUNT=0
112
125
  MONITOR_FAILURE_COUNT=0
113
126
  CONSECUTIVE_FAILURES=0
114
127
  PREV_CONTEXT_HASH=""
128
+ PREV_PRD_HASH=""
129
+ PREV_PRD_US_LIST=""
130
+ _PRD_CHANGED=0
115
131
  ITERATION=0
116
132
  START_TIME=$(date +%s)
117
133
  BASELINE_COMMIT="" # git HEAD at campaign start (captured before loop)
118
134
  CAMPAIGN_REPORT_GENERATED=0 # guard against double-generation in cleanup trap
135
+ SV_REPORT_GENERATED=0 # guard against double-generation in generate_sv_report
119
136
  VERIFIED_US="" # comma-separated list of verified US IDs (per-us mode)
120
137
  CONSENSUS_ROUND=0 # current consensus round for current US
121
138
  US_LIST="" # comma-separated US IDs from PRD (per-us mode)
139
+ LOCKFILE_ACQUIRED=0
140
+ LOCK_WORKER_MODEL="${LOCK_WORKER_MODEL:-0}" # 0|1 — set by --lock-worker-model; disables progressive upgrade
141
+ _SAME_US_FAIL_COUNT=0 # consecutive same-US fail counter (upgrade trigger at >= 2)
142
+ _LAST_FAILED_US="" # last failed US ID (same-US tracking for upgrade logic)
143
+ _MODEL_UPGRADED=0 # 1 if Worker model was auto-upgraded during campaign
144
+ _ORIGINAL_WORKER_MODEL="" # WORKER_MODEL saved before first upgrade (for restore on pass)
145
+ _ORIGINAL_WORKER_CODEX_REASONING="" # WORKER_CODEX_REASONING saved before first upgrade
122
146
 
123
147
  # =============================================================================
124
148
  # Utility Functions
125
149
  # =============================================================================
126
150
 
127
151
  DEBUG="${DEBUG:-0}"
128
- DEBUG_LOG="$ROOT/.claude/ralph-desk/logs/${LOOP_NAME:-unknown}/debug.log"
152
+ DEBUG_LOG="$ANALYTICS_DIR/debug.log"
129
153
 
130
- log() {
131
- echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
154
+ # Source shared business logic
155
+ LIB_DIR="$(cd "$(dirname "$0")" && pwd)"
156
+ source "$LIB_DIR/lib_ralph_desk.zsh"
157
+
158
+ # A16: Warn if running in foreground (may conflict with Claude Code pane)
159
+ if [[ -z "${RLP_BACKGROUND:-}" ]]; then
160
+ echo "⚠ WARNING: Running in foreground. This may conflict with Claude Code's pane." >&2
161
+ echo " Recommended: launch via Bash tool with run_in_background: true" >&2
162
+ echo " Set RLP_BACKGROUND=1 to suppress this warning." >&2
163
+ fi
164
+
165
+ # check_dead_pane() — determine if pane command indicates a dead/exited process
166
+ # Engine-aware: bash is normal for codex workers (trigger runs in bash),
167
+ # but indicates dead pane for claude workers.
168
+ # Args: $1=pane_current_command $2=engine (claude|codex) $3=role (worker|verifier)
169
+ # Returns: 0 if dead, 1 if alive
170
+ check_dead_pane() {
171
+ local poll_cmd="$1"
172
+ local engine="${2:-claude}"
173
+ local role="${3:-worker}"
174
+
175
+ if [[ -z "$poll_cmd" ]]; then
176
+ return 0 # empty = dead
177
+ elif [[ "$poll_cmd" == "zsh" ]]; then
178
+ return 0 # bare zsh = dead
179
+ elif [[ "$poll_cmd" == "bash" && "$engine" != "codex" ]]; then
180
+ return 0 # bash = dead for claude (codex uses bash trigger)
181
+ fi
182
+ return 1 # alive
132
183
  }
133
184
 
134
- log_debug() {
135
- if (( DEBUG )); then
136
- mkdir -p "$(dirname "$DEBUG_LOG")" 2>/dev/null
137
- echo "[$(date '+%Y-%m-%d %H:%M:%S')] DEBUG: $*" >> "$DEBUG_LOG"
185
+ # launch_worker_codex() — launch codex Worker via trigger script (non-interactive exec)
186
+ # Args: $1=pane_id $2=trigger_file $3=iteration
187
+ # Returns: 0 always (codex failures detected by poll_for_signal)
188
+ launch_worker_codex() {
189
+ local pane_id="$1"
190
+ local trigger_file="$2"
191
+ local iter="$3"
192
+
193
+ log " Launching Worker codex via trigger script in pane $pane_id..."
194
+ paste_to_pane "$pane_id" "bash $trigger_file"
195
+ tmux send-keys -t "$pane_id" Enter
196
+ log_debug "Worker codex trigger sent: $trigger_file"
197
+ sleep 3 # brief wait for codex to start
198
+ return 0
199
+ }
200
+
201
+ # launch_worker_claude() — launch claude Worker TUI, send instruction, verify submission
202
+ # Handles: TUI startup, wait_for_pane_ready, instruction send, 15-iteration submit loop,
203
+ # restart recovery on submit failure.
204
+ # Args: $1=pane_id $2=prompt_file $3=iteration $4=worker_launch_cmd
205
+ # Returns: 0 on success, 1 on fatal failure (caller writes BLOCKED)
206
+ launch_worker_claude() {
207
+ local pane_id="$1"
208
+ local prompt_file="$2"
209
+ local iter="$3"
210
+ local worker_launch="$4"
211
+
212
+ log " Launching Worker claude in pane $pane_id..."
213
+ paste_to_pane "$pane_id" "$worker_launch"
214
+ tmux send-keys -t "$pane_id" Enter
215
+
216
+ # Wait for claude TUI to be ready
217
+ if ! wait_for_pane_ready "$pane_id" 30; then
218
+ log_error "Worker claude failed to start"
219
+ return 1
138
220
  fi
221
+
222
+ # Send instruction to claude TUI
223
+ sleep 3
224
+ local worker_instruction="Read and execute the instructions in $prompt_file"
225
+ paste_to_pane "$pane_id" "$worker_instruction"
226
+ tmux send-keys -t "$pane_id" Enter
227
+ log_debug "Worker instruction sent directly (${#worker_instruction} chars)"
228
+
229
+ # 15-iteration submit loop — verify claude started working
230
+ local submit_attempts=0
231
+ while (( submit_attempts < 15 )); do
232
+ sleep 2
233
+ local pane_check
234
+ pane_check=$(tmux capture-pane -t "$pane_id" -p 2>/dev/null)
235
+ if echo "$pane_check" | grep -qi "esc to interrupt\|thinking\|working\|kneading\|crunching\|clauding\|billowing\|brewing\|tinkering\|burrowing\|saut\|Exploring\|Running\|exec\|Explored\|Prestidigitating\|Undulating\|Reading\|Bash\|Edit\|Write\|Grep\|Glob" 2>/dev/null; then
236
+ log_debug "Worker started working after $((submit_attempts + 1)) submit checks"
237
+ log_debug "[FLOW] iter=$iter worker_submit_check=OK attempts=$((submit_attempts + 1))"
238
+ break
239
+ fi
240
+ # Every 3 failed attempts, re-send full instruction
241
+ if (( submit_attempts > 0 && submit_attempts % 3 == 0 )); then
242
+ log_debug "Re-sending full worker instruction (attempt $submit_attempts)"
243
+ tmux send-keys -t "$pane_id" C-u 2>/dev/null
244
+ sleep 0.2
245
+ paste_to_pane "$pane_id" "$worker_instruction"
246
+ sleep 0.15
247
+ tmux send-keys -t "$pane_id" Enter
248
+ sleep 1
249
+ fi
250
+ tmux send-keys -t "$pane_id" C-m 2>/dev/null
251
+ sleep 0.3
252
+ tmux send-keys -t "$pane_id" C-m 2>/dev/null
253
+ (( submit_attempts++ ))
254
+ done
255
+
256
+ # If 15 attempts failed, restart claude and retry
257
+ if (( submit_attempts >= 15 )); then
258
+ log " WARNING: Worker instruction not consumed after 15 attempts — restarting claude"
259
+ log_debug "[GOV] iter=$iter worker_instruction_failed=true attempts=15 action=restart_claude"
260
+ tmux send-keys -t "$pane_id" C-c 2>/dev/null
261
+ sleep 0.5
262
+ tmux send-keys -t "$pane_id" "/exit" Enter 2>/dev/null
263
+ sleep 2
264
+ wait_for_pane_ready "$pane_id" 10 2>/dev/null || true
265
+ paste_to_pane "$pane_id" "$worker_launch"
266
+ tmux send-keys -t "$pane_id" Enter
267
+ if wait_for_pane_ready "$pane_id" 30; then
268
+ sleep 3
269
+ paste_to_pane "$pane_id" "$worker_instruction"
270
+ tmux send-keys -t "$pane_id" Enter
271
+ log " Worker restarted and instruction re-sent"
272
+ log_debug "[FLOW] iter=$iter worker_restart_recovery=success"
273
+ else
274
+ log_error "Worker restart failed — pane not ready"
275
+ log_debug "[FLOW] iter=$iter worker_restart_recovery=failed"
276
+ fi
277
+ fi
278
+
279
+ return 0
139
280
  }
140
281
 
141
- log_error() {
142
- echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" >&2
282
+ # launch_verifier_codex() — launch codex Verifier in pane (non-interactive)
283
+ # Args: $1=pane_id $2=prompt_file $3=iteration $4=launch_cmd
284
+ # Returns: 0 always
285
+ launch_verifier_codex() {
286
+ local pane_id="$1"
287
+ local prompt_file="$2"
288
+ local iter="$3"
289
+ local verifier_launch="$4"
290
+
291
+ log " Launching Verifier codex in pane $pane_id..."
292
+ paste_to_pane "$pane_id" "$verifier_launch"
293
+ tmux send-keys -t "$pane_id" Enter
294
+ sleep 3
295
+ return 0
143
296
  }
144
297
 
145
- # --- governance.md s7: Atomic file writes (tmux pattern) ---
146
- # All file writes by the Leader use tmp+mv to prevent corruption.
147
- atomic_write() {
148
- local target="$1"
149
- local tmp="${target}.tmp.$$"
150
- cat > "$tmp"
151
- mv "$tmp" "$target"
298
+ # launch_verifier_claude() launch claude Verifier TUI, send instruction, verify submission
299
+ # Args: $1=pane_id $2=prompt_file $3=iteration $4=launch_cmd
300
+ # Returns: 0 on success
301
+ launch_verifier_claude() {
302
+ local pane_id="$1"
303
+ local prompt_file="$2"
304
+ local iter="$3"
305
+ local verifier_launch="$4"
306
+
307
+ log " Launching Verifier claude in pane $pane_id..."
308
+ paste_to_pane "$pane_id" "$verifier_launch"
309
+ tmux send-keys -t "$pane_id" Enter
310
+
311
+ if ! wait_for_pane_ready "$pane_id" 30; then
312
+ log_error "Verifier failed to start"
313
+ return 1
314
+ fi
315
+
316
+ sleep 3
317
+ local verifier_instruction="Read and execute the instructions in $prompt_file"
318
+ paste_to_pane "$pane_id" "$verifier_instruction"
319
+ tmux send-keys -t "$pane_id" Enter
320
+ log_debug "Verifier instruction sent directly"
321
+
322
+ # Submit loop — verify verifier started working
323
+ local submit_attempts=0
324
+ while (( submit_attempts < 15 )); do
325
+ sleep 2
326
+ local vs_check
327
+ vs_check=$(tmux capture-pane -t "$pane_id" -p 2>/dev/null)
328
+ if echo "$vs_check" | grep -qi "esc to interrupt\|thinking\|working\|kneading\|crunching\|clauding\|billowing\|brewing\|tinkering\|burrowing\|saut\|Exploring\|Running\|exec\|Explored" 2>/dev/null; then
329
+ log_debug "Verifier started working after $((submit_attempts + 1)) checks"
330
+ break
331
+ fi
332
+ if (( submit_attempts == 8 )); then
333
+ log_debug "Adaptive instruction retry: clearing line and re-typing"
334
+ tmux send-keys -t "$pane_id" C-u 2>/dev/null
335
+ sleep 0.1
336
+ paste_to_pane "$pane_id" "$verifier_instruction"
337
+ tmux send-keys -t "$pane_id" Enter
338
+ fi
339
+ tmux send-keys -t "$pane_id" C-m 2>/dev/null
340
+ sleep 0.3
341
+ tmux send-keys -t "$pane_id" C-m 2>/dev/null
342
+ (( submit_attempts++ ))
343
+ done
344
+ return 0
345
+ }
346
+
347
+ # handle_worker_exit_codex() — handle codex worker process exit (1-shot exec)
348
+ # On exit: check done-claim, auto-generate iter-signal.
349
+ # Args: $1=iteration $2=signal_file
350
+ # Returns: 0 (signal generated), 1 (error)
351
+ handle_worker_exit_codex() {
352
+ local iter="$1"
353
+ local signal_file="$2"
354
+
355
+ log " Codex worker process exited. Checking for done-claim..."
356
+ if [[ -f "$DONE_CLAIM_FILE" ]]; then
357
+ local dc_us_id
358
+ dc_us_id=$(jq -r '.us_id // "unknown"' "$DONE_CLAIM_FILE" 2>/dev/null)
359
+ log " Codex worker completed with done-claim (us_id=$dc_us_id). Auto-generating signal."
360
+ echo '{"iteration":'"$iter"',"status":"verify","us_id":"'"$dc_us_id"'","summary":"auto-generated after codex exec exit","timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'"}' > "$signal_file"
361
+ else
362
+ log " WARNING: Codex worker exited without done-claim. Generating verify signal for current US."
363
+ local current_us
364
+ current_us=$(jq -r '.us_id // "US-001"' "$DESK/memos/${SLUG}-iter-signal.json" 2>/dev/null || echo "US-001")
365
+ local mem_us
366
+ mem_us=$(sed -n 's/.*Next.*US-\([0-9]*\).*/US-\1/p' "$DESK/memos/${SLUG}-memory.md" 2>/dev/null | head -1)
367
+ [[ -n "$mem_us" ]] && current_us="$mem_us"
368
+ echo '{"iteration":'"$iter"',"status":"verify","us_id":"'"$current_us"'","summary":"auto-generated after codex exec exit (no done-claim)","timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'"}' > "$signal_file"
369
+ fi
370
+ return 0
371
+ }
372
+
373
+ # handle_worker_exit_claude() — handle claude worker process exit (restart with backoff)
374
+ # Args: $1=pane_id $2=iteration $3=trigger_file
375
+ # Returns: 0 (restarted), 1 (max restarts exceeded)
376
+ handle_worker_exit_claude() {
377
+ local pane_id="$1"
378
+ local iter="$2"
379
+ local trigger_file="$3"
380
+
381
+ log_error "Worker exited without writing signal file"
382
+ if restart_worker "$pane_id" "$iter" "$trigger_file"; then
383
+ return 0
384
+ else
385
+ return 1
386
+ fi
152
387
  }
153
388
 
154
389
  # --- omc-teams pattern: Kill-and-replace dead/stuck worker panes ---
@@ -205,9 +440,13 @@ check_dependencies() {
205
440
  missing=1
206
441
  fi
207
442
 
208
- if ! command -v claude >/dev/null 2>&1; then
209
- log_error "claude CLI is required but not found. See: https://docs.anthropic.com/en/docs/claude-cli"
210
- missing=1
443
+ # claude required only when claude engine is used for Worker or Verifier execution;
444
+ # codex-only campaigns can run without claude generate_sv_report degrades gracefully
445
+ if [[ "$WORKER_ENGINE" != "codex" || "$VERIFIER_ENGINE" != "codex" ]]; then
446
+ if ! command -v claude >/dev/null 2>&1; then
447
+ log_error "claude CLI is required but not found. See: https://docs.anthropic.com/en/docs/claude-cli"
448
+ missing=1
449
+ fi
211
450
  fi
212
451
 
213
452
  if ! command -v jq >/dev/null 2>&1; then
@@ -216,14 +455,9 @@ check_dependencies() {
216
455
  fi
217
456
 
218
457
  # Codex binary required only when engine=codex or consensus verification is enabled
219
- if [[ "$WORKER_ENGINE" = "codex" || "$VERIFIER_ENGINE" = "codex" || "$VERIFY_CONSENSUS" = "1" ]]; then
458
+ if [[ "$WORKER_ENGINE" = "codex" || "$VERIFIER_ENGINE" = "codex" || "$VERIFY_CONSENSUS" = "1" || "$FINAL_CONSENSUS" = "1" ]]; then
220
459
  if ! command -v codex >/dev/null 2>&1; then
221
- if [[ "$VERIFY_CONSENSUS" = "1" ]]; then
222
- log_error "codex CLI is required for consensus verification (VERIFY_CONSENSUS=1)."
223
- else
224
- log_error "codex CLI is required when WORKER_ENGINE or VERIFIER_ENGINE is 'codex'."
225
- fi
226
- log_error "Install with: npm install -g @openai/codex"
460
+ log_error "codex CLI not found. Install: npm install -g @openai/codex"
227
461
  missing=1
228
462
  fi
229
463
  fi
@@ -232,52 +466,19 @@ check_dependencies() {
232
466
  exit 1
233
467
  fi
234
468
 
235
- # Resolve full path to claude binary for reliable launches
236
- CLAUDE_BIN=$(command -v claude 2>/dev/null || echo "claude")
237
- log " Claude binary: $CLAUDE_BIN"
469
+ # Resolve full path to claude binary when claude engine is in use
470
+ if [[ "$WORKER_ENGINE" != "codex" || "$VERIFIER_ENGINE" != "codex" ]]; then
471
+ CLAUDE_BIN=$(command -v claude 2>/dev/null || echo "claude")
472
+ log " Claude binary: $CLAUDE_BIN"
473
+ fi
238
474
 
239
475
  # Resolve codex binary if needed
240
- if [[ "$WORKER_ENGINE" = "codex" || "$VERIFIER_ENGINE" = "codex" || "$VERIFY_CONSENSUS" = "1" ]]; then
476
+ if [[ "$WORKER_ENGINE" = "codex" || "$VERIFIER_ENGINE" = "codex" || "$VERIFY_CONSENSUS" = "1" || "$FINAL_CONSENSUS" = "1" ]]; then
241
477
  CODEX_BIN=$(command -v codex 2>/dev/null || echo "codex")
242
478
  log " Codex binary: $CODEX_BIN"
243
479
  fi
244
480
  }
245
481
 
246
- # =============================================================================
247
- # Scaffold Validation
248
- # =============================================================================
249
-
250
- validate_scaffold() {
251
- local errors=0
252
-
253
- if [[ ! -f "$WORKER_PROMPT_BASE" ]]; then
254
- log_error "Worker prompt not found: $WORKER_PROMPT_BASE"
255
- errors=1
256
- fi
257
-
258
- if [[ ! -f "$VERIFIER_PROMPT_BASE" ]]; then
259
- log_error "Verifier prompt not found: $VERIFIER_PROMPT_BASE"
260
- errors=1
261
- fi
262
-
263
- if [[ ! -f "$CONTEXT_FILE" ]]; then
264
- log_error "Context file not found: $CONTEXT_FILE"
265
- errors=1
266
- fi
267
-
268
- if [[ ! -f "$MEMORY_FILE" ]]; then
269
- log_error "Memory file not found: $MEMORY_FILE"
270
- errors=1
271
- fi
272
-
273
- if (( errors )); then
274
- log_error "Scaffold validation failed. Run init_ralph_desk.zsh first."
275
- exit 1
276
- fi
277
-
278
- mkdir -p "$LOGS_DIR"
279
- }
280
-
281
482
  # =============================================================================
282
483
  # Session Management (tmux pattern: pane IDs)
283
484
  # =============================================================================
@@ -423,6 +624,17 @@ check_copy_mode() {
423
624
  # Verification-Based Send Retry (tmux pattern)
424
625
  # =============================================================================
425
626
 
627
+ # --- Reliable text paste via tmux buffer (avoids send-keys -l char-by-char issues) ---
628
+ paste_to_pane() {
629
+ local pane_id="$1"
630
+ local text="$2"
631
+ local tmpbuf="/tmp/.rlp-desk-paste-$$.tmp"
632
+ echo -n "$text" > "$tmpbuf"
633
+ tmux load-buffer -b rlp-paste "$tmpbuf" 2>/dev/null
634
+ tmux paste-buffer -b rlp-paste -d -t "$pane_id" 2>/dev/null
635
+ rm -f "$tmpbuf"
636
+ }
637
+
426
638
  # --- governance.md s7 step 5: Send with copy-mode guard and retry ---
427
639
  safe_send_keys() {
428
640
  local pane_id="$1"
@@ -460,9 +672,9 @@ safe_send_keys() {
460
672
  tmux send-keys -t "$pane_id" "2" Enter
461
673
  sleep 0.2
462
674
  fi
463
- # Send text in literal mode with -- separator
464
- log_debug " Sending text to pane $pane_id (${#text} chars)"
465
- tmux send-keys -t "$pane_id" -l -- "$text"
675
+ # Send text via buffer paste (reliable for long strings)
676
+ log_debug " Pasting text to pane $pane_id (${#text} chars)"
677
+ paste_to_pane "$pane_id" "$text"
466
678
 
467
679
  # Allow input buffer to settle (tmux: 150ms)
468
680
  sleep 0.15
@@ -472,9 +684,7 @@ safe_send_keys() {
472
684
  while (( round < 6 )); do
473
685
  sleep 0.1
474
686
  if (( round == 0 && pane_busy )); then
475
- # Busy pane: Tab+C-m queue semantics (tmux pattern)
476
- tmux send-keys -t "$pane_id" Tab
477
- sleep 0.08
687
+ # Busy pane: just C-m (DO NOT send Tab — it toggles Claude Code permission mode)
478
688
  tmux send-keys -t "$pane_id" C-m
479
689
  else
480
690
  tmux send-keys -t "$pane_id" C-m
@@ -507,7 +717,7 @@ safe_send_keys() {
507
717
  if ! check_copy_mode "$pane_id"; then
508
718
  return 1
509
719
  fi
510
- tmux send-keys -t "$pane_id" -l -- "$text"
720
+ paste_to_pane "$pane_id" "$text"
511
721
  sleep 0.12
512
722
  local retry_round=0
513
723
  while (( retry_round < 4 )); do
@@ -655,12 +865,19 @@ check_and_nudge_idle_pane() {
655
865
  local now
656
866
  now=$(date +%s)
657
867
  if (( now - idle_since > IDLE_NUDGE_THRESHOLD )); then
658
- local count=${(P)nudge_count_var}
659
- if (( count < MAX_NUDGES )); then
660
- log " Nudging idle pane $pane_id (nudge $((count + 1))/$MAX_NUDGES)"
661
- safe_send_keys "$pane_id" ""
662
- (( count++ ))
663
- eval "$nudge_count_var=$count"
868
+ # A12 fix: NEVER nudge if pane is busy (thinking/working) — nudge interrupts claude
869
+ local _nudge_capture
870
+ _nudge_capture=$(tmux capture-pane -t "$pane_id" -p -S -5 2>/dev/null)
871
+ if echo "$_nudge_capture" | grep -qi "esc to interrupt\|thinking\|working\|kneading\|crunching\|clauding\|billowing\|brewing\|tinkering\|burrowing\|saut\|razzle\|bunning\|zesting\|fermenting\|actualizing\|composing\|evaporating\|churning" 2>/dev/null; then
872
+ log_debug " Pane $pane_id appears busy (thinking/working), skipping nudge"
873
+ else
874
+ local count=${(P)nudge_count_var}
875
+ if (( count < MAX_NUDGES )); then
876
+ log " Nudging idle pane $pane_id (nudge $((count + 1))/$MAX_NUDGES)"
877
+ safe_send_keys "$pane_id" ""
878
+ (( count++ ))
879
+ eval "$nudge_count_var=$count"
880
+ fi
664
881
  fi
665
882
  fi
666
883
  else
@@ -678,6 +895,13 @@ restart_worker() {
678
895
  local pane_id="$1"
679
896
  local iter="$2"
680
897
  local trigger_file="$3"
898
+
899
+ # Codex workers are 1-shot exec; restart is not applicable
900
+ if [[ "$WORKER_ENGINE" = "codex" ]]; then
901
+ log_debug "restart_worker called for codex engine — no-op (1-shot exec)"
902
+ return 1
903
+ fi
904
+
681
905
  local restart_count="${WORKER_RESTARTS[$iter]:-0}"
682
906
 
683
907
  if (( restart_count >= MAX_RESTARTS )); then
@@ -710,6 +934,25 @@ restart_worker() {
710
934
  # Write-Then-Notify: Trigger Script Generation (tmux CRITICAL pattern)
711
935
  # =============================================================================
712
936
 
937
+ # Per-US PRD injection helper
938
+ # Substitutes the full PRD path with a per-US split path in the Worker prompt base.
939
+ # Falls back to the full PRD with a stderr warning if the split file is missing.
940
+ # Args: $1=prompt_base_file $2=full_prd_path $3=per_us_prd_path (empty = no substitution)
941
+ inject_per_us_prd() {
942
+ local prompt_base="$1"
943
+ local full_prd="$2"
944
+ local per_us_prd="${3:-}"
945
+
946
+ if [[ -n "$per_us_prd" && -f "$per_us_prd" ]]; then
947
+ sed "s|$full_prd|$per_us_prd|g" "$prompt_base"
948
+ else
949
+ if [[ -n "$per_us_prd" ]]; then
950
+ echo "WARNING: per-US split file not found: $per_us_prd — falling back to full PRD injection" >&2
951
+ fi
952
+ cat "$prompt_base"
953
+ fi
954
+ }
955
+
713
956
  # --- governance.md s7 step 4+5: Write prompt and trigger to files ---
714
957
  # NEVER send prompt content through tmux send-keys.
715
958
  # Write payloads to files, send only short trigger commands (<200 chars).
@@ -727,14 +970,31 @@ write_worker_trigger() {
727
970
  local prev_iter=$((iter - 1))
728
971
  local fix_contract_file="$LOGS_DIR/iter-$(printf '%03d' $prev_iter).fix-contract.md"
729
972
 
973
+ # Compute next unverified US before prompt assembly (required for per-US PRD injection)
974
+ local next_us=""
975
+ if [[ "$VERIFY_MODE" = "per-us" && -n "$US_LIST" ]]; then
976
+ for us in $(echo "$US_LIST" | tr ',' ' '); do
977
+ if ! echo ",$VERIFIED_US," | grep -q ",$us,"; then
978
+ next_us="$us"
979
+ break
980
+ fi
981
+ done
982
+ fi
983
+
730
984
  {
731
- cat "$WORKER_PROMPT_BASE"
985
+ # Per-US PRD injection: substitute full PRD path with per-US split path when available
986
+ local per_us_prd=""
987
+ [[ -n "$next_us" ]] && per_us_prd="$DESK/plans/prd-${SLUG}-${next_us}.md"
988
+ inject_per_us_prd "$WORKER_PROMPT_BASE" "$DESK/plans/prd-${SLUG}.md" "$per_us_prd"
732
989
  echo ""
733
990
  echo "---"
734
991
  echo "## Iteration Context"
735
992
  echo "- **Iteration**: $iter"
736
993
  echo "- **Memory Stop Status**: $(sed -n '/^## Stop Status$/,/^$/{ /^## /d; /^$/d; p; }' "$MEMORY_FILE" 2>/dev/null | head -1)"
737
994
  echo "- **Next Iteration Contract**: ${contract:-Start from the beginning}"
995
+ if (( _PRD_CHANGED )); then
996
+ echo "NOTE: PRD was updated since last iteration. New/changed US may exist."
997
+ fi
738
998
 
739
999
  # Include fix contract if previous verifier failed
740
1000
  if [[ -f "$fix_contract_file" ]]; then
@@ -749,15 +1009,6 @@ write_worker_trigger() {
749
1009
 
750
1010
  # Per-US mode: tell Worker exactly which US to work on
751
1011
  if [[ "$VERIFY_MODE" = "per-us" && -n "$US_LIST" ]]; then
752
- # Find next unverified US
753
- local next_us=""
754
- for us in $(echo "$US_LIST" | tr ',' ' '); do
755
- if ! echo ",$VERIFIED_US," | grep -q ",$us,"; then
756
- next_us="$us"
757
- break
758
- fi
759
- done
760
-
761
1012
  if [[ -n "$next_us" ]]; then
762
1013
  echo ""
763
1014
  echo "---"
@@ -766,6 +1017,13 @@ write_worker_trigger() {
766
1017
  echo "The Leader has determined that **${next_us}** is the next unverified story."
767
1018
  echo "You MUST implement ONLY **${next_us}** in this iteration."
768
1019
  echo "Do NOT implement any other user stories."
1020
+ # Per-US test-spec injection: point Worker to scoped test-spec if available
1021
+ local per_us_test_spec="$DESK/plans/test-spec-${SLUG}-${next_us}.md"
1022
+ if [[ -f "$per_us_test_spec" ]]; then
1023
+ echo "- **Test Spec**: Read ONLY \`$per_us_test_spec\` (scoped to ${next_us})"
1024
+ else
1025
+ echo "- **Test Spec**: Read \`$DESK/plans/test-spec-${SLUG}.md\` (full — find ${next_us} section)"
1026
+ fi
769
1027
  echo "When done, signal verify with us_id=\"${next_us}\" (not \"ALL\")."
770
1028
  echo "Signal format: {\"iteration\": N, \"status\": \"verify\", \"us_id\": \"${next_us}\", ...}"
771
1029
  echo ""
@@ -793,12 +1051,12 @@ write_worker_trigger() {
793
1051
  # Write trigger script (DO NOT use exec -- breaks heartbeat cleanup)
794
1052
  # Engine-specific launch command (expanded at write time)
795
1053
  if [[ "$WORKER_ENGINE" = "codex" ]]; then
796
- local engine_cmd="${CODEX_BIN:-codex} -m $WORKER_CODEX_MODEL \\
1054
+ local engine_cmd="${CODEX_BIN:-codex} exec \\
1055
+ -m $WORKER_CODEX_MODEL \\
797
1056
  -c model_reasoning_effort=\"$WORKER_CODEX_REASONING\" \\
798
1057
  --dangerously-bypass-approvals-and-sandbox \\
799
- \"\$(cat $prompt_file)\" \\
800
- 2>&1 | tee $output_log"
801
- local engine_comment="# Run codex with fresh context (governance.md s7 step 5)"
1058
+ \"\$(cat $prompt_file)\""
1059
+ local engine_comment="# Run codex exec with fresh context (no pipe — codex requires terminal)"
802
1060
  else
803
1061
  local engine_cmd="$CLAUDE_BIN -p \"\$(cat $prompt_file)\" \\
804
1062
  --model $WORKER_MODEL \\
@@ -929,282 +1187,6 @@ TRIGGER_EOF
929
1187
  log " Verifier trigger: $trigger_file"
930
1188
  }
931
1189
 
932
- # =============================================================================
933
- # Status Updates
934
- # =============================================================================
935
-
936
- # --- governance.md s7 step 8: Update status.json ---
937
- update_status() {
938
- local phase="$1"
939
- local last_result="$2"
940
-
941
- # Build verified_us as JSON array
942
- local verified_us_json="[]"
943
- if [[ -n "$VERIFIED_US" ]]; then
944
- verified_us_json=$(echo "$VERIFIED_US" | tr ',' '\n' | jq -R . | jq -s .)
945
- fi
946
-
947
- # Build consensus fields
948
- local consensus_json=""
949
- if [[ "$VERIFY_CONSENSUS" = "1" ]]; then
950
- consensus_json=',
951
- "consensus_scope": "'"$CONSENSUS_SCOPE"'",
952
- "consensus_round": '"$CONSENSUS_ROUND"',
953
- "claude_verdict": "'"${CLAUDE_VERDICT:-}"'",
954
- "codex_verdict": "'"${CODEX_VERDICT:-}"'"'
955
- fi
956
-
957
- echo '{
958
- "slug": "'"$SLUG"'",
959
- "baseline_commit": "'"${BASELINE_COMMIT:-none}"'",
960
- "iteration": '"$ITERATION"',
961
- "max_iter": '"$MAX_ITER"',
962
- "phase": "'"$phase"'",
963
- "worker_model": "'"$WORKER_MODEL"'",
964
- "verifier_model": "'"$VERIFIER_MODEL"'",
965
- "worker_engine": "'"$WORKER_ENGINE"'",
966
- "verifier_engine": "'"$VERIFIER_ENGINE"'",
967
- "worker_codex_model": "'"$WORKER_CODEX_MODEL"'",
968
- "worker_codex_reasoning": "'"$WORKER_CODEX_REASONING"'",
969
- "verifier_codex_model": "'"$VERIFIER_CODEX_MODEL"'",
970
- "verifier_codex_reasoning": "'"$VERIFIER_CODEX_REASONING"'",
971
- "verify_mode": "'"$VERIFY_MODE"'",
972
- "verify_consensus": '"$VERIFY_CONSENSUS"',
973
- "last_result": "'"$last_result"'",
974
- "consecutive_failures": '"$CONSECUTIVE_FAILURES"',
975
- "verified_us": '"$verified_us_json"''"$consensus_json"',
976
- "updated_at_utc": "'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'"
977
- }' | atomic_write "$STATUS_FILE"
978
- }
979
-
980
- # --- governance.md s7 step 8: Write result log ---
981
- write_result_log() {
982
- local iter="$1"
983
- local result="$2"
984
- local result_file="$LOGS_DIR/iter-$(printf '%03d' $iter).result.md"
985
-
986
- local git_diff=""
987
- if git -C "$ROOT" rev-parse HEAD &>/dev/null; then
988
- git_diff=$(git -C "$ROOT" diff --stat HEAD 2>/dev/null || echo "(no git diff available)")
989
- else
990
- git_diff="(no commits in repo — cannot diff)"
991
- fi
992
- # Include untracked new files in result log
993
- local result_untracked
994
- result_untracked=$(git -C "$ROOT" ls-files --others --exclude-standard 2>/dev/null | head -20)
995
- if [[ -n "$result_untracked" ]]; then
996
- git_diff="${git_diff}
997
-
998
- Untracked new files:
999
- ${result_untracked}"
1000
- fi
1001
-
1002
- {
1003
- echo "# Iteration $iter Result"
1004
- echo ""
1005
- echo "## Status"
1006
- echo "$result [leader-measured]"
1007
- echo ""
1008
- echo "## Files Changed"
1009
- echo '```'
1010
- echo "$git_diff"
1011
- echo '```'
1012
- echo "[git-measured]"
1013
- echo ""
1014
- echo "## Timestamp"
1015
- echo "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
1016
- } | atomic_write "$result_file"
1017
- }
1018
-
1019
- # --- step 7d: Archive iteration artifacts (done-claim + verdict) to logs/ ---
1020
- archive_iter_artifacts() {
1021
- local iter="$1"
1022
- local iter_padded
1023
- iter_padded=$(printf '%03d' "$iter")
1024
- if [[ -f "$DONE_CLAIM_FILE" ]]; then
1025
- cp "$DONE_CLAIM_FILE" "$LOGS_DIR/iter-${iter_padded}-done-claim.json" 2>/dev/null
1026
- fi
1027
- if [[ -f "$VERDICT_FILE" ]]; then
1028
- cp "$VERDICT_FILE" "$LOGS_DIR/iter-${iter_padded}-verify-verdict.json" 2>/dev/null
1029
- fi
1030
- }
1031
-
1032
- # --- AC5: Write per-iteration cost estimate to cost-log.jsonl ---
1033
- write_cost_log() {
1034
- local iter="$1"
1035
- local iter_padded
1036
- iter_padded=$(printf '%03d' "$iter")
1037
-
1038
- local prompt_bytes=0 claim_bytes=0 verdict_bytes=0
1039
- local worker_prompt_file="$LOGS_DIR/iter-${iter_padded}.worker-prompt.md"
1040
- [[ -f "$worker_prompt_file" ]] && prompt_bytes=$(wc -c < "$worker_prompt_file" 2>/dev/null || echo 0)
1041
- [[ -f "$DONE_CLAIM_FILE" ]] && claim_bytes=$(wc -c < "$DONE_CLAIM_FILE" 2>/dev/null || echo 0)
1042
- [[ -f "$VERDICT_FILE" ]] && verdict_bytes=$(wc -c < "$VERDICT_FILE" 2>/dev/null || echo 0)
1043
-
1044
- local estimated_tokens=$(( (prompt_bytes + claim_bytes + verdict_bytes) / 4 ))
1045
-
1046
- echo '{"iteration":'"$iter"',"estimated_tokens":'"$estimated_tokens"',"token_source":"estimated","prompt_bytes":'"$prompt_bytes"',"claim_bytes":'"$claim_bytes"',"verdict_bytes":'"$verdict_bytes"',"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'"}' >> "$COST_LOG"
1047
- }
1048
-
1049
- # --- AC4: Generate campaign-report.md on all terminal states ---
1050
- generate_campaign_report() {
1051
- # Guard: idempotent — only generate once per campaign run
1052
- if (( CAMPAIGN_REPORT_GENERATED )); then return 0; fi
1053
- CAMPAIGN_REPORT_GENERATED=1
1054
-
1055
- local final_status="UNKNOWN"
1056
- if [[ -f "$COMPLETE_SENTINEL" ]]; then final_status="COMPLETE"
1057
- elif [[ -f "$BLOCKED_SENTINEL" ]]; then final_status="BLOCKED"
1058
- else final_status="TIMEOUT"; fi
1059
-
1060
- local report_file="$LOGS_DIR/campaign-report.md"
1061
-
1062
- # AC9: Version existing report before writing new one
1063
- if [[ -f "$report_file" ]]; then
1064
- local v=1
1065
- while [[ -f "${report_file%.md}-v${v}.md" ]]; do (( v++ )); done
1066
- mv "$report_file" "${report_file%.md}-v${v}.md"
1067
- fi
1068
-
1069
- local end_time
1070
- end_time=$(date +%s)
1071
- local elapsed=$(( end_time - START_TIME ))
1072
-
1073
- local baseline_commit_val="${BASELINE_COMMIT:-none}"
1074
- local files_changed=""
1075
- if [[ "$baseline_commit_val" != "none" ]]; then
1076
- files_changed=$(git -C "$ROOT" diff --stat "${baseline_commit_val}" 2>/dev/null || echo "(git diff unavailable)")
1077
- elif git -C "$ROOT" rev-parse HEAD &>/dev/null; then
1078
- files_changed=$(git -C "$ROOT" diff --stat HEAD 2>/dev/null || echo "(git diff unavailable)")
1079
- else
1080
- files_changed="(no commits in repo — cannot diff)"
1081
- fi
1082
- # Include untracked new files
1083
- local untracked
1084
- untracked=$(git -C "$ROOT" ls-files --others --exclude-standard 2>/dev/null | head -20)
1085
- if [[ -n "$untracked" ]]; then
1086
- files_changed="${files_changed}
1087
-
1088
- Untracked new files:
1089
- ${untracked}"
1090
- fi
1091
-
1092
- local sv_summary=""
1093
- if (( WITH_SELF_VERIFICATION )); then
1094
- local sv_report
1095
- sv_report=$(ls -t "$LOGS_DIR"/self-verification-report-*.md 2>/dev/null | head -1)
1096
- if [[ -n "$sv_report" ]]; then
1097
- sv_summary="See: $(basename "$sv_report")"
1098
- else
1099
- sv_summary="SV report generation requires Agent mode. Flag recorded in session-config."
1100
- fi
1101
- else
1102
- sv_summary="N/A — --with-self-verification not enabled"
1103
- fi
1104
-
1105
- {
1106
- echo "# Campaign Report: $SLUG"
1107
- echo ""
1108
- echo "Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ) | Status: $final_status | Iterations: $ITERATION"
1109
- echo ""
1110
- echo "## Objective"
1111
- local prd_file="$DESK/plans/prd-$SLUG.md"
1112
- if [[ -f "$prd_file" ]]; then
1113
- grep '^## Objective' -A3 "$prd_file" 2>/dev/null | tail -n +2 | head -3
1114
- else
1115
- echo "(PRD not found)"
1116
- fi
1117
- echo ""
1118
- echo "## Execution Summary"
1119
- echo "- Terminal state: $final_status"
1120
- echo "- Iterations run: $ITERATION / $MAX_ITER"
1121
- echo "- Elapsed: ${elapsed}s"
1122
- echo "- Worker model: $WORKER_MODEL ($WORKER_ENGINE)"
1123
- echo "- Verifier model: $VERIFIER_MODEL ($VERIFIER_ENGINE)"
1124
- echo ""
1125
- echo "## US Status"
1126
- echo "- Verified: ${VERIFIED_US:-none}"
1127
- echo "- Consecutive failures at end: $CONSECUTIVE_FAILURES"
1128
- echo ""
1129
- echo "## Verification Results"
1130
- local ri=1
1131
- while (( ri <= ITERATION )); do
1132
- local iter_dc="$LOGS_DIR/iter-$(printf '%03d' $ri)-done-claim.json"
1133
- if [[ -f "$iter_dc" ]]; then
1134
- local us_id
1135
- us_id=$(jq -r '.us_id // "unknown"' "$iter_dc" 2>/dev/null)
1136
- echo "- $(basename "$iter_dc"): us_id=$us_id"
1137
- fi
1138
- (( ri++ ))
1139
- done
1140
- echo ""
1141
- echo "## Issues Encountered"
1142
- local fi_found=0
1143
- local fi_i=1
1144
- while (( fi_i <= ITERATION )); do
1145
- local fix_f="$LOGS_DIR/iter-$(printf '%03d' $fi_i).fix-contract.md"
1146
- if [[ -f "$fix_f" ]]; then
1147
- echo "- $(basename "$fix_f")"
1148
- fi_found=1
1149
- fi
1150
- (( fi_i++ ))
1151
- done
1152
- (( fi_found == 0 )) && echo "- None"
1153
- echo ""
1154
- echo "## Cost & Performance"
1155
- if [[ -f "$COST_LOG" ]]; then
1156
- local total_tokens=0
1157
- while IFS= read -r line; do
1158
- local t
1159
- t=$(echo "$line" | jq -r '.estimated_tokens // 0' 2>/dev/null || echo 0)
1160
- total_tokens=$(( total_tokens + t ))
1161
- done < "$COST_LOG"
1162
- echo "- Total estimated tokens: $total_tokens (source: estimated, tmux mode)"
1163
- echo "- See: cost-log.jsonl for per-iteration breakdown"
1164
- else
1165
- echo "- No cost data available"
1166
- fi
1167
- echo ""
1168
- echo "## SV Summary"
1169
- echo "$sv_summary"
1170
- echo ""
1171
- echo "## Files Changed"
1172
- echo '```'
1173
- echo "$files_changed"
1174
- echo '```'
1175
- echo "Note: Files Changed may include pre-existing uncommitted changes if the campaign started in a dirty worktree."
1176
- } | atomic_write "$report_file"
1177
-
1178
- log "Campaign report written: $report_file"
1179
- }
1180
-
1181
- # =============================================================================
1182
- # Sentinel Writers
1183
- # =============================================================================
1184
-
1185
- # --- governance.md s7: Only the Leader writes sentinels ---
1186
- write_complete_sentinel() {
1187
- local summary="$1"
1188
- echo "# Campaign Complete
1189
-
1190
- Completed at iteration $ITERATION.
1191
- $summary
1192
-
1193
- Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | atomic_write "$COMPLETE_SENTINEL"
1194
- log "COMPLETE sentinel written: $COMPLETE_SENTINEL"
1195
- }
1196
-
1197
- write_blocked_sentinel() {
1198
- local reason="$1"
1199
- echo "# Campaign Blocked
1200
-
1201
- Blocked at iteration $ITERATION.
1202
- Reason: $reason
1203
-
1204
- Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | atomic_write "$BLOCKED_SENTINEL"
1205
- log "BLOCKED sentinel written: $BLOCKED_SENTINEL"
1206
- }
1207
-
1208
1190
  # =============================================================================
1209
1191
  # Cleanup (trap handler)
1210
1192
  # =============================================================================
@@ -1213,7 +1195,11 @@ cleanup() {
1213
1195
  log "Cleaning up..."
1214
1196
 
1215
1197
  # Remove lockfile
1216
- rm -f "$DESK/logs/.rlp-desk-$SLUG.lock" 2>/dev/null
1198
+ if (( LOCKFILE_ACQUIRED )); then
1199
+ rm -f "$LOCKFILE_PATH" 2>/dev/null
1200
+ else
1201
+ log_debug "cleanup: lockfile not owned by this process, skipping removal"
1202
+ fi
1217
1203
 
1218
1204
  # Kill claude processes then kill panes
1219
1205
  log_debug "cleanup: WORKER_PANE=${WORKER_PANE:-unset} VERIFIER_PANE=${VERIFIER_PANE:-unset}"
@@ -1242,6 +1228,9 @@ cleanup() {
1242
1228
  # AC4: Generate campaign report on all terminal states (always-on)
1243
1229
  generate_campaign_report
1244
1230
 
1231
+ # US-001: Generate SV report after campaign report (tmux mode)
1232
+ generate_sv_report
1233
+
1245
1234
  # Print summary
1246
1235
  local end_time
1247
1236
  end_time=$(date +%s)
@@ -1254,6 +1243,13 @@ cleanup() {
1254
1243
  elif [[ -f "$BLOCKED_SENTINEL" ]]; then final_status="BLOCKED"
1255
1244
  else final_status="TIMEOUT"; fi
1256
1245
 
1246
+ # --- Update metadata.json with final status ---
1247
+ if [[ -f "$METADATA_FILE" ]]; then
1248
+ jq --arg status "$final_status" --arg end_time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
1249
+ '.campaign_status = $status | .end_time = $end_time' \
1250
+ "$METADATA_FILE" > "${METADATA_FILE}.tmp" && mv "${METADATA_FILE}.tmp" "$METADATA_FILE"
1251
+ fi
1252
+
1257
1253
  if (( DEBUG )); then
1258
1254
  local end_ts=$(date +%s)
1259
1255
  local elapsed=$((end_ts - START_TIME))
@@ -1350,6 +1346,7 @@ poll_for_signal() {
1350
1346
  local trigger_file="$4"
1351
1347
  local role="$5" # "worker" or "verifier"
1352
1348
  local nudge_count=0
1349
+ local api_retry_count=0
1353
1350
  local poll_start
1354
1351
  poll_start=$(date +%s)
1355
1352
 
@@ -1374,6 +1371,54 @@ poll_for_signal() {
1374
1371
  return 0 # success
1375
1372
  fi
1376
1373
 
1374
+ # A4 fallback: done-claim exists but no signal → Worker forgot iter-signal
1375
+ # ONLY for Worker polling — Verifier waits for verdict file, not done-claim
1376
+ if [[ "$role" != *erifier* && -f "$DONE_CLAIM_FILE" && ! -f "$signal_file" ]]; then
1377
+ local dc_us_id
1378
+ dc_us_id=$(jq -r '.us_id // "unknown"' "$DONE_CLAIM_FILE" 2>/dev/null)
1379
+ if [[ -n "$dc_us_id" && "$dc_us_id" != "null" ]]; then
1380
+ log " WARNING: done-claim exists for $dc_us_id but no iter-signal. Auto-generating signal (A4 fallback)."
1381
+ log_debug "[GOV] iter=$ITERATION done_claim_without_signal=true us_id=$dc_us_id action=auto_generate_signal"
1382
+ echo '{"iteration":'"$ITERATION"',"status":"verify","us_id":"'"$dc_us_id"'","summary":"auto-generated by A4 fallback (done-claim without signal)","timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'"}' > "$signal_file"
1383
+ return 0
1384
+ fi
1385
+ fi
1386
+
1387
+ # API transient-error recovery with bounded backoff
1388
+ local pane_output_for_retry
1389
+ pane_output_for_retry=$(tmux capture-pane -t "$pane_id" -p 2>/dev/null || true)
1390
+ local is_api_text_retry=0
1391
+ if [[ -n "$pane_output_for_retry" ]] &&
1392
+ ( echo "$pane_output_for_retry" | grep -qiE '(^|[^[:digit:]])500([^[:digit:]]|$)' \
1393
+ || echo "$pane_output_for_retry" | grep -qiE '(^|[^[:digit:]])529([^[:digit:]]|$)' \
1394
+ || echo "$pane_output_for_retry" | grep -qi 'overloaded' \
1395
+ || echo "$pane_output_for_retry" | grep -qi 'too many requests' \
1396
+ || echo "$pane_output_for_retry" | grep -qi 'service unavailable' ); then
1397
+ is_api_text_retry=1
1398
+ fi
1399
+
1400
+ if (( is_api_text_retry )) || is_api_error "$pane_id"; then
1401
+ (( api_retry_count++ ))
1402
+ log_debug "[FLOW] iter=$ITERATION api_retry=${api_retry_count}/${_API_MAX_RETRIES} role=${role} reason=tmux_pane_api_error"
1403
+ if (( api_retry_count >= _API_MAX_RETRIES )); then
1404
+ log_error "API unavailable after ${_API_MAX_RETRIES} retries"
1405
+ write_blocked_sentinel "API unavailable after ${_API_MAX_RETRIES} retries"
1406
+ return 2
1407
+ fi
1408
+ # A5: If pane shows "queued messages" or rate-limit corruption, restart pane
1409
+ if echo "$pane_output_for_retry" | grep -qi 'queued messages'; then
1410
+ log " A5: Rate-limited pane shows 'queued messages' — restarting $role pane"
1411
+ log_debug "[GOV] iter=$ITERATION phase=rate_limit_pane_restart role=$role reason=queued_messages"
1412
+ tmux send-keys -t "$pane_id" C-c 2>/dev/null; sleep 0.5
1413
+ tmux send-keys -t "$pane_id" "/exit" Enter 2>/dev/null; sleep 2
1414
+ wait_for_pane_ready "$pane_id" 10 2>/dev/null || true
1415
+ fi
1416
+ sleep "$_API_RETRY_INTERVAL_S"
1417
+ continue
1418
+ else
1419
+ api_retry_count=0
1420
+ fi
1421
+
1377
1422
  # Check heartbeat freshness (tmux pattern)
1378
1423
  if [[ -f "$heartbeat_file" ]]; then
1379
1424
  if check_heartbeat_exited "$heartbeat_file"; then
@@ -1383,9 +1428,13 @@ poll_for_signal() {
1383
1428
  log " Signal file detected after process exit: $signal_file"
1384
1429
  return 0
1385
1430
  fi
1386
- log_error "$role exited without writing signal file"
1387
- # Attempt restart with exponential backoff
1388
- if restart_worker "$pane_id" "$ITERATION" "$trigger_file"; then
1431
+ # Dispatch to engine-specific exit handler
1432
+ if [[ "$WORKER_ENGINE" = "codex" && "$role" != *erifier* ]]; then
1433
+ handle_worker_exit_codex "$ITERATION" "$signal_file"
1434
+ return 0
1435
+ fi
1436
+ # Claude path (or verifier of any engine)
1437
+ if handle_worker_exit_claude "$pane_id" "$ITERATION" "$trigger_file"; then
1389
1438
  # Reset poll timer for the restart
1390
1439
  poll_start=$(date +%s)
1391
1440
  nudge_count=0
@@ -1421,6 +1470,17 @@ poll_for_signal() {
1421
1470
  fi
1422
1471
  fi
1423
1472
 
1473
+ # Dead pane detection during poll: check if claude/codex process died
1474
+ local poll_cmd
1475
+ poll_cmd=$(tmux display-message -p -t "$pane_id" '#{pane_current_command}' 2>/dev/null)
1476
+ # Dead pane detection — delegates to check_dead_pane() for engine-aware logic
1477
+ if check_dead_pane "$poll_cmd" "$WORKER_ENGINE" "$role"; then
1478
+ log " WARNING: $role pane $pane_id has bare shell ($poll_cmd) — process died during execution"
1479
+ log_debug "[GOV] iter=$ITERATION pane_dead_during_poll=true pane=$pane_id cmd=$poll_cmd role=$role"
1480
+ # Return failure so caller can handle recovery
1481
+ return 1
1482
+ fi
1483
+
1424
1484
  # Auto-approve permission prompts during poll
1425
1485
  local poll_capture
1426
1486
  poll_capture=$(tmux capture-pane -t "$pane_id" -p 2>/dev/null)
@@ -1438,38 +1498,6 @@ poll_for_signal() {
1438
1498
  done
1439
1499
  }
1440
1500
 
1441
- # =============================================================================
1442
- # Circuit Breaker: Stale Context Detection
1443
- # =============================================================================
1444
-
1445
- # --- governance.md s7 step 8: Stale context detection ---
1446
- compute_context_hash() {
1447
- if [[ -f "$CONTEXT_FILE" ]]; then
1448
- md5 -q "$CONTEXT_FILE" 2>/dev/null || md5sum "$CONTEXT_FILE" 2>/dev/null | cut -d' ' -f1
1449
- else
1450
- echo "no-context"
1451
- fi
1452
- }
1453
-
1454
- check_stale_context() {
1455
- local current_hash
1456
- current_hash=$(compute_context_hash)
1457
-
1458
- if [[ "$current_hash" == "$PREV_CONTEXT_HASH" ]]; then
1459
- (( STALE_CONTEXT_COUNT++ ))
1460
- log " WARNING: Context unchanged ($STALE_CONTEXT_COUNT/3 stale iterations)"
1461
- if (( STALE_CONTEXT_COUNT >= 3 )); then
1462
- log_error "Circuit breaker: context unchanged for 3 consecutive iterations"
1463
- return 1
1464
- fi
1465
- else
1466
- STALE_CONTEXT_COUNT=0
1467
- fi
1468
-
1469
- PREV_CONTEXT_HASH="$current_hash"
1470
- return 0
1471
- }
1472
-
1473
1501
  # =============================================================================
1474
1502
  # Consensus Verification (run two verifiers sequentially in same pane)
1475
1503
  # =============================================================================
@@ -1487,10 +1515,23 @@ run_single_verifier() {
1487
1515
  local trigger_file="$LOGS_DIR/iter-$(printf '%03d' $iter).verifier${suffix}-trigger.sh"
1488
1516
  local prompt_file="$LOGS_DIR/iter-$(printf '%03d' $iter).verifier${suffix}-prompt.md"
1489
1517
 
1490
- # Clean previous Verifier session
1518
+ # Clean previous Verifier session (with dead pane detection)
1491
1519
  local verifier_cmd
1492
1520
  verifier_cmd=$(tmux display-message -p -t "$VERIFIER_PANE" '#{pane_current_command}' 2>/dev/null)
1493
- if [[ "$verifier_cmd" == "node" || "$verifier_cmd" == "claude" || "$verifier_cmd" == "codex" ]]; then
1521
+ if [[ -z "$verifier_cmd" ]]; then
1522
+ log " Verifier pane $VERIFIER_PANE is gone — replacing..."
1523
+ log_debug "[GOV] iter=$iter pane_dead=true pane_id=$VERIFIER_PANE action=replace_pane"
1524
+ replace_worker_pane "$VERIFIER_PANE" "verifier"
1525
+ VERIFIER_PANE=$(jq -r '.panes.verifier' "$SESSION_CONFIG")
1526
+ log " New verifier pane: $VERIFIER_PANE"
1527
+ elif [[ "$verifier_cmd" == "zsh" || "$verifier_cmd" == "bash" ]]; then
1528
+ log " Verifier pane $VERIFIER_PANE has bare shell ($verifier_cmd) — resetting..."
1529
+ log_debug "[GOV] iter=$iter pane_dead=true pane_id=$VERIFIER_PANE cmd=$verifier_cmd action=reset_shell"
1530
+ tmux send-keys -t "$VERIFIER_PANE" C-c C-u 2>/dev/null
1531
+ sleep 0.2
1532
+ tmux send-keys -t "$VERIFIER_PANE" "clear" Enter 2>/dev/null
1533
+ sleep 0.3
1534
+ elif [[ "$verifier_cmd" == "node" || "$verifier_cmd" == "claude" || "$verifier_cmd" == "codex" ]]; then
1494
1535
  tmux send-keys -t "$VERIFIER_PANE" C-c 2>/dev/null
1495
1536
  sleep 0.5
1496
1537
  tmux send-keys -t "$VERIFIER_PANE" "/exit" Enter 2>/dev/null
@@ -1505,55 +1546,19 @@ run_single_verifier() {
1505
1546
  # Remove previous verdict file
1506
1547
  rm -f "$VERDICT_FILE" 2>/dev/null
1507
1548
 
1508
- # Launch verifier
1549
+ # Launch verifier — dispatch to engine-specific function
1550
+ local verifier_launch
1509
1551
  if [[ "$engine" = "codex" ]]; then
1510
- # Codex: use non-interactive exec mode in pane (more reliable than TUI for sequential runs)
1511
- local codex_cmd="${CODEX_BIN:-codex} exec \"\$(cat $prompt_file)\" -m $VERIFIER_CODEX_MODEL -c model_reasoning_effort=\"$VERIFIER_CODEX_REASONING\" --dangerously-bypass-approvals-and-sandbox"
1512
- log " Running $suffix verifier (codex exec) in pane $VERIFIER_PANE..."
1513
- tmux send-keys -t "$VERIFIER_PANE" -l -- "$codex_cmd"
1514
- tmux send-keys -t "$VERIFIER_PANE" Enter
1515
- log_debug "Verifier$suffix codex exec sent directly"
1552
+ verifier_launch="${CODEX_BIN:-codex} exec \"\$(cat $prompt_file)\" -m $VERIFIER_CODEX_MODEL -c model_reasoning_effort=\"$VERIFIER_CODEX_REASONING\" --dangerously-bypass-approvals-and-sandbox"
1553
+ launch_verifier_codex "$VERIFIER_PANE" "$prompt_file" "$iter" "$verifier_launch"
1554
+ log_debug "Verifier$suffix codex exec dispatched"
1516
1555
  else
1517
- # Claude: use interactive TUI
1518
- local verifier_launch="$CLAUDE_BIN --model $model --dangerously-skip-permissions"
1519
- log " Launching $suffix verifier (claude) in pane $VERIFIER_PANE..."
1520
- tmux send-keys -t "$VERIFIER_PANE" -l -- "$verifier_launch"
1521
- tmux send-keys -t "$VERIFIER_PANE" Enter
1522
-
1523
- if ! wait_for_pane_ready "$VERIFIER_PANE" 30; then
1556
+ verifier_launch="$CLAUDE_BIN --model $model --dangerously-skip-permissions"
1557
+ if ! launch_verifier_claude "$VERIFIER_PANE" "$prompt_file" "$iter" "$verifier_launch"; then
1524
1558
  log_error "Verifier$suffix failed to start"
1525
1559
  return 1
1526
1560
  fi
1527
-
1528
- sleep 3
1529
- local verifier_instruction="Read and execute the instructions in $prompt_file"
1530
- tmux send-keys -t "$VERIFIER_PANE" -l -- "$verifier_instruction"
1531
- tmux send-keys -t "$VERIFIER_PANE" Enter
1532
- log_debug "Verifier$suffix instruction sent directly"
1533
-
1534
- # Verify claude actually started working
1535
- local v_submit=0
1536
- while (( v_submit < 15 )); do
1537
- sleep 2
1538
- local v_check
1539
- v_check=$(tmux capture-pane -t "$VERIFIER_PANE" -p 2>/dev/null)
1540
- if echo "$v_check" | grep -qi "esc to interrupt\|thinking\|working\|kneading\|crunching\|clauding\|billowing\|brewing\|tinkering\|burrowing\|saut" 2>/dev/null; then
1541
- log_debug "Verifier$suffix started working after $((v_submit + 1)) checks"
1542
- break
1543
- fi
1544
- # After 8 failed attempts, try C-u clear + re-type (omc-teams adaptive retry)
1545
- if (( v_submit == 8 )); then
1546
- log_debug "Adaptive instruction retry: clearing line and re-typing"
1547
- tmux send-keys -t "$VERIFIER_PANE" C-u 2>/dev/null
1548
- sleep 0.1
1549
- tmux send-keys -t "$VERIFIER_PANE" -l -- "$verifier_instruction"
1550
- tmux send-keys -t "$VERIFIER_PANE" Enter
1551
- fi
1552
- tmux send-keys -t "$VERIFIER_PANE" C-m 2>/dev/null
1553
- sleep 0.3
1554
- tmux send-keys -t "$VERIFIER_PANE" C-m 2>/dev/null
1555
- (( v_submit++ ))
1556
- done
1561
+ log_debug "Verifier$suffix claude dispatched"
1557
1562
  fi
1558
1563
 
1559
1564
  # Poll for verdict
@@ -1581,6 +1586,10 @@ run_single_verifier() {
1581
1586
  # Claude: use full poll_for_signal with heartbeat/nudge
1582
1587
  log " Polling for verify-verdict.json ($suffix)..."
1583
1588
  if ! poll_for_signal "$VERDICT_FILE" "$VERIFIER_HEARTBEAT" "$VERIFIER_PANE" "$verifier_launch" "Verifier$suffix"; then
1589
+ local verifier_poll_rc=$?
1590
+ if (( verifier_poll_rc == 2 )); then
1591
+ return 1
1592
+ fi
1584
1593
  log_error "Verifier$suffix poll failed"
1585
1594
  return 1
1586
1595
  fi
@@ -1592,6 +1601,110 @@ run_single_verifier() {
1592
1601
  return 0
1593
1602
  }
1594
1603
 
1604
+ # --- Sequential final verify: run per-US scoped verifiers instead of one big ALL verify ---
1605
+ # Returns 0 if all US pass + integration check pass, 1 if any US fails, 2 if integration fails.
1606
+ # Sets FAILED_US global on failure.
1607
+ run_sequential_final_verify() {
1608
+ local iter="$1"
1609
+ FAILED_US=""
1610
+
1611
+ log " Sequential final verify: ${US_LIST} (${VERIFY_MODE} mode)"
1612
+ log_debug "[FLOW] iter=$iter phase=sequential_final_verify us_list=$US_LIST"
1613
+
1614
+ for us in $(echo "$US_LIST" | tr ',' ' '); do
1615
+ log " Final verify: checking $us..."
1616
+
1617
+ # Temporarily override signal file to scope verifier to this US
1618
+ local orig_signal
1619
+ orig_signal=$(cat "$SIGNAL_FILE" 2>/dev/null)
1620
+ echo "{\"status\":\"verify\",\"us_id\":\"$us\",\"summary\":\"sequential final verify\"}" | atomic_write "$SIGNAL_FILE"
1621
+
1622
+ # Write scoped verifier trigger
1623
+ write_verifier_trigger "$iter"
1624
+ local verifier_prompt="$LOGS_DIR/iter-$(printf '%03d' $iter).verifier-prompt.md"
1625
+
1626
+ # Clean verifier pane
1627
+ local verifier_cmd
1628
+ verifier_cmd=$(tmux display-message -p -t "$VERIFIER_PANE" '#{pane_current_command}' 2>/dev/null)
1629
+ if [[ "$verifier_cmd" == "node" || "$verifier_cmd" == "claude" || "$verifier_cmd" == "codex" ]]; then
1630
+ tmux send-keys -t "$VERIFIER_PANE" C-c 2>/dev/null; sleep 0.5
1631
+ tmux send-keys -t "$VERIFIER_PANE" "/exit" Enter 2>/dev/null; sleep 2
1632
+ fi
1633
+ wait_for_pane_ready "$VERIFIER_PANE" 10 2>/dev/null || true
1634
+
1635
+ # Launch verifier
1636
+ local verifier_launch
1637
+ if [[ "$VERIFIER_ENGINE" = "codex" ]]; then
1638
+ verifier_launch="${CODEX_BIN:-codex} -m $VERIFIER_CODEX_MODEL -c model_reasoning_effort=\"$VERIFIER_CODEX_REASONING\" --dangerously-bypass-approvals-and-sandbox"
1639
+ launch_verifier_codex "$VERIFIER_PANE" "$verifier_prompt" "$iter" "$verifier_launch"
1640
+ else
1641
+ verifier_launch="$CLAUDE_BIN --model $VERIFIER_MODEL --dangerously-skip-permissions"
1642
+ launch_verifier_claude "$VERIFIER_PANE" "$verifier_prompt" "$iter" "$verifier_launch" || {
1643
+ log_error "Failed to launch verifier for $us"
1644
+ FAILED_US="$us"
1645
+ return 1
1646
+ }
1647
+ fi
1648
+
1649
+ # Poll for verdict
1650
+ rm -f "$VERDICT_FILE"
1651
+ local poll_rc=0
1652
+ poll_for_signal "$VERDICT_FILE" "$ITER_TIMEOUT" "verdict" || poll_rc=$?
1653
+ if (( poll_rc != 0 )); then
1654
+ log_error "Verifier poll failed for $us (rc=$poll_rc)"
1655
+ FAILED_US="$us"
1656
+ return 1
1657
+ fi
1658
+
1659
+ # Check verdict
1660
+ local verdict
1661
+ verdict=$(jq -r '.verdict' "$VERDICT_FILE" 2>/dev/null)
1662
+ if [[ "$verdict" != "pass" ]]; then
1663
+ FAILED_US="$us"
1664
+ log " Sequential final verify FAILED at $us"
1665
+ log_debug "[FLOW] iter=$iter phase=sequential_final_verify failed_us=$us verdict=$verdict"
1666
+ return 1
1667
+ fi
1668
+ log " Sequential final verify: $us PASSED"
1669
+
1670
+ # Archive per-US final verdict
1671
+ cp "$VERDICT_FILE" "$LOGS_DIR/iter-$(printf '%03d' $iter).final-verdict-${us}.json" 2>/dev/null
1672
+ done
1673
+
1674
+ # Integration check: run tests if VERIFICATION_CMD is set
1675
+ if [[ -n "${VERIFICATION_CMD:-}" ]]; then
1676
+ log " Running integration test suite after sequential verify..."
1677
+ log_debug "[FLOW] iter=$iter phase=integration_check cmd=$VERIFICATION_CMD"
1678
+ if ! eval "$VERIFICATION_CMD" > /dev/null 2>&1; then
1679
+ log " Integration test suite FAILED"
1680
+ FAILED_US="integration"
1681
+ return 2
1682
+ fi
1683
+ log " Integration test suite PASSED"
1684
+ fi
1685
+
1686
+ log " Sequential final verify: ALL PASSED"
1687
+ return 0
1688
+ }
1689
+
1690
+ # --- US-005: Determine whether consensus verification should run for this signal ---
1691
+ # Returns 0 (use consensus) or 1 (single engine).
1692
+ # VERIFY_CONSENSUS + CONSENSUS_SCOPE handles per-US consensus.
1693
+ # FINAL_CONSENSUS independently enables consensus for the final ALL verify only.
1694
+ _should_use_consensus() {
1695
+ local signal_us_id="${1:-}"
1696
+ if [[ "$VERIFY_CONSENSUS" = "1" ]]; then
1697
+ case "$CONSENSUS_SCOPE" in
1698
+ all) return 0 ;;
1699
+ final-only) [[ "$signal_us_id" == "ALL" ]] && return 0 ;;
1700
+ esac
1701
+ fi
1702
+ if [[ "$FINAL_CONSENSUS" = "1" && "$signal_us_id" == "ALL" ]]; then
1703
+ return 0
1704
+ fi
1705
+ return 1
1706
+ }
1707
+
1595
1708
  # --- US-004: Run consensus verification (claude + codex sequentially) ---
1596
1709
  run_consensus_verification() {
1597
1710
  local iter="$1"
@@ -1607,18 +1720,45 @@ run_consensus_verification() {
1607
1720
  log " Consensus round $CONSENSUS_ROUND/6..."
1608
1721
 
1609
1722
  # Run claude verifier first
1723
+ local _claude_t0=$(date +%s)
1610
1724
  if ! run_single_verifier "$iter" "claude" "$VERIFIER_MODEL" "-claude" "$claude_verdict_file"; then
1611
1725
  log_error "Claude verifier failed in consensus round $CONSENSUS_ROUND"
1612
1726
  return 1
1613
1727
  fi
1728
+ ITER_VERIFIER_CLAUDE_DURATION_S=$(( $(date +%s) - _claude_t0 ))
1614
1729
  CLAUDE_VERDICT=$(jq -r '.verdict' "$claude_verdict_file" 2>/dev/null)
1730
+ # A12 fix: validate claude verdict is not null/empty — if so, retry once before proceeding
1731
+ if [[ -z "$CLAUDE_VERDICT" || "$CLAUDE_VERDICT" == "null" ]]; then
1732
+ log " WARNING: Claude verdict is '$CLAUDE_VERDICT' — likely interrupted. Retrying claude verifier..."
1733
+ log_debug "[GOV] iter=$iter phase=consensus_claude_retry reason=null_verdict"
1734
+ rm -f "$claude_verdict_file" 2>/dev/null
1735
+ if ! run_single_verifier "$iter" "claude" "$VERIFIER_MODEL" "-claude" "$claude_verdict_file"; then
1736
+ log_error "Claude verifier retry also failed"
1737
+ return 1
1738
+ fi
1739
+ CLAUDE_VERDICT=$(jq -r '.verdict' "$claude_verdict_file" 2>/dev/null)
1740
+ if [[ -z "$CLAUDE_VERDICT" || "$CLAUDE_VERDICT" == "null" ]]; then
1741
+ log_error "Claude verdict still null after retry — consensus cannot proceed"
1742
+ return 1
1743
+ fi
1744
+ fi
1615
1745
  log_debug "[GOV] iter=$iter phase=consensus_claude verdict=$CLAUDE_VERDICT model=$VERIFIER_MODEL"
1616
1746
 
1747
+ # F8: --consensus-fail-fast — skip second verifier if first fails
1748
+ if [[ "$CONSENSUS_FAIL_FAST" = "1" && "$CLAUDE_VERDICT" = "fail" ]]; then
1749
+ log " Consensus fail-fast: claude=fail, skipping codex verifier"
1750
+ log_debug "[GOV] iter=$iter phase=consensus_fail_fast claude=fail codex=skipped"
1751
+ CODEX_VERDICT="skipped"
1752
+ return 2 # disagreement/fail signal
1753
+ fi
1754
+
1617
1755
  # Run codex verifier second
1756
+ local _codex_t0=$(date +%s)
1618
1757
  if ! run_single_verifier "$iter" "codex" "$VERIFIER_CODEX_MODEL" "-codex" "$codex_verdict_file"; then
1619
1758
  log_error "Codex verifier failed in consensus round $CONSENSUS_ROUND"
1620
1759
  return 1
1621
1760
  fi
1761
+ ITER_VERIFIER_CODEX_DURATION_S=$(( $(date +%s) - _codex_t0 ))
1622
1762
  CODEX_VERDICT=$(jq -r '.verdict' "$codex_verdict_file" 2>/dev/null)
1623
1763
  log_debug "[GOV] iter=$iter phase=consensus_codex verdict=$CODEX_VERDICT model=$VERIFIER_CODEX_MODEL reasoning=$VERIFIER_CODEX_REASONING"
1624
1764
 
@@ -1722,43 +1862,37 @@ run_consensus_verification() {
1722
1862
  return 1
1723
1863
  }
1724
1864
 
1725
- # =============================================================================
1726
- # Security Warning
1727
- # =============================================================================
1728
-
1729
- print_security_warning() {
1730
- echo ""
1731
- echo "================================================================"
1732
- echo " WARNING: Running with --dangerously-skip-permissions"
1733
- echo ""
1734
- echo " The claude CLI will execute tools (file writes, shell commands)"
1735
- echo " without asking for confirmation. Only run this on code you"
1736
- echo " trust in an environment you control."
1737
- echo "================================================================"
1738
- echo ""
1739
- }
1740
-
1741
1865
  # =============================================================================
1742
1866
  # Main Leader Loop
1743
1867
  # =============================================================================
1744
1868
 
1745
1869
  main() {
1746
1870
  # --- Lockfile: prevent duplicate execution ---
1747
- local lockfile="$DESK/logs/.rlp-desk-$SLUG.lock"
1871
+ local lockfile="$LOCKFILE_PATH"
1748
1872
  mkdir -p "$(dirname "$lockfile")" 2>/dev/null
1749
1873
  if ! (set -C; echo $$ > "$lockfile") 2>/dev/null; then
1750
1874
  local lock_pid
1751
1875
  lock_pid=$(cat "$lockfile" 2>/dev/null)
1752
1876
  if kill -0 "$lock_pid" 2>/dev/null; then
1753
- log_error "Another instance is already running (PID $lock_pid)"
1877
+ log_error "Another instance is already running (PID $lock_pid). Kill $lock_pid or rm $lockfile"
1754
1878
  exit 1
1755
1879
  fi
1756
1880
  # Stale lock — overwrite
1881
+ log "Stale lock detected (PID ${lock_pid:-unknown} not running), recovering"
1757
1882
  echo $$ > "$lockfile"
1883
+ LOCKFILE_ACQUIRED=1
1884
+ else
1885
+ LOCKFILE_ACQUIRED=1
1758
1886
  fi
1759
- mkdir -p "$LOGS_DIR" 2>/dev/null
1887
+ trap cleanup EXIT INT TERM
1888
+ mkdir -p "$LOGS_DIR" "$RUNTIME_DIR" 2>/dev/null
1760
1889
 
1761
- # --- AC7: debug.log versioning: rename existing debug.log before fresh run ---
1890
+ # --- Analytics directory: create only when --debug or --with-self-verification ---
1891
+ if (( DEBUG )) || (( WITH_SELF_VERIFICATION )); then
1892
+ mkdir -p "$ANALYTICS_DIR" 2>/dev/null
1893
+ fi
1894
+
1895
+ # --- debug.log versioning (in analytics dir) ---
1762
1896
  if (( DEBUG )) && [[ -f "$DEBUG_LOG" ]]; then
1763
1897
  local dbg_n=1
1764
1898
  while [[ -f "${DEBUG_LOG%.log}-v${dbg_n}.log" ]]; do
@@ -1767,6 +1901,34 @@ main() {
1767
1901
  mv "$DEBUG_LOG" "${DEBUG_LOG%.log}-v${dbg_n}.log"
1768
1902
  fi
1769
1903
 
1904
+ # --- campaign.jsonl versioning (in analytics dir, after mkdir) ---
1905
+ if (( DEBUG )) || (( WITH_SELF_VERIFICATION )); then
1906
+ if [[ -f "$CAMPAIGN_JSONL" ]]; then
1907
+ local cj_n=1
1908
+ while [[ -f "${CAMPAIGN_JSONL%.jsonl}-v${cj_n}.jsonl" ]]; do
1909
+ (( cj_n++ ))
1910
+ done
1911
+ mv "$CAMPAIGN_JSONL" "${CAMPAIGN_JSONL%.jsonl}-v${cj_n}.jsonl"
1912
+ fi
1913
+ fi
1914
+
1915
+ # --- metadata.json: write at campaign start ---
1916
+ if (( DEBUG )) || (( WITH_SELF_VERIFICATION )); then
1917
+ jq -n \
1918
+ --arg slug "$SLUG" \
1919
+ --arg project_root "$ROOT" \
1920
+ --arg campaign_status "running" \
1921
+ --arg start_time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
1922
+ --arg end_time "" \
1923
+ --arg worker_model "$WORKER_MODEL" \
1924
+ --arg verifier_model "$VERIFIER_MODEL" \
1925
+ --argjson debug "$DEBUG" \
1926
+ --argjson with_sv "$WITH_SELF_VERIFICATION" \
1927
+ --argjson consensus "$VERIFY_CONSENSUS" \
1928
+ '{slug: $slug, project_root: $project_root, campaign_status: $campaign_status, start_time: $start_time, end_time: $end_time, worker_model: $worker_model, verifier_model: $verifier_model, debug: $debug, with_self_verification: $with_sv, consensus: $consensus}' \
1929
+ > "$METADATA_FILE"
1930
+ fi
1931
+
1770
1932
  # --- Startup ---
1771
1933
  log "Ralph Desk Tmux Runner starting..."
1772
1934
  log " Slug: $SLUG"
@@ -1776,6 +1938,7 @@ main() {
1776
1938
  log " Verifier model: $VERIFIER_MODEL"
1777
1939
  log " Verify mode: $VERIFY_MODE"
1778
1940
  log " Verify consensus:$VERIFY_CONSENSUS"
1941
+ log " Final consensus: $FINAL_CONSENSUS"
1779
1942
  log " Consensus scope: $CONSENSUS_SCOPE"
1780
1943
  log " Poll interval: ${POLL_INTERVAL}s"
1781
1944
  log " Iter timeout: ${ITER_TIMEOUT}s"
@@ -1819,9 +1982,9 @@ main() {
1819
1982
  US_LIST=$(grep -oE 'US-[0-9]+' "$prd_file" | sort -u | tr '\n' ',' | sed 's/,$//')
1820
1983
  fi
1821
1984
 
1822
- # Initialize VERIFIED_US from memory's Completed Stories (carry over previous runs)
1823
- local memory_file="$DESK/memos/${SLUG}-memory.md"
1824
- if [[ -f "$memory_file" ]]; then
1985
+ # Initialize VERIFIED_US from memory's Completed Stories (carry over previous runs)
1986
+ local memory_file="$DESK/memos/${SLUG}-memory.md"
1987
+ if [[ -f "$memory_file" ]]; then
1825
1988
  local completed_us
1826
1989
  completed_us=$(sed -n '/^## Completed Stories$/,/^## /p' "$memory_file" 2>/dev/null | grep '^- US-' | sed 's/^- \(US-[0-9]*\):.*/\1/' | sort -u | tr '\n' ',' | sed 's/,$//')
1827
1990
  if [[ -n "$completed_us" ]]; then
@@ -1830,8 +1993,23 @@ main() {
1830
1993
  log_debug "[FLOW] loaded_verified_us_from_memory=$VERIFIED_US"
1831
1994
  fi
1832
1995
  fi
1996
+
1997
+ # D1: Fallback — restore verified_us from status.json if memory had none
1998
+ if [[ -z "$VERIFIED_US" && -f "$STATUS_FILE" ]]; then
1999
+ local status_verified
2000
+ status_verified=$(jq -r '.verified_us // [] | join(",")' "$STATUS_FILE" 2>/dev/null)
2001
+ if [[ -n "$status_verified" ]]; then
2002
+ VERIFIED_US="$status_verified"
2003
+ log " Restored verified_us from status.json: $VERIFIED_US"
2004
+ log_debug "[FLOW] restored_verified_us_from_status=$VERIFIED_US"
2005
+ fi
2006
+ fi
1833
2007
  fi
1834
2008
 
2009
+ # Initialize PRD snapshot state for live update detection
2010
+ PREV_PRD_HASH=$(compute_prd_hash)
2011
+ PREV_PRD_US_LIST=$(count_prd_us)
2012
+
1835
2013
  # Dependency checks
1836
2014
  check_dependencies
1837
2015
 
@@ -1854,7 +2032,7 @@ main() {
1854
2032
  PREV_CONTEXT_HASH=$(compute_context_hash)
1855
2033
 
1856
2034
  # --- governance.md s7: Leader Loop ---
1857
- local HARD_CEILING=$(( ITER_TIMEOUT * 3 )) # absolute max per iteration (no extensions beyond this)
2035
+ local HARD_CEILING=$(( ITER_TIMEOUT * 3 )) # logged but NOT enforced Worker extends indefinitely when active
1858
2036
 
1859
2037
  for (( ITERATION = 1; ITERATION <= MAX_ITER; ITERATION++ )); do
1860
2038
  log ""
@@ -1896,96 +2074,66 @@ main() {
1896
2074
  # Reset per-iteration state
1897
2075
  local worker_nudge_count=0
1898
2076
  local verifier_nudge_count=0
2077
+ ITER_VERIFIER_START=""
2078
+ ITER_VERIFIER_END=""
2079
+
2080
+ # --- US-004: detect PRD changes for live update + re-split ---
2081
+ check_prd_update
1899
2082
 
1900
2083
  # --- governance.md s7 step 4: Build worker prompt + trigger ---
1901
2084
  write_worker_trigger "$ITERATION"
1902
2085
  local worker_prompt="$LOGS_DIR/iter-$(printf '%03d' $ITERATION).worker-prompt.md"
1903
2086
 
2087
+ # AC1: capture worker start timestamp
2088
+ ITER_WORKER_START=$(date +%s)
2089
+
1904
2090
  update_status "worker" "running"
1905
2091
 
1906
- # --- governance.md s7 step 5: Execute Worker (interactive TUI, tmux pattern) ---
1907
- # Step 5a: Launch interactive worker engine in Worker pane
2092
+ # --- governance.md s7 step 5: Execute Worker (dispatched to engine-specific function) ---
2093
+ log_debug "[FLOW] iter=$ITERATION phase=worker engine=$WORKER_ENGINE model=$WORKER_MODEL dispatched=true"
2094
+
1908
2095
  local worker_launch
1909
2096
  if [[ "$WORKER_ENGINE" = "codex" ]]; then
1910
- worker_launch="${CODEX_BIN:-codex} -m $WORKER_CODEX_MODEL -c model_reasoning_effort=\"$WORKER_CODEX_REASONING\" --dangerously-bypass-approvals-and-sandbox"
1911
- log " Launching Worker codex in pane $WORKER_PANE..."
2097
+ local worker_trigger="$LOGS_DIR/iter-$(printf '%03d' $ITERATION).worker-trigger.sh"
2098
+ worker_launch="bash $worker_trigger"
2099
+ launch_worker_codex "$WORKER_PANE" "$worker_trigger" "$ITERATION"
1912
2100
  else
1913
2101
  worker_launch="$CLAUDE_BIN --model $WORKER_MODEL --dangerously-skip-permissions"
1914
- log " Launching Worker claude in pane $WORKER_PANE..."
1915
- fi
1916
- tmux send-keys -t "$WORKER_PANE" -l -- "$worker_launch"
1917
- tmux send-keys -t "$WORKER_PANE" Enter
1918
- log_debug "[FLOW] iter=$ITERATION phase=worker engine=$WORKER_ENGINE model=$WORKER_MODEL dispatched=true"
1919
-
1920
- # Step 5b: Wait for claude TUI to be ready (tmux pattern)
1921
- if ! wait_for_pane_ready "$WORKER_PANE" 30; then
1922
- log_error "Worker claude failed to start"
1923
- write_blocked_sentinel "Worker claude failed to start in pane"
1924
- update_status "blocked" "worker_start_failed"
1925
- return 1
1926
- fi
1927
-
1928
- # Step 5c: Wait for claude to fully initialize, then send instruction directly
1929
- sleep 3
1930
- local worker_instruction="Read and execute the instructions in $worker_prompt"
1931
- tmux send-keys -t "$WORKER_PANE" -l -- "$worker_instruction"
1932
- tmux send-keys -t "$WORKER_PANE" Enter
1933
- log_debug "Worker instruction sent directly (${#worker_instruction} chars)"
1934
-
1935
- # Verify claude actually started working — keep sending C-m until activity detected
1936
- local submit_attempts=0
1937
- while (( submit_attempts < 15 )); do
1938
- sleep 2
1939
- local pane_check
1940
- pane_check=$(tmux capture-pane -t "$WORKER_PANE" -p 2>/dev/null)
1941
- if echo "$pane_check" | grep -qi "esc to interrupt\|thinking\|working\|kneading\|crunching\|clauding\|billowing\|brewing\|tinkering\|burrowing\|saut\|Exploring\|Running\|exec\|Explored" 2>/dev/null; then
1942
- log_debug "Worker started working after $((submit_attempts + 1)) submit checks"
1943
- log_debug "[FLOW] iter=$ITERATION worker_submit_check=OK attempts=$((submit_attempts + 1))"
1944
- break
1945
- fi
1946
- # After 8 failed attempts, try C-u clear + re-type (omc-teams adaptive retry)
1947
- if (( submit_attempts == 8 )); then
1948
- log_debug "Adaptive instruction retry: clearing line and re-typing"
1949
- tmux send-keys -t "$WORKER_PANE" C-u 2>/dev/null
1950
- sleep 0.1
1951
- tmux send-keys -t "$WORKER_PANE" -l -- "$worker_instruction"
1952
- tmux send-keys -t "$WORKER_PANE" Enter
2102
+ if ! launch_worker_claude "$WORKER_PANE" "$worker_prompt" "$ITERATION" "$worker_launch"; then
2103
+ write_blocked_sentinel "Worker claude failed to start in pane"
2104
+ update_status "blocked" "worker_start_failed"
2105
+ return 1
1953
2106
  fi
1954
- tmux send-keys -t "$WORKER_PANE" C-m 2>/dev/null
1955
- sleep 0.3
1956
- tmux send-keys -t "$WORKER_PANE" C-m 2>/dev/null
1957
- (( submit_attempts++ ))
1958
- done
1959
- if (( submit_attempts >= 15 )); then
1960
- log " WARNING: Could not confirm Worker started working after 15 attempts"
1961
- log_debug "[FLOW] iter=$ITERATION worker_submit_check=FAILED attempts=15"
1962
2107
  fi
1963
2108
 
1964
2109
  # --- governance.md s7 step 5+6: Poll for Worker completion ---
1965
2110
  log " Polling for iter-signal.json..."
1966
2111
  local worker_poll_done=0
1967
2112
  while (( ! worker_poll_done )); do
2113
+ local worker_poll_rc=0
1968
2114
  if poll_for_signal "$SIGNAL_FILE" "$WORKER_HEARTBEAT" "$WORKER_PANE" "$worker_launch" "Worker"; then
1969
2115
  worker_poll_done=1
1970
2116
  log_debug "[FLOW] iter=$ITERATION poll_signal_received=true"
1971
2117
  else
2118
+ worker_poll_rc=$?
2119
+ if (( worker_poll_rc == 2 )); then
2120
+ return 1
2121
+ fi
1972
2122
  # Check if Worker is still actively running (not stuck)
1973
2123
  local worker_cmd
1974
2124
  worker_cmd=$(tmux display-message -p -t "$WORKER_PANE" '#{pane_current_command}' 2>/dev/null)
1975
2125
  if [[ "$worker_cmd" == "node" || "$worker_cmd" == "claude" || "$worker_cmd" == "codex" ]]; then
1976
- # Check hard ceiling before extending
2126
+ # Process alive — extend indefinitely (no hard ceiling kill)
2127
+ # Stale-context breaker and nudge system handle truly stuck workers
1977
2128
  local iter_elapsed=$(( $(date +%s) - ITER_START_TIME ))
2129
+ local ceiling_exceeded=""
1978
2130
  if (( iter_elapsed >= HARD_CEILING )); then
1979
- log_error "Worker hit hard ceiling (${HARD_CEILING}s = 3x iter_timeout). Killing iteration."
1980
- log_debug "[GOV] iter=$ITERATION hard_ceiling_hit=true elapsed=${iter_elapsed}s ceiling=${HARD_CEILING}s process=$worker_cmd"
1981
- tmux send-keys -t "$WORKER_PANE" C-c 2>/dev/null
1982
- sleep 1
1983
- write_blocked_sentinel "Worker hit hard ceiling (${HARD_CEILING}s). Pane preserved for inspection."
1984
- update_status "blocked" "hard_timeout"
1985
- return 1
2131
+ ceiling_exceeded=" [EXCEEDED hard_ceiling=${HARD_CEILING}s not enforced, logged only]"
2132
+ log " WARNING: Worker exceeded soft hard-ceiling (${iter_elapsed}s >= ${HARD_CEILING}s) but still active. Continuing..."
2133
+ log_debug "[GOV] iter=$ITERATION hard_ceiling_exceeded=true elapsed=${iter_elapsed}s ceiling=${HARD_CEILING}s process=$worker_cmd action=log_only_no_kill"
1986
2134
  fi
1987
- log " Worker timed out but still active ($worker_cmd). Extending poll... (${iter_elapsed}s/${HARD_CEILING}s)"
1988
- log_debug "[GOV] iter=$ITERATION timeout_active=true process=$worker_cmd elapsed=${iter_elapsed}s ceiling=${HARD_CEILING}s"
2135
+ log " Worker timed out but still active ($worker_cmd). Extending poll... (${iter_elapsed}s, no ceiling)${ceiling_exceeded}"
2136
+ log_debug "[GOV] iter=$ITERATION timeout_active=true process=$worker_cmd elapsed=${iter_elapsed}s action=extend_indefinitely"
1989
2137
  log_debug "[FLOW] iter=$ITERATION poll_extended=true worker_cmd=$worker_cmd"
1990
2138
  update_status "worker" "slow"
1991
2139
  # Loop continues — re-poll same iteration
@@ -2019,6 +2167,11 @@ main() {
2019
2167
  # Reset monitor failure count on success
2020
2168
  MONITOR_FAILURE_COUNT=0
2021
2169
 
2170
+ # AC1: capture worker end timestamp; reset consensus timing
2171
+ ITER_WORKER_END=$(date +%s)
2172
+ ITER_VERIFIER_CLAUDE_DURATION_S=""
2173
+ ITER_VERIFIER_CODEX_DURATION_S=""
2174
+
2022
2175
  # --- governance.md s7 step 6: Read iter-signal.json via jq (JSON only, no markdown) ---
2023
2176
  local signal_status
2024
2177
  signal_status=$(jq -r '.status' "$SIGNAL_FILE" 2>/dev/null)
@@ -2045,17 +2198,34 @@ main() {
2045
2198
  signal_us_id=$(jq -r '.us_id // empty' "$SIGNAL_FILE" 2>/dev/null)
2046
2199
  log " Worker claims done (us_id=${signal_us_id:-all}). Dispatching Verifier..."
2047
2200
 
2201
+ # AC1: capture verifier start timestamp
2202
+ ITER_VERIFIER_START=$(date +%s)
2203
+
2048
2204
  update_status "verifier" "running"
2049
2205
 
2050
- # --- Consensus scope check ---
2051
- local use_consensus=0
2052
- if [[ "$VERIFY_CONSENSUS" = "1" ]]; then
2053
- case "$CONSENSUS_SCOPE" in
2054
- all) use_consensus=1 ;;
2055
- final-only) [[ "$signal_us_id" == "ALL" ]] && use_consensus=1 ;;
2056
- esac
2206
+ # --- Sequential final verify: per-US scoped checks instead of one big ALL verify ---
2207
+ if [[ "$signal_us_id" == "ALL" && "$VERIFY_MODE" == "per-us" && -n "$US_LIST" ]]; then
2208
+ log " Final ALL verify: using sequential per-US strategy (timeout prevention)"
2209
+ local seq_rc=0
2210
+ run_sequential_final_verify "$ITERATION" || seq_rc=$?
2211
+ if (( seq_rc == 0 )); then
2212
+ write_complete_sentinel "Sequential final verify passed (all US verified individually)"
2213
+ update_status "complete" "pass"
2214
+ write_campaign_jsonl "$ITERATION" "ALL" "pass"
2215
+ return 0
2216
+ else
2217
+ # Sequential verify failed — fall through to fix loop with failed US
2218
+ log " Sequential final verify failed at ${FAILED_US:-unknown}. Entering fix loop."
2219
+ signal_us_id="${FAILED_US:-ALL}"
2220
+ # Synthesize a fail verdict for the fix loop
2221
+ echo "{\"verdict\":\"fail\",\"summary\":\"Sequential final verify failed at ${FAILED_US:-unknown}\",\"issues\":[{\"severity\":\"critical\",\"criterion\":\"${FAILED_US:-ALL}\",\"description\":\"Failed during sequential final verification\"}]}" | atomic_write "$VERDICT_FILE"
2222
+ fi
2057
2223
  fi
2058
2224
 
2225
+ # --- Consensus scope check (US-005: _should_use_consensus handles VERIFY_CONSENSUS + FINAL_CONSENSUS) ---
2226
+ local use_consensus=0
2227
+ _should_use_consensus "$signal_us_id" && use_consensus=1
2228
+
2059
2229
  # --- Consensus vs single verification ---
2060
2230
  if (( use_consensus )); then
2061
2231
  # US-004: Run consensus verification (claude + codex sequentially)
@@ -2077,70 +2247,54 @@ main() {
2077
2247
  write_verifier_trigger "$ITERATION"
2078
2248
  local verifier_prompt="$LOGS_DIR/iter-$(printf '%03d' $ITERATION).verifier-prompt.md"
2079
2249
 
2080
- # Step 7a: Clean previous Verifier session if running
2250
+ # Step 7a: Clean previous Verifier session (with dead pane detection)
2081
2251
  local verifier_cmd
2082
2252
  verifier_cmd=$(tmux display-message -p -t "$VERIFIER_PANE" '#{pane_current_command}' 2>/dev/null)
2083
- if [[ "$verifier_cmd" == "node" || "$verifier_cmd" == "claude" || "$verifier_cmd" == "codex" ]]; then
2253
+ if [[ -z "$verifier_cmd" ]]; then
2254
+ log " Verifier pane $VERIFIER_PANE is gone — replacing..."
2255
+ log_debug "[GOV] iter=$ITERATION pane_dead=true pane_id=$VERIFIER_PANE action=replace_pane"
2256
+ replace_worker_pane "$VERIFIER_PANE" "verifier"
2257
+ VERIFIER_PANE=$(jq -r '.panes.verifier' "$SESSION_CONFIG")
2258
+ log " New verifier pane: $VERIFIER_PANE"
2259
+ elif [[ "$verifier_cmd" == "zsh" || "$verifier_cmd" == "bash" ]]; then
2260
+ log " Verifier pane $VERIFIER_PANE has bare shell ($verifier_cmd) — resetting..."
2261
+ log_debug "[GOV] iter=$ITERATION pane_dead=true pane_id=$VERIFIER_PANE cmd=$verifier_cmd action=reset_shell"
2262
+ tmux send-keys -t "$VERIFIER_PANE" C-c C-u 2>/dev/null
2263
+ sleep 0.2
2264
+ tmux send-keys -t "$VERIFIER_PANE" "clear" Enter 2>/dev/null
2265
+ sleep 0.3
2266
+ elif [[ "$verifier_cmd" == "node" || "$verifier_cmd" == "claude" || "$verifier_cmd" == "codex" ]]; then
2084
2267
  tmux send-keys -t "$VERIFIER_PANE" C-c 2>/dev/null
2085
2268
  sleep 0.5
2086
2269
  tmux send-keys -t "$VERIFIER_PANE" "/exit" Enter 2>/dev/null
2087
2270
  sleep 2
2088
- wait_for_pane_ready "$VERIFIER_PANE" 5 2>/dev/null || true
2089
2271
  fi
2272
+ wait_for_pane_ready "$VERIFIER_PANE" 10 2>/dev/null || true
2090
2273
 
2091
2274
  local verifier_launch
2092
2275
  if [[ "$VERIFIER_ENGINE" = "codex" ]]; then
2093
2276
  verifier_launch="${CODEX_BIN:-codex} -m $VERIFIER_CODEX_MODEL -c model_reasoning_effort=\"$VERIFIER_CODEX_REASONING\" --dangerously-bypass-approvals-and-sandbox"
2094
- log " Launching Verifier codex in pane $VERIFIER_PANE..."
2095
2277
  else
2096
2278
  verifier_launch="$CLAUDE_BIN --model $VERIFIER_MODEL --dangerously-skip-permissions"
2097
- log " Launching Verifier claude in pane $VERIFIER_PANE..."
2098
2279
  fi
2099
- tmux send-keys -t "$VERIFIER_PANE" -l -- "$verifier_launch"
2100
- tmux send-keys -t "$VERIFIER_PANE" Enter
2101
2280
  log_debug "[FLOW] iter=$ITERATION phase=verifier engine=$VERIFIER_ENGINE model=$VERIFIER_MODEL scope=${signal_us_id:-all} dispatched=true"
2102
2281
 
2103
- # Step 7b: Wait for TUI to be ready
2104
- if ! wait_for_pane_ready "$VERIFIER_PANE" 30; then
2105
- log_error "Verifier failed to start"
2106
- update_status "verifier" "start_failed"
2107
- continue
2108
- fi
2109
-
2110
- # Step 7c: Send instruction
2111
- sleep 3
2112
- local verifier_instruction="Read and execute the instructions in $verifier_prompt"
2113
- tmux send-keys -t "$VERIFIER_PANE" -l -- "$verifier_instruction"
2114
- tmux send-keys -t "$VERIFIER_PANE" Enter
2115
- log_debug "Verifier instruction sent directly"
2116
-
2117
- # Verify verifier actually started working
2118
- local vs_submit=0
2119
- while (( vs_submit < 15 )); do
2120
- sleep 2
2121
- local vs_check
2122
- vs_check=$(tmux capture-pane -t "$VERIFIER_PANE" -p 2>/dev/null)
2123
- if echo "$vs_check" | grep -qi "esc to interrupt\|thinking\|working\|kneading\|crunching\|clauding\|billowing\|brewing\|tinkering\|burrowing\|saut\|Exploring\|Running\|exec\|Explored" 2>/dev/null; then
2124
- log_debug "Verifier started working after $((vs_submit + 1)) checks"
2125
- break
2126
- fi
2127
- # After 8 failed attempts, try C-u clear + re-type (omc-teams adaptive retry)
2128
- if (( vs_submit == 8 )); then
2129
- log_debug "Adaptive instruction retry: clearing line and re-typing"
2130
- tmux send-keys -t "$VERIFIER_PANE" C-u 2>/dev/null
2131
- sleep 0.1
2132
- tmux send-keys -t "$VERIFIER_PANE" -l -- "$verifier_instruction"
2133
- tmux send-keys -t "$VERIFIER_PANE" Enter
2282
+ if [[ "$VERIFIER_ENGINE" = "codex" ]]; then
2283
+ launch_verifier_codex "$VERIFIER_PANE" "$verifier_prompt" "$ITERATION" "$verifier_launch"
2284
+ else
2285
+ if ! launch_verifier_claude "$VERIFIER_PANE" "$verifier_prompt" "$ITERATION" "$verifier_launch"; then
2286
+ update_status "verifier" "start_failed"
2287
+ continue
2134
2288
  fi
2135
- tmux send-keys -t "$VERIFIER_PANE" C-m 2>/dev/null
2136
- sleep 0.3
2137
- tmux send-keys -t "$VERIFIER_PANE" C-m 2>/dev/null
2138
- (( vs_submit++ ))
2139
- done
2289
+ fi
2140
2290
 
2141
2291
  # Poll for verify-verdict.json
2142
2292
  log " Polling for verify-verdict.json..."
2143
2293
  if ! poll_for_signal "$VERDICT_FILE" "$VERIFIER_HEARTBEAT" "$VERIFIER_PANE" "$verifier_launch" "Verifier"; then
2294
+ local verifier_poll_rc=$?
2295
+ if (( verifier_poll_rc == 2 )); then
2296
+ return 1
2297
+ fi
2144
2298
  log_error "Verifier poll failed"
2145
2299
  # Verifier is dead/stuck — BLOCK and let user decide
2146
2300
  write_blocked_sentinel "Verifier process dead/stuck (poll failed). Pane preserved for inspection."
@@ -2149,6 +2303,9 @@ main() {
2149
2303
  fi
2150
2304
  fi
2151
2305
 
2306
+ # AC1: capture verifier end timestamp
2307
+ ITER_VERIFIER_END=$(date +%s)
2308
+
2152
2309
  # --- governance.md s7 step 7: Read verdict via jq ---
2153
2310
  local verdict
2154
2311
  verdict=$(jq -r '.verdict' "$VERDICT_FILE" 2>/dev/null)
@@ -2166,6 +2323,18 @@ main() {
2166
2323
  pass)
2167
2324
  CONSECUTIVE_FAILURES=0
2168
2325
  CONSENSUS_ROUND=0
2326
+ _SAME_US_FAIL_COUNT=0
2327
+ _LAST_FAILED_US=""
2328
+ if (( _MODEL_UPGRADED )); then
2329
+ log " Worker model restored: ${WORKER_MODEL} → ${_ORIGINAL_WORKER_MODEL} (pass verdict)"
2330
+ log_debug "[DECIDE] iter=$ITERATION phase=model_select model_restore=true from=${WORKER_MODEL} to=${_ORIGINAL_WORKER_MODEL}"
2331
+ WORKER_MODEL="$_ORIGINAL_WORKER_MODEL"
2332
+ if [[ "$WORKER_ENGINE" = "codex" ]]; then
2333
+ WORKER_CODEX_MODEL="$WORKER_MODEL"
2334
+ WORKER_CODEX_REASONING="$_ORIGINAL_WORKER_CODEX_REASONING"
2335
+ fi
2336
+ _MODEL_UPGRADED=0
2337
+ fi
2169
2338
 
2170
2339
  # --- Per-US tracking ---
2171
2340
  if [[ "$VERIFY_MODE" = "per-us" && -n "$signal_us_id" && "$signal_us_id" != "ALL" ]]; then
@@ -2183,6 +2352,7 @@ main() {
2183
2352
  # Final full verify passed or complete recommended
2184
2353
  write_complete_sentinel "$verdict_summary"
2185
2354
  update_status "complete" "pass"
2355
+ write_campaign_jsonl "$ITERATION" "${signal_us_id:-ALL}" "pass"
2186
2356
  return 0
2187
2357
  else
2188
2358
  log " Verifier passed but did not recommend complete. Continuing."
@@ -2192,6 +2362,7 @@ main() {
2192
2362
  fail)
2193
2363
  # --- governance.md s7½: Fix Loop (adapted for tmux lean mode) ---
2194
2364
  (( CONSECUTIVE_FAILURES++ ))
2365
+ check_model_upgrade "${signal_us_id:-unknown}"
2195
2366
  local verdict_summary_fail
2196
2367
  verdict_summary_fail=$(jq -r '.summary // "no summary"' "$VERDICT_FILE" 2>/dev/null)
2197
2368
  log " Verifier FAILED (consecutive: $CONSECUTIVE_FAILURES). Building fix contract..."
@@ -2213,11 +2384,19 @@ main() {
2213
2384
  log " Fix contract: $fix_contract"
2214
2385
  log_debug "[DECIDE] iter=$ITERATION phase=fix_loop trigger=$verdict consecutive_failures=$CONSECUTIVE_FAILURES fix_contract=$fix_contract"
2215
2386
 
2216
- # Circuit breaker: consecutive failures
2387
+ # Circuit breaker: consecutive failures (with architecture escalation when at model ceiling)
2217
2388
  if (( CONSECUTIVE_FAILURES >= EFFECTIVE_CB_THRESHOLD )); then
2218
- log_debug "[GOV] iter=$ITERATION circuit_breaker=consecutive_failures detail=\"${EFFECTIVE_CB_THRESHOLD} consecutive verification failures\""
2219
- log_error "Circuit breaker: ${EFFECTIVE_CB_THRESHOLD} consecutive verification failures"
2220
- write_blocked_sentinel "${EFFECTIVE_CB_THRESHOLD} consecutive verification failures"
2389
+ # For codex: use full model:reasoning string (WORKER_MODEL loses reasoning suffix after upgrade)
2390
+ _ceiling_model_str="$([[ "$WORKER_ENGINE" = "codex" ]] && echo "${WORKER_CODEX_MODEL}:${WORKER_CODEX_REASONING}" || echo "$WORKER_MODEL")"
2391
+ if (( _MODEL_UPGRADED )) && [[ -z "$(get_next_model "$_ceiling_model_str")" ]]; then
2392
+ log_debug "[GOV] iter=$ITERATION circuit_breaker=consecutive_failures detail=\"architecture escalation: Worker at ceiling (${WORKER_MODEL}), ${EFFECTIVE_CB_THRESHOLD} consecutive failures\""
2393
+ log_error "Circuit breaker: architecture escalation — Worker upgraded to ceiling (${WORKER_MODEL}), ${EFFECTIVE_CB_THRESHOLD} consecutive failures"
2394
+ write_blocked_sentinel "architecture escalation: Worker upgraded to ceiling model (${WORKER_MODEL}), ${EFFECTIVE_CB_THRESHOLD} consecutive verification failures"
2395
+ else
2396
+ log_debug "[GOV] iter=$ITERATION circuit_breaker=consecutive_failures detail=\"${EFFECTIVE_CB_THRESHOLD} consecutive verification failures\""
2397
+ log_error "Circuit breaker: ${EFFECTIVE_CB_THRESHOLD} consecutive verification failures"
2398
+ write_blocked_sentinel "${EFFECTIVE_CB_THRESHOLD} consecutive verification failures"
2399
+ fi
2221
2400
  update_status "blocked" "consecutive_failures"
2222
2401
  return 1
2223
2402
  fi
@@ -2261,6 +2440,7 @@ main() {
2261
2440
 
2262
2441
  # --- AC5: Write per-iteration cost estimate ---
2263
2442
  write_cost_log "$ITERATION"
2443
+ write_campaign_jsonl "$ITERATION" "${signal_us_id:-unknown}" "${signal_status:-unknown}"
2264
2444
 
2265
2445
  # --- governance.md s7 step 8: Write result log ---
2266
2446
  write_result_log "$ITERATION" "$signal_status"
@@ -2279,7 +2459,6 @@ main() {
2279
2459
 
2280
2460
  # Max iterations reached
2281
2461
  log "Max iterations ($MAX_ITER) reached."
2282
- generate_campaign_report # AC4: TIMEOUT terminal path
2283
2462
  update_status "timeout" "max_iter"
2284
2463
  return 1
2285
2464
  }
@@ -2288,6 +2467,45 @@ main() {
2288
2467
  # Entry Point
2289
2468
  # =============================================================================
2290
2469
 
2470
+ # --- CLI: parse --worker-model / --verifier-model flags ---
2471
+ # These flags override env-var defaults (WORKER_ENGINE, WORKER_MODEL, etc.)
2472
+ # Format: "model:reasoning" → codex engine; "model-name" → claude engine
2473
+ _cli_i=1
2474
+ while (( _cli_i <= $# )); do
2475
+ case "${@[$_cli_i]}" in
2476
+ --worker-model)
2477
+ (( _cli_i++ ))
2478
+ _cli_parsed=$(parse_model_flag "${@[$_cli_i]:-}" "worker") || exit 1
2479
+ WORKER_ENGINE="${_cli_parsed%% *}"
2480
+ _cli_rest="${_cli_parsed#* }"
2481
+ WORKER_MODEL="${_cli_rest%% *}"
2482
+ if [[ "$WORKER_ENGINE" = "codex" ]]; then
2483
+ WORKER_CODEX_MODEL="$WORKER_MODEL"
2484
+ WORKER_CODEX_REASONING="${_cli_rest##* }"
2485
+ fi
2486
+ ;;
2487
+ --verifier-model)
2488
+ (( _cli_i++ ))
2489
+ _cli_parsed=$(parse_model_flag "${@[$_cli_i]:-}" "verifier") || exit 1
2490
+ VERIFIER_ENGINE="${_cli_parsed%% *}"
2491
+ _cli_rest="${_cli_parsed#* }"
2492
+ VERIFIER_MODEL="${_cli_rest%% *}"
2493
+ if [[ "$VERIFIER_ENGINE" = "codex" ]]; then
2494
+ VERIFIER_CODEX_MODEL="$VERIFIER_MODEL"
2495
+ VERIFIER_CODEX_REASONING="${_cli_rest##* }"
2496
+ fi
2497
+ ;;
2498
+ --lock-worker-model)
2499
+ LOCK_WORKER_MODEL=1
2500
+ ;;
2501
+ --final-consensus)
2502
+ FINAL_CONSENSUS=1
2503
+ ;;
2504
+ esac
2505
+ (( _cli_i++ ))
2506
+ done
2507
+ unset _cli_i _cli_parsed _cli_rest
2508
+
2291
2509
  # Require tmux — tmux mode only works inside an active tmux session
2292
2510
  if [[ -z "${TMUX:-}" ]]; then
2293
2511
  echo "ERROR: tmux mode requires running inside a tmux session."