@cnrai/pave 0.3.32 → 0.3.34

package/test/http500-backoff.test.js

@@ -0,0 +1,854 @@
+#!/usr/bin/env node
+/**
+ * Tests for HTTP 500 back-off and circuit-breaker (Issue #256)
+ *
+ * Verifies:
+ * - consecutiveFailures counter exists and is tracked
+ * - MAX_CONSECUTIVE_FAILURES constant defined
+ * - Back-off tiers: 1-2 normal, 3-5 = 2x, 6+ = 3x (capped)
+ * - Circuit-breaker exits loop after max failures
+ * - writeStatus includes lastError and consecutiveFailures
+ * - Error state in STATES enum
+ * - 409 errors not counted as consecutive failures
+ * - Success resets consecutiveFailures to 0
+ *
+ * Designed to run standalone: node test/http500-backoff.test.js
+ */
+
+const path = require('path');
+const fs = require('fs');
+
+let passed = 0;
+let failed = 0;
+
+function assert(condition, msg) {
+  if (!condition) throw new Error('Assertion failed: ' + msg);
+}
+
+function runTest(name, fn) {
+  try {
+    fn();
+    console.log('\u2705 ' + name);
+    passed++;
+  } catch (e) {
+    console.log('\u274C ' + name + ': ' + e.message);
+    failed++;
+  }
+}
+
+// Read source files for inspection
+const pavePath = path.join(__dirname, '..', 'index.js');
+const paveSource = fs.readFileSync(pavePath, 'utf8');
+
+const registryPath = path.join(__dirname, '..', 'lib', 'agent-registry.js');
+const registrySource = fs.readFileSync(registryPath, 'utf8');
+
+// ============================================================
+// 1. STATES enum has ERROR state
+// ============================================================
+
+runTest('STATES enum includes ERROR state', () => {
+  assert(registrySource.indexOf("ERROR: 'error'") !== -1,
+    'should have ERROR state in STATES enum');
+});
+
+runTest('STATES enum includes required states for this feature', () => {
+  const requiredStates = ['ERROR', 'SLEEPING', 'WORKING', 'STOPPED'];
+  for (let i = 0; i < requiredStates.length; i++) {
+    assert(registrySource.indexOf(requiredStates[i] + ':') !== -1,
+      'should have ' + requiredStates[i] + ' in STATES enum');
+  }
+});
+
+// ============================================================
+// 2. Consecutive failure tracking variables
+// ============================================================
+
+runTest('consecutiveFailures counter declared and initialized to 0', () => {
+  assert(paveSource.indexOf('let consecutiveFailures = 0') !== -1,
+    'should declare consecutiveFailures initialized to 0');
+});
+
+runTest('MAX_CONSECUTIVE_FAILURES constant set to 10', () => {
+  assert(paveSource.indexOf('MAX_CONSECUTIVE_FAILURES = 10') !== -1,
+    'should set MAX_CONSECUTIVE_FAILURES to 10');
+});
+
+runTest('MAX_BACKOFF_MS defined at 15 minutes', () => {
+  assert(paveSource.indexOf('MAX_BACKOFF_MS') !== -1,
+    'should define MAX_BACKOFF_MS');
+  assert(paveSource.indexOf('15 * 60 * 1000') !== -1,
+    'should cap backoff at 15 minutes');
+});
+
+runTest('lastError variable declared and initialized to null', () => {
+  assert(paveSource.indexOf('let lastError = null') !== -1,
+    'should declare lastError initialized to null');
+});
+
+// ============================================================
+// 3. Failure increment on error paths
+// ============================================================
+
+runTest('consecutiveFailures incremented on response.success=false', () => {
+  // With tiered error handling (#271), consecutiveFailures++ is inside
+  // applyTieredErrorHandling which is called from the !response.success path
+  // Use 'if (!response.success)' to avoid matching the JSDoc comment
+  const idx = paveSource.indexOf('if (!response.success)');
+  assert(idx !== -1, 'should have response.success check');
+  const section = paveSource.substring(idx, idx + 800);
+  // The path should call applyTieredErrorHandling (which internally increments for non-server errors)
+  assert(section.indexOf('applyTieredErrorHandling') !== -1,
+    'should call applyTieredErrorHandling for tiered error classification');
+});
+
+runTest('lastError set on response.success=false', () => {
+  // Use 'if (!response.success)' to avoid matching JSDoc
+  const idx = paveSource.indexOf('if (!response.success)');
+  assert(idx !== -1, 'should have response.success check');
+  const section = paveSource.substring(idx, idx + 600);
+  // lastError is set either directly (body-too-large path) or via applyTieredErrorHandling
+  assert(section.indexOf('lastError = responseError') !== -1 || section.indexOf('applyTieredErrorHandling') !== -1,
+    'should set lastError on failure (directly or via applyTieredErrorHandling)');
+});
+
+runTest('error log includes failure count on response failure', () => {
+  // With tiered handling, the error log is produced by applyTieredErrorHandling
+  // which includes 'consecutive failure' in its log messages
+  const funcIdx = paveSource.indexOf('function applyTieredErrorHandling');
+  assert(funcIdx !== -1, 'should have applyTieredErrorHandling function');
+  const funcSection = paveSource.substring(funcIdx, funcIdx + 1000);
+  assert(funcSection.indexOf('consecutive failure') !== -1,
+    'applyTieredErrorHandling should show failure count');
+  assert(funcSection.indexOf('MAX_CONSECUTIVE_FAILURES') !== -1,
+    'applyTieredErrorHandling should show max in error log');
+});
+
+runTest('consecutiveFailures incremented in catch block (non-409)', () => {
+  // With tiered error handling (#271), the catch block calls applyTieredErrorHandling
+  // instead of directly incrementing consecutiveFailures
+  const idx = paveSource.indexOf('Handle HTTP 409');
+  assert(idx !== -1, 'should find the catch block with 409 handling');
+  const section = paveSource.substring(idx, idx + 1200);
+  assert(section.indexOf('applyTieredErrorHandling') !== -1,
+    'catch block should call applyTieredErrorHandling for tiered classification');
+});
+
+runTest('lastError set in catch block via applyTieredErrorHandling', () => {
+  // applyTieredErrorHandling sets lastError internally with appropriate prefix
+  const idx = paveSource.indexOf('Handle HTTP 409');
+  assert(idx !== -1, 'should find catch block');
+  const section = paveSource.substring(idx, idx + 1200);
+  assert(section.indexOf('applyTieredErrorHandling(err.message') !== -1,
+    'should pass err.message to applyTieredErrorHandling');
+});
+
+runTest('catch block error log includes failure count', () => {
+  // The error log is now produced by applyTieredErrorHandling
+  const funcIdx = paveSource.indexOf('function applyTieredErrorHandling');
+  assert(funcIdx !== -1, 'should have applyTieredErrorHandling function');
+  const funcSection = paveSource.substring(funcIdx, funcIdx + 1000);
+  assert(funcSection.indexOf('consecutive failure') !== -1,
+    'should show failure count in error log');
+});
+
+// ============================================================
+// 4. 409 errors NOT counted as consecutive failures
+// ============================================================
+
+runTest('HTTP 409 does not increment consecutiveFailures', () => {
+  const idx = paveSource.indexOf("err.message.includes('409')");
+  assert(idx !== -1, 'should detect 409 errors');
+  const section = paveSource.substring(idx, idx + 400);
+  // After the 409 block there may be an `else if` (e.g., body-too-large) or `else`
+  const blockEnd = section.indexOf('} else');
+  assert(blockEnd !== -1, 'should have else clause after 409 handling');
+  const block409 = section.substring(0, blockEnd);
+  assert(block409.indexOf('consecutiveFailures++') === -1,
+    'should NOT increment consecutiveFailures for 409 errors');
+});
+
+runTest('409 block has comment explaining why not counted', () => {
+  const idx = paveSource.indexOf("err.message.includes('409')");
+  assert(idx !== -1, 'should detect 409 errors');
+  const section = paveSource.substring(idx, idx + 300);
+  assert(section.indexOf('don\'t count') !== -1 || section.indexOf('not a provider error') !== -1,
+    'should have comment explaining why 409 is not counted');
+});
+
+runTest('409 block sets lastError to descriptive message (not stale)', () => {
+  const idx = paveSource.indexOf("err.message.includes('409')");
+  assert(idx !== -1, 'should detect 409 errors');
+  const section = paveSource.substring(idx, idx + 400);
+  // After the 409 block there may be an `else if` (e.g., body-too-large) or `else`
+  const blockEnd = section.indexOf('} else');
+  assert(blockEnd !== -1, 'should have else clause');
+  const block409 = section.substring(0, blockEnd);
+  assert(block409.indexOf('lastError =') !== -1,
+    'should set lastError in 409 block so status reflects current condition');
+  assert(block409.indexOf('409') !== -1,
+    'lastError message should mention 409');
+});
+
+// ============================================================
+// 5. Success resets consecutiveFailures
+// ============================================================
+
+runTest('consecutiveFailures reset to 0 on success', () => {
+  const idx = paveSource.indexOf('Send succeeded -- commit all deferred state updates');
+  assert(idx !== -1, 'should find success comment');
+  const section = paveSource.substring(idx, idx + 300);
+  assert(section.indexOf('consecutiveFailures = 0') !== -1,
+    'should reset consecutiveFailures to 0 on success');
+});
+
+runTest('lastError reset to null on success', () => {
+  const idx = paveSource.indexOf('Send succeeded -- commit all deferred state updates');
+  assert(idx !== -1, 'should find success comment');
+  const section = paveSource.substring(idx, idx + 300);
+  assert(section.indexOf('lastError = null') !== -1,
+    'should reset lastError to null on success');
+});
+
+// ============================================================
+// 6. Back-off tiers
+// ============================================================
+
+runTest('effectiveSleepMs defaults to sleepMs (tier 1: failures 1-2)', () => {
+  assert(paveSource.indexOf('let effectiveSleepMs = sleepMs') !== -1,
+    'should define effectiveSleepMs starting at sleepMs');
+});
+
+runTest('back-off tier 2: failures 3-5 use 2x sleepMs', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 600);
+  assert(section.indexOf('consecutiveFailures >= 3') !== -1,
+    'should check for >= 3 consecutive failures');
+  assert(section.indexOf('sleepMs * 2') !== -1,
+    'should use 2x sleepMs for tier 2');
+});
+
+runTest('back-off tier 3: failures 6+ use 3x sleepMs', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 600);
+  assert(section.indexOf('consecutiveFailures >= 6') !== -1,
+    'should check for >= 6 consecutive failures');
+  assert(section.indexOf('sleepMs * 3') !== -1,
+    'should use 3x sleepMs for tier 3');
+});
+
+runTest('back-off capped at MAX_BACKOFF_MS', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 600);
+  // Both tier 2 and tier 3 should cap with Math.min
+  const tier2Idx = section.indexOf('sleepMs * 2');
+  const tier3Idx = section.indexOf('sleepMs * 3');
+  assert(tier2Idx !== -1 && tier3Idx !== -1, 'should have both tiers');
+  // MAX_BACKOFF_MS should appear in the section (used in Math.min)
+  assert(section.indexOf('MAX_BACKOFF_MS') !== -1,
+    'should cap effectiveSleepMs at MAX_BACKOFF_MS');
+});
+
+runTest('tier 3 checked before tier 2 (higher tier first)', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 400);
+  const tier3Idx = section.indexOf('consecutiveFailures >= 6');
+  const tier2Idx = section.indexOf('consecutiveFailures >= 3');
+  assert(tier3Idx < tier2Idx,
+    'tier 3 (>= 6) should be checked before tier 2 (>= 3) in if/else chain');
+});
+
+// ============================================================
+// 7. Back-off log messages
+// ============================================================
+
+runTest('back-off log only shows "Backing off" at tier 2+ (>= 3 failures)', () => {
+  const idx = paveSource.indexOf('Backing off');
+  assert(idx !== -1, 'should have "Backing off" log message');
+  // The condition should be >= 3, not > 0
+  const section = paveSource.substring(Math.max(0, idx - 200), idx + 50);
+  assert(section.indexOf('consecutiveFailures >= 3') !== -1,
+    'Backing off log should only appear when consecutiveFailures >= 3');
+});
+
+runTest('failures 1-2 use "Retrying" message (not "Backing off")', () => {
+  const idx = paveSource.indexOf('Retrying after error');
+  assert(idx !== -1, 'should have "Retrying after error" message for failures 1-2');
+  const section = paveSource.substring(Math.max(0, idx - 200), idx + 50);
+  assert(section.indexOf('consecutiveFailures > 0') !== -1,
+    'Retrying message should appear when consecutiveFailures > 0 (and < 3)');
+});
+
+runTest('normal sleep log used when no failures', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 1500);
+  assert(section.indexOf('consecutiveFailures >= 3') !== -1,
+    'should check for tier 2+ failures for Backing off');
+  assert(section.indexOf('consecutiveFailures > 0') !== -1,
+    'should check for any failures for Retrying');
+  assert(section.indexOf('Sleeping for') !== -1,
+    'should have normal sleeping log for zero-failure case');
+});
+
+// ============================================================
+// 8. Circuit-breaker
+// ============================================================
+
+runTest('circuit-breaker checks MAX_CONSECUTIVE_FAILURES', () => {
+  assert(paveSource.indexOf('consecutiveFailures >= MAX_CONSECUTIVE_FAILURES') !== -1,
+    'should check if consecutiveFailures >= MAX_CONSECUTIVE_FAILURES');
+});
+
+runTest('circuit-breaker writes ERROR state', () => {
+  const idx = paveSource.indexOf('consecutiveFailures >= MAX_CONSECUTIVE_FAILURES');
+  assert(idx !== -1, 'should find circuit-breaker check');
+  const section = paveSource.substring(idx, idx + 800);
+  assert(section.indexOf('STATES.ERROR') !== -1,
+    'should write ERROR state');
+});
+
+runTest('circuit-breaker breaks out of loop', () => {
+  const idx = paveSource.indexOf('consecutiveFailures >= MAX_CONSECUTIVE_FAILURES');
+  assert(idx !== -1, 'should find circuit-breaker check');
+  const section = paveSource.substring(idx, idx + 1000);
+  assert(section.indexOf('break') !== -1,
+    'should break out of the agent loop');
+});
+
+runTest('circuit-breaker ERROR status includes lastError', () => {
+  const idx = paveSource.indexOf('consecutiveFailures >= MAX_CONSECUTIVE_FAILURES');
+  assert(idx !== -1, 'should find circuit-breaker check');
+  const section = paveSource.substring(idx, idx + 1500);
+  // Match shorthand `lastError,` or explicit `lastError:`
+  assert(/\blastError[,:\s]/.test(section),
+    'should include lastError in ERROR status');
+});
+
+runTest('circuit-breaker ERROR status includes consecutiveFailures', () => {
+  const idx = paveSource.indexOf('consecutiveFailures >= MAX_CONSECUTIVE_FAILURES');
+  assert(idx !== -1, 'should find circuit-breaker check');
+  const section = paveSource.substring(idx, idx + 1500);
+  // Match shorthand `consecutiveFailures,` or explicit `consecutiveFailures:`
+  assert(/\bconsecutiveFailures[,:\s]/.test(section),
+    'should include consecutiveFailures in ERROR status');
+});
+
+runTest('circuit-breaker log message is descriptive', () => {
+  const idx = paveSource.indexOf('Agent stopping after');
+  assert(idx !== -1, 'should have clear "Agent stopping after" log message');
+  const section = paveSource.substring(idx, idx + 200);
+  assert(section.indexOf('consecutive failures') !== -1,
+    'should mention consecutive failures in stopping message');
+  assert(section.indexOf('lastError') !== -1 || section.indexOf('Last error') !== -1,
+    'should mention last error in stopping message');
+});
+
+// ============================================================
+// 9. Sleeping writeStatus includes error fields
+// ============================================================
+
+runTest('sleeping writeStatus includes lastError field', () => {
+  // Find the sleeping writeStatus (the one with STATES.SLEEPING)
+  const idx = paveSource.indexOf('state: registry.STATES.SLEEPING');
+  assert(idx !== -1, 'should find SLEEPING writeStatus');
+  const section = paveSource.substring(idx, idx + 500);
+  // Match shorthand `lastError,` or explicit `lastError:`
+  assert(/\blastError[,:\s]/.test(section),
+    'should include lastError in sleeping writeStatus');
+});
+
+runTest('sleeping writeStatus includes consecutiveFailures field', () => {
+  const idx = paveSource.indexOf('state: registry.STATES.SLEEPING');
+  assert(idx !== -1, 'should find SLEEPING writeStatus');
+  const section = paveSource.substring(idx, idx + 500);
+  // Match shorthand `consecutiveFailures,` or explicit `consecutiveFailures:`
+  assert(/\bconsecutiveFailures[,:\s]/.test(section),
+    'should include consecutiveFailures in sleeping writeStatus');
+});
+
+runTest('sleeping writeStatus shows different currentTask for errors vs normal', () => {
+  // Use effectiveSleepMs as anchor since it's unique to the main loop's sleep section
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs in main loop');
+  const section = paveSource.substring(idx, idx + 3000);
+  assert(section.indexOf('backing off after error') !== -1,
+    'should show "backing off" task when there are 3+ failures');
+  assert(section.indexOf('retrying after error') !== -1,
+    'should show "retrying" task when there are 1-2 failures');
+  assert(section.indexOf("'sleeping for '") !== -1,
+    'should show normal sleeping task when no failures');
+});
+
+runTest('currentTask messaging aligns with backoff tiers', () => {
+  // Use effectiveSleepMs as anchor since it's unique to the main loop's sleep section
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs in main loop');
+  const section = paveSource.substring(idx, idx + 3000);
+  // The "backing off" message should only appear for >= 3 failures
+  // (matching the tier logic that doubles sleep at 3+)
+  const backoffIdx = section.indexOf('backing off after error');
+  const retryIdx = section.indexOf('retrying after error');
+  assert(backoffIdx !== -1 && retryIdx !== -1,
+    'both backing off and retrying messages should exist');
+  // "consecutiveFailures >= 3" should gate the "backing off" message
+  assert(section.indexOf('consecutiveFailures >= 3') !== -1,
+    'backing off threshold should match the >= 3 tier boundary');
+});
+
+// ============================================================
+// 10. signalAwareSleep uses effectiveSleepMs
+// ============================================================
+
+runTest('signalAwareSleep uses effectiveSleepMs', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 5000);
+  assert(section.indexOf('signalAwareSleep(effectiveSleepMs') !== -1,
+    'signalAwareSleep should use effectiveSleepMs, not raw sleepMs');
+});
+
+// ============================================================
+// 11. Circuit-breaker is placed BEFORE back-off (order matters)
+// ============================================================
+
+runTest('circuit-breaker check comes before back-off calculation', () => {
+  const cbIdx = paveSource.indexOf('consecutiveFailures >= MAX_CONSECUTIVE_FAILURES');
+  const boIdx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(cbIdx !== -1 && boIdx !== -1, 'both should exist');
+  assert(cbIdx < boIdx,
+    'circuit-breaker should be checked before back-off is calculated');
+});
+
+// ============================================================
+// 12. Copilot review fixes
+// ============================================================
+
+runTest('response failure uses response.error when available', () => {
+  // Use 'if (!response.success)' to avoid matching JSDoc
+  const idx = paveSource.indexOf('if (!response.success)');
+  assert(idx !== -1, 'should find response.success check');
+  const section = paveSource.substring(idx, idx + 400);
+  assert(section.indexOf('response.error') !== -1 || section.indexOf('response && response.error') !== -1,
+    'should use response.error for actionable error message');
+});
+
+runTest('catch block passes err.message to applyTieredErrorHandling', () => {
+  const idx = paveSource.indexOf('Handle HTTP 409');
+  assert(idx !== -1, 'should find catch block comment');
+  const section = paveSource.substring(idx, idx + 1200);
+  // The catch block should delegate to applyTieredErrorHandling with err.message
+  assert(section.indexOf("applyTieredErrorHandling(err.message || 'Unknown error'") !== -1,
+    'catch block should pass err.message to applyTieredErrorHandling');
+});
+
+runTest('normal exit path closes agent log stream', () => {
+  const idx = paveSource.indexOf('Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal cleanup section');
+  const section = paveSource.substring(idx, idx + 1500);
+  assert(section.indexOf('closeAgentLogStream') !== -1,
+    'should close agent log stream on normal exit (incl. circuit-breaker)');
+});
+
+runTest('normal exit path restores console overrides', () => {
+  const idx = paveSource.indexOf('Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal cleanup section');
+  const section = paveSource.substring(idx, idx + 1500);
+  assert(section.indexOf('originalConsoleLog') !== -1,
+    'should restore console.log on normal exit');
+  assert(section.indexOf('originalConsoleWarn') !== -1,
+    'should restore console.warn on normal exit');
+});
+
+runTest('circuit-breaker exit uses trippedCircuitBreaker flag', () => {
+  const idx = paveSource.indexOf('Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal cleanup section');
+  const section = paveSource.substring(idx, idx + 2000);
+  assert(section.indexOf('trippedCircuitBreaker') !== -1,
+    'should use trippedCircuitBreaker flag (not failure count) for exit code');
+  assert(section.indexOf('? 1 : 0') !== -1,
+    'should return 1 on circuit-breaker exit, 0 on normal exit');
+});
+
+runTest('trippedCircuitBreaker flag set before break', () => {
+  const idx = paveSource.indexOf('Agent stopping after');
+  assert(idx !== -1, 'should find circuit-breaker log');
+  const section = paveSource.substring(idx, idx + 2000);
+  const flagIdx = section.indexOf('trippedCircuitBreaker = true');
+  const breakIdx = section.indexOf('break;');
+  assert(flagIdx !== -1, 'should set trippedCircuitBreaker = true');
+  assert(breakIdx !== -1, 'should break');
+  assert(flagIdx < breakIdx,
+    'trippedCircuitBreaker should be set before break');
+});
+
+// ============================================================
+// 11. Normal exit writes STOPPED + releases lock (#256 round 3)
+// ============================================================
+
+runTest('normal exit writes STOPPED status (not circuit-breaker)', () => {
+  const idx = paveSource.indexOf('Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal cleanup section');
+  const section = paveSource.substring(idx, idx + 2000);
+  assert(section.indexOf('STATES.STOPPED') !== -1,
+    'should write STOPPED status on normal exit');
+  assert(section.indexOf('!trippedCircuitBreaker') !== -1,
+    'should only write STOPPED when NOT circuit-breaker (ERROR already written)');
+});
+
+runTest('normal exit releases agent lock', () => {
+  const idx = paveSource.indexOf('Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal cleanup section');
+  const section = paveSource.substring(idx, idx + 2000);
+  assert(section.indexOf('releaseAgent(agentName)') !== -1,
+    'should call releaseAgent to release lock on any exit path');
+});
+
+runTest('cleanup mirrors SIGINT handler (status + lock)', () => {
+  const idx = paveSource.indexOf('Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal cleanup section');
+  const section = paveSource.substring(idx, idx + 2000);
+  // Must have both writeStatus and releaseAgent
+  assert(section.indexOf('writeStatus') !== -1 && section.indexOf('releaseAgent') !== -1,
+    'normal exit should mirror SIGINT cleanup: writeStatus + releaseAgent');
+});
+
+// ============================================================
+// 12. ERROR state preserved by listAgents (#256 round 3)
+// ============================================================
+
+runTest('listAgents preserves ERROR state (not overwritten to CRASHED)', () => {
+  assert(registrySource.indexOf("status.state !== STATES.ERROR") !== -1,
+    'listAgents should exclude ERROR from CRASHED override (dead PID with ERROR = intentional)');
+});
+
+runTest('listAgents comment explains terminal states', () => {
+  const idx = registrySource.indexOf('Detect crashed agents');
+  assert(idx !== -1, 'should find crashed detection section');
+  const section = registrySource.substring(idx, idx + 300);
+  assert(section.indexOf('terminal') !== -1 || section.indexOf('STOPPED') !== -1,
+    'comment should explain why ERROR and STOPPED are excluded from CRASHED override');
+});
+
+// ============================================================
+// 14. SIGINT handler preserves ERROR state (#256 round 5)
+// ============================================================
+
+runTest('SIGINT cleanup guards against overwriting circuit-breaker ERROR state', () => {
+  // The SIGINT/SIGTERM handler should check trippedCircuitBreaker
+  // before writing STOPPED status, to avoid overwriting ERROR
+  const idx = paveSource.indexOf('Handle Ctrl+C gracefully');
+  assert(idx !== -1, 'should find SIGINT handler section');
+  const section = paveSource.substring(idx, idx + 2000);
+  assert(section.indexOf('trippedCircuitBreaker') !== -1,
+    'SIGINT handler should check trippedCircuitBreaker before writing STOPPED');
+});
+
+runTest('trippedCircuitBreaker declared at function scope (before cleanup)', () => {
+  // trippedCircuitBreaker must be declared BEFORE the cleanup handler
+  // so it's accessible from both the handler and the try block
+  const cbDecl = paveSource.indexOf('let trippedCircuitBreaker');
+  const cleanupDecl = paveSource.indexOf('const cleanup = async');
+  assert(cbDecl !== -1, 'should find trippedCircuitBreaker declaration');
+  assert(cleanupDecl !== -1, 'should find cleanup handler declaration');
+  assert(cbDecl < cleanupDecl,
+    'trippedCircuitBreaker must be declared before cleanup handler to avoid ReferenceError');
+});
+
+// ============================================================
+// 13. sendMessageAndStream returns error field on failure (#256 round 4)
+// ============================================================
+
+runTest('finish() accepts optional errorMsg parameter', () => {
+  const idx = paveSource.indexOf('function sendMessageAndStream');
+  assert(idx !== -1, 'should find sendMessageAndStream');
+  const section = paveSource.substring(idx, idx + 5000);
+  assert(section.indexOf('finish = (success = true, errorMsg = null)') !== -1,
+    'finish should accept an optional errorMsg parameter');
+});
+
+runTest('finish() includes error field in resolved object', () => {
+  const idx = paveSource.indexOf('function sendMessageAndStream');
+  assert(idx !== -1, 'should find sendMessageAndStream');
+  const section = paveSource.substring(idx, idx + 5000);
+  assert(section.indexOf('errorMsg != null') !== -1,
+    'finish should use != null check (not truthy) to include error field');
+  assert(section.indexOf('result.error = errorMsg') !== -1,
+    'finish should include error in the resolved object when errorMsg is provided');
+});
+
+runTest('session.error passes error message to finish()', () => {
+  const idx = paveSource.indexOf('function sendMessageAndStream');
+  assert(idx !== -1, 'should find sendMessageAndStream');
+  const section = paveSource.substring(idx, idx + 15000);
+  assert(section.indexOf('finish(false, errorMsg)') !== -1,
+    'session.error handler should pass the error message to finish()');
+});
+
+// ============================================================
+// 18. Server close with timeout and signal cleanup
+// ============================================================
+
+runTest('normal exit awaits server.close with timeout', () => {
+  // Find the normal exit cleanup section (after the loop)
+  const idx = paveSource.indexOf('// Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal exit cleanup section');
+  const section = paveSource.substring(idx, idx + 1000);
+  assert(section.indexOf('server.close(') !== -1,
+    'normal exit should call server.close()');
+  assert(section.indexOf('setTimeout(resolve, 5000)') !== -1,
+    'normal exit should have 5s timeout for server close');
+  assert(section.indexOf('await new Promise') !== -1,
+    'normal exit should await server close completion');
+});
+
+runTest('normal exit removes SIGINT/SIGTERM listeners', () => {
+  const idx = paveSource.indexOf('// Cleanup on normal exit');
+  assert(idx !== -1, 'should find normal exit cleanup section');
+  const section = paveSource.substring(idx, idx + 1000);
+  assert(section.indexOf("removeListener('SIGINT', cleanup)") !== -1,
+    'normal exit should remove SIGINT listener');
+  assert(section.indexOf("removeListener('SIGTERM', cleanup)") !== -1,
+    'normal exit should remove SIGTERM listener');
+});
+
+runTest('error exit awaits server.close with timeout', () => {
+  // Find the catch block cleanup
+  const idx = paveSource.indexOf('// Close log stream on error exit path');
+  assert(idx !== -1, 'should find error exit cleanup section');
+  const section = paveSource.substring(idx, idx + 1500);
+  assert(section.indexOf('server.close(') !== -1,
+    'error exit should call server.close()');
+  assert(section.indexOf('setTimeout(resolve, 5000)') !== -1,
+    'error exit should have 5s timeout for server close');
+  assert(section.indexOf("removeListener('SIGINT', cleanup)") !== -1,
+    'error exit should remove SIGINT listener');
+});
+
+// ============================================================
+// Done
+// ============================================================
+
+// ============================================================
+// 19. Tiered error handling (#271)
+// ============================================================
+
+runTest('classifyError function exists', () => {
+  assert(paveSource.indexOf('function classifyError(') !== -1,
+    'should have classifyError function for tiered error handling');
+});
+
+runTest('applyTieredErrorHandling function exists', () => {
+  assert(paveSource.indexOf('function applyTieredErrorHandling(') !== -1,
+    'should have applyTieredErrorHandling function shared by both error paths');
+});
+
+runTest('classifyError detects server errors (500/502/503)', () => {
+  const idx = paveSource.indexOf('function classifyError(');
+  assert(idx !== -1, 'should find classifyError');
+  const section = paveSource.substring(idx, idx + 500);
+  assert(section.indexOf('server-error') !== -1,
+    'should return server-error category');
+  assert(section.indexOf('50[023]') !== -1 || (section.indexOf('500') !== -1 && section.indexOf('502') !== -1 && section.indexOf('503') !== -1),
+    'should detect 500, 502, 503 status codes');
+});
+
+runTest('classifyError detects rate limiting (429)', () => {
+  const idx = paveSource.indexOf('function classifyError(');
+  assert(idx !== -1, 'should find classifyError');
+  const section = paveSource.substring(idx, idx + 500);
+  assert(section.indexOf('rate-limited') !== -1,
+    'should return rate-limited category');
+  assert(section.indexOf('429') !== -1,
+    'should detect 429 status code');
+  assert(section.indexOf('rate limit') !== -1,
+    'should detect "rate limit" text');
+});
+
+runTest('classifyError detects auth errors (401/403)', () => {
+  const idx = paveSource.indexOf('function classifyError(');
+  assert(idx !== -1, 'should find classifyError');
+  const section = paveSource.substring(idx, idx + 800);
+  assert(section.indexOf('auth-error') !== -1,
+    'should return auth-error category');
+  assert(section.indexOf('unauthorized') !== -1 || section.indexOf('forbidden') !== -1,
+    'should detect unauthorized/forbidden text');
+});
+
+runTest('classifyError uses case-insensitive matching', () => {
+  const idx = paveSource.indexOf('function classifyError(');
+  assert(idx !== -1, 'should find classifyError');
+  const section = paveSource.substring(idx, idx + 500);
+  assert(section.indexOf('.toLowerCase()') !== -1,
+    'should normalize error message to lowercase for case-insensitive matching');
+});
+
+runTest('server errors do NOT increment consecutiveFailures', () => {
+  const idx = paveSource.indexOf('function applyTieredErrorHandling(');
+  assert(idx !== -1, 'should find applyTieredErrorHandling');
+  const section = paveSource.substring(idx, idx + 1000);
+  // Find the server-error branch
+  const serverIdx = section.indexOf("category === 'server-error'");
+  assert(serverIdx !== -1, 'should check for server-error category');
+  // The server-error branch should NOT have consecutiveFailures++
+  const nextElse = section.indexOf('} else', serverIdx);
+  const serverBranch = section.substring(serverIdx, nextElse);
+  assert(serverBranch.indexOf('consecutiveFailures++') === -1,
+    'server error branch should NOT increment consecutiveFailures');
+});
+
+runTest('non-server errors DO increment consecutiveFailures', () => {
+  const idx = paveSource.indexOf('function applyTieredErrorHandling(');
+  assert(idx !== -1, 'should find applyTieredErrorHandling');
+  const section = paveSource.substring(idx, idx + 1000);
+  // rate-limited, auth-error, and other should all increment
+  const matches = section.match(/consecutiveFailures\+\+/g);
+  assert(matches && matches.length >= 3,
+    'rate-limited, auth-error, and other categories should all increment consecutiveFailures');
+});
+
+runTest('server error uses fixed 5-minute backoff', () => {
+  assert(paveSource.indexOf('SERVER_ERROR_BACKOFF_MS') !== -1,
+    'should define SERVER_ERROR_BACKOFF_MS constant');
+  assert(paveSource.indexOf('5 * 60 * 1000') !== -1,
+    'should have 5-minute constant');
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 600);
+  assert(section.indexOf('isServerError') !== -1,
+    'should check isServerError flag');
+  assert(section.indexOf('SERVER_ERROR_BACKOFF_MS') !== -1,
+    'should use SERVER_ERROR_BACKOFF_MS for server error backoff');
+});
+
+runTest('server error backoff capped at MAX_BACKOFF_MS', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 600);
+  const serverBranch = section.indexOf('isServerError');
+  assert(serverBranch !== -1, 'should check isServerError');
+  const branchSection = section.substring(serverBranch, serverBranch + 200);
+  assert(branchSection.indexOf('MAX_BACKOFF_MS') !== -1,
+    'server error backoff should be capped at MAX_BACKOFF_MS');
+});
+
+runTest('sleepTask includes server error messaging', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 3000);
+  assert(section.indexOf('server error - retrying') !== -1,
+    'sleepTask should include server-error-specific message');
+});
+
+runTest('tiered handling applied to both error paths', () => {
+  // Count applyTieredErrorHandling calls  should be at least 2
+  // (one in !response.success path, one in catch path)
+  let count = 0;
+  let searchIdx = 0;
+  while (true) {
+    const found = paveSource.indexOf('applyTieredErrorHandling(', searchIdx);
+    if (found === -1) break;
+    count++;
+    searchIdx = found + 1;
+  }
+  assert(count >= 2,
+    'applyTieredErrorHandling should be called from both !response.success and catch paths (found ' + count + ' calls)');
+});
+
+runTest('error-aware retry message when lastError is set', () => {
+  const _idx = paveSource.indexOf('lastError');
+  // Find the message selection section
+  const msgIdx = paveSource.indexOf('} else if (lastError)');
+  assert(msgIdx !== -1, 'should have lastError check in message selection');
+  const section = paveSource.substring(msgIdx, msgIdx + 500);
+  assert(section.indexOf('previous iteration ended with an error') !== -1,
+    'should include error context in retry message');
+});
+
+runTest('error-aware message includes actual lastError value', () => {
+  const msgIdx = paveSource.indexOf('} else if (lastError)');
+  assert(msgIdx !== -1, 'should have lastError check in message selection');
+  const section = paveSource.substring(msgIdx, msgIdx + 500);
+  assert(section.indexOf('lastError') !== -1 && section.indexOf("(' + lastError + ')") !== -1 || section.indexOf('+ lastError +') !== -1,
+    'retry message should include the actual error value for context');
+});
+
+runTest('post-send idle check uses reduced 30s timeout (#271)', () => {
+  // Find the post-send idle check by looking for the specific comment
+  const idx = paveSource.indexOf('verify the session is actually idle');
+  assert(idx !== -1, 'should find post-send idle check comment');
+  const section = paveSource.substring(idx, idx + 1200);
+  assert(section.indexOf('30 * 1000') !== -1,
+    'should use 30-second timeout (reduced from 10 minutes per #271)');
+  assert(section.indexOf('pollIntervalMs: 1000') !== -1,
+    'should use 1-second poll interval');
+});
+
+runTest('sleepTask is computed once and reused', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx, idx + 3500);
+  // sleepTask should be defined once and used for both agentContext and writeStatus
+  assert(section.indexOf('const sleepTask =') !== -1,
+    'should define sleepTask as const');
+  assert(section.indexOf('agentContext.currentTask = sleepTask') !== -1,
+    'should use sleepTask for agentContext');
+  assert(section.indexOf('currentTask: sleepTask') !== -1,
+    'should use sleepTask for writeStatus (no duplication)');
+});
+
+runTest('lastErrorCategory variable declared and initialized to null', () => {
+  assert(paveSource.indexOf('let lastErrorCategory = null') !== -1,
+    'should declare lastErrorCategory initialized to null');
+});
+
+runTest('applyTieredErrorHandling sets lastErrorCategory', () => {
+  const idx = paveSource.indexOf('function applyTieredErrorHandling(');
+  assert(idx !== -1, 'should find applyTieredErrorHandling');
+  const section = paveSource.substring(idx, idx + 1000);
+  assert(section.indexOf('lastErrorCategory = category') !== -1,
+    'should set lastErrorCategory from classified category');
+});
+
+runTest('isServerError uses lastErrorCategory not string prefix', () => {
+  const idx = paveSource.indexOf('let effectiveSleepMs = sleepMs');
+  assert(idx !== -1, 'should find effectiveSleepMs');
+  const section = paveSource.substring(idx - 200, idx);
+  assert(section.indexOf("lastErrorCategory === 'server-error'") !== -1,
+    'should use lastErrorCategory for server error check (not string prefix)');
+  assert(section.indexOf("lastError.startsWith('Server error:')") === -1,
+    'should NOT use string prefix matching for server error detection');
+});
+
+runTest('lastErrorCategory reset on success', () => {
+  const idx = paveSource.indexOf('Send succeeded -- commit all deferred state updates');
+  assert(idx !== -1, 'should find success comment');
+  const section = paveSource.substring(idx, idx + 300);
+  assert(section.indexOf('lastErrorCategory = null') !== -1,
+    'should reset lastErrorCategory to null on success');
+});
+
+runTest('applyTieredErrorHandling has no unused context parameter', () => {
+  const idx = paveSource.indexOf('function applyTieredErrorHandling(');
+  assert(idx !== -1, 'should find applyTieredErrorHandling');
+  const section = paveSource.substring(idx, idx + 100);
+  // Should only accept errorMsg, not context
+  assert(section.indexOf('context') === -1,
+    'should not have unused context parameter');
+});
+
+// ============================================================
+// Done
+// ============================================================
+
+console.log('\n' + passed + '/' + (passed + failed) + ' tests passed');
+if (failed > 0) {
+  console.log(failed + ' test(s) FAILED');
+  process.exitCode = 1;
+} else {
+  console.log('All tests passed!');
+}