@covibes/zeroshot 5.2.1 → 5.3.0

package/task-lib/store.js

@@ -1,31 +1,37 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
+import { existsSync, mkdirSync, readFileSync, writeFileSync, unlinkSync, statSync } from 'fs';
 import { TASKS_DIR, TASKS_FILE, LOGS_DIR, SCHEDULES_FILE } from './config.js';
 import { generateName } from './name-generator.js';
 import lockfile from 'proper-lockfile';
 
-// Lock options for sync API (no retries allowed)
+// Stale lock timeout - 5 seconds is plenty for JSON read/write
+const LOCK_STALE_MS = 5000;
+
+// Lock options with async retry support
 const LOCK_OPTIONS = {
-  stale: 30000, // Consider lock stale after 30s
+  stale: LOCK_STALE_MS,
+  retries: {
+    retries: 20,
+    minTimeout: 100,
+    maxTimeout: 200,
+    randomize: true,
+  },
 };
 
-// Retry wrapper for sync lock acquisition
-function lockWithRetry(file, options, maxRetries = 100, delayMs = 100) {
-  for (let i = 0; i < maxRetries; i++) {
-    try {
-      return lockfile.lockSync(file, options);
-    } catch (err) {
-      if (err.code === 'ELOCKED' && i < maxRetries - 1) {
-        // File is locked, wait and retry
-        const start = Date.now();
-        while (Date.now() - start < delayMs) {
-          // Busy wait (sync)
-        }
-        continue;
+/**
+ * Remove lock file if it's stale (older than LOCK_STALE_MS)
+ */
+function cleanStaleLock(filePath) {
+  const lockPath = filePath + '.lock';
+  try {
+    if (existsSync(lockPath)) {
+      const age = Date.now() - statSync(lockPath).mtimeMs;
+      if (age > LOCK_STALE_MS) {
+        unlinkSync(lockPath);
       }
-      throw err;
     }
+  } catch {
+    // Ignore - another process may have cleaned it
   }
-  throw new Error(`Failed to acquire lock after ${maxRetries} retries`);
 }
 
 export function ensureDirs() {
@@ -60,9 +66,9 @@ export function saveTasks(tasks) {
 /**
  * Atomic read-modify-write with file locking
  * @param {Function} modifier - Function that receives tasks object and returns modified tasks
- * @returns {any} - Return value from modifier function
+ * @returns {Promise<any>} - Return value from modifier function
  */
-export function withTasksLock(modifier) {
+export async function withTasksLock(modifier) {
   ensureDirs();
 
   // Create file if it doesn't exist (needed for locking)
@@ -72,8 +78,11 @@ export function withTasksLock(modifier) {
 
   let release;
   try {
-    // Acquire lock (blocks until available)
-    release = lockWithRetry(TASKS_FILE, LOCK_OPTIONS);
+    // Clean stale locks from crashed processes
+    cleanStaleLock(TASKS_FILE);
+
+    // Acquire lock with async API (proper retries without CPU spin-wait)
+    release = await lockfile.lock(TASKS_FILE, LOCK_OPTIONS);
 
     // Read current state
     const content = readFileSync(TASKS_FILE, 'utf-8');
@@ -93,7 +102,7 @@ export function withTasksLock(modifier) {
     return result;
   } finally {
     if (release) {
-      release();
+      await release();
     }
   }
 }
@@ -122,8 +131,8 @@ export function addTask(task) {
   });
 }
 
-export function removeTask(id) {
-  withTasksLock((tasks) => {
+export async function removeTask(id) {
+  await withTasksLock((tasks) => {
     delete tasks[id];
   });
 }
@@ -138,7 +147,7 @@ export function generateScheduleId() {
 
 // Schedule management - same pattern with locking
 
-function withSchedulesLock(modifier) {
+async function withSchedulesLock(modifier) {
   ensureDirs();
 
   if (!existsSync(SCHEDULES_FILE)) {
@@ -147,7 +156,11 @@ function withSchedulesLock(modifier) {
 
   let release;
   try {
-    release = lockWithRetry(SCHEDULES_FILE, LOCK_OPTIONS);
+    // Clean stale locks from crashed processes
+    cleanStaleLock(SCHEDULES_FILE);
+
+    // Acquire lock with async API (proper retries without CPU spin-wait)
+    release = await lockfile.lock(SCHEDULES_FILE, LOCK_OPTIONS);
 
     const content = readFileSync(SCHEDULES_FILE, 'utf-8');
     let schedules;
@@ -163,7 +176,7 @@ function withSchedulesLock(modifier) {
     return result;
   } finally {
     if (release) {
-      release();
+      await release();
     }
   }
 }
@@ -210,8 +223,8 @@ export function updateSchedule(id, updates) {
   });
 }
 
-export function removeSchedule(id) {
-  withSchedulesLock((schedules) => {
+export async function removeSchedule(id) {
+  await withSchedulesLock((schedules) => {
     delete schedules[id];
   });
 }

package/task-lib/watcher.js

@@ -1,24 +1,18 @@
 #!/usr/bin/env node
 
 /**
- * Watcher process - spawns and monitors a claude process
+ * Watcher process - spawns and monitors a CLI process
  * Runs detached from parent, updates task status on completion
- *
- * Uses regular spawn (not PTY) - Claude CLI with --print is non-interactive
- * PTY causes EIO errors when processes are killed/OOM'd
  */
 
 import { spawn } from 'child_process';
 import { appendFileSync } from 'fs';
-import { dirname } from 'path';
-import { fileURLToPath } from 'url';
 import { updateTask } from './store.js';
+import { detectStreamingModeError, recoverStructuredOutput } from './claude-recovery.js';
 import { createRequire } from 'module';
 
 const require = createRequire(import.meta.url);
-const { getClaudeCommand } = require('../lib/settings.js');
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
+const { normalizeProviderName } = require('../lib/provider-names');
 
 const [, , taskId, cwd, logFile, argsJson, configJson] = process.argv;
 const args = JSON.parse(argsJson);
@@ -28,80 +22,76 @@ function log(msg) {
   appendFileSync(logFile, msg);
 }
 
-// Build environment - inherit user's auth method (API key or subscription)
-const env = { ...process.env };
-
-// Add model flag - priority: config.model > ANTHROPIC_MODEL env var
-const claudeArgs = [...args];
-const model = config.model || env.ANTHROPIC_MODEL;
-if (model && !claudeArgs.includes('--model')) {
-  claudeArgs.unshift('--model', model);
-}
+const providerName = normalizeProviderName(config.provider || 'claude');
+const enableRecovery = providerName === 'claude';
 
-// Get configured Claude command (supports custom commands like 'ccr code')
-const { command: claudeCommand, args: claudeExtraArgs } = getClaudeCommand();
-const finalArgs = [...claudeExtraArgs, ...claudeArgs];
+const env = { ...process.env, ...(config.env || {}) };
+const command = config.command || 'claude';
+const finalArgs = [...args];
 
-// Spawn claude using regular child_process (not PTY)
-// --print mode is non-interactive, PTY adds overhead and causes EIO on OOM
-const child = spawn(claudeCommand, finalArgs, {
+const child = spawn(command, finalArgs, {
   cwd,
   env,
   stdio: ['ignore', 'pipe', 'pipe'],
 });
 
-// Update task with PID
 updateTask(taskId, { pid: child.pid });
 
-// For JSON schema output with silent mode, capture ONLY the structured_output JSON
 const silentJsonMode =
-  config.outputFormat === 'json' && config.jsonSchema && config.silentJsonOutput;
+  config.outputFormat === 'json' && config.jsonSchema && config.silentJsonOutput && enableRecovery;
+
 let finalResultJson = null;
+let streamingModeError = null;
 
-// Buffer for incomplete lines (need complete lines to add timestamps)
 let stdoutBuffer = '';
 
-// Process stdout data
-// CRITICAL: Prepend timestamp to each line for real-time tracking in cluster
-// Format: [1733301234567]{json...} - consumers parse timestamp for accurate timing
 child.stdout.on('data', (data) => {
   const chunk = data.toString();
   const timestamp = Date.now();
 
   if (silentJsonMode) {
-    // Parse each line to find the one with structured_output
     stdoutBuffer += chunk;
     const lines = stdoutBuffer.split('\n');
-    stdoutBuffer = lines.pop() || ''; // Keep incomplete line in buffer
+    stdoutBuffer = lines.pop() || '';
 
     for (const line of lines) {
       if (!line.trim()) continue;
+      if (enableRecovery) {
+        const detectedError = detectStreamingModeError(line);
+        if (detectedError) {
+          streamingModeError = { ...detectedError, timestamp };
+          continue;
+        }
+      }
       try {
         const json = JSON.parse(line);
         if (json.structured_output) {
           finalResultJson = line;
         }
       } catch {
-        // Not JSON or incomplete, skip
+        // Not JSON, skip
       }
     }
   } else {
-    // Normal mode - stream with timestamps on each complete line
     stdoutBuffer += chunk;
     const lines = stdoutBuffer.split('\n');
-    stdoutBuffer = lines.pop() || ''; // Keep incomplete line in buffer
+    stdoutBuffer = lines.pop() || '';
 
     for (const line of lines) {
-      // Timestamp each line: [epochMs]originalContent
+      if (enableRecovery) {
+        const detectedError = detectStreamingModeError(line);
+        if (detectedError) {
+          streamingModeError = { ...detectedError, timestamp };
+          continue;
+        }
+      }
       log(`[${timestamp}]${line}\n`);
     }
   }
 });
 
-// Buffer for stderr incomplete lines
 let stderrBuffer = '';
 
-// Stream stderr to log with timestamps
 child.stderr.on('data', (data) => {
   const chunk = data.toString();
   const timestamp = Date.now();
@@ -115,55 +105,84 @@ child.stderr.on('data', (data) => {
   }
 });
 
-// Handle process exit
-child.on('close', (code, signal) => {
+child.on('close', async (code, signal) => {
   const timestamp = Date.now();
 
-  // Flush any remaining buffered stdout
   if (stdoutBuffer.trim()) {
-    if (silentJsonMode) {
-      try {
-        const json = JSON.parse(stdoutBuffer);
-        if (json.structured_output) {
-          finalResultJson = stdoutBuffer;
+    if (enableRecovery) {
+      const detectedError = detectStreamingModeError(stdoutBuffer);
+      if (detectedError) {
+        streamingModeError = { ...detectedError, timestamp };
+      } else if (silentJsonMode) {
+        try {
+          const json = JSON.parse(stdoutBuffer);
+          if (json.structured_output) {
+            finalResultJson = stdoutBuffer;
+          }
+        } catch {
+          // Not valid JSON
         }
-      } catch {
-        // Not valid JSON
+      } else {
+        log(`[${timestamp}]${stdoutBuffer}\n`);
       }
-    } else {
+    } else if (!silentJsonMode) {
       log(`[${timestamp}]${stdoutBuffer}\n`);
     }
   }
 
-  // Flush any remaining buffered stderr
   if (stderrBuffer.trim()) {
     log(`[${timestamp}]${stderrBuffer}\n`);
   }
 
-  // In silent JSON mode, log ONLY the final structured_output JSON
+  let recovered = null;
+  if (enableRecovery && code !== 0 && streamingModeError?.sessionId) {
+    recovered = recoverStructuredOutput(streamingModeError.sessionId);
+    if (recovered?.payload) {
+      const recoveredLine = JSON.stringify(recovered.payload);
+      if (silentJsonMode) {
+        finalResultJson = recoveredLine;
+      } else {
+        log(`[${timestamp}]${recoveredLine}\n`);
+      }
+    } else if (streamingModeError.line) {
+      if (silentJsonMode) {
+        log(streamingModeError.line + '\n');
+      } else {
+        log(`[${streamingModeError.timestamp}]${streamingModeError.line}\n`);
+      }
+    }
+  }
+
   if (silentJsonMode && finalResultJson) {
     log(finalResultJson + '\n');
   }
 
-  // Skip footer for pure JSON output
   if (config.outputFormat !== 'json') {
     log(`\n${'='.repeat(50)}\n`);
     log(`Finished: ${new Date().toISOString()}\n`);
     log(`Exit code: ${code}, Signal: ${signal}\n`);
   }
 
-  // Simple status: completed if exit 0, failed otherwise
-  const status = code === 0 ? 'completed' : 'failed';
-  updateTask(taskId, {
-    status,
-    exitCode: code,
-    error: signal ? `Killed by ${signal}` : null,
-  });
+  const resolvedCode = recovered?.payload ? 0 : code;
+  const status = resolvedCode === 0 ? 'completed' : 'failed';
+  try {
+    await updateTask(taskId, {
+      status,
+      exitCode: resolvedCode,
+      error: resolvedCode === 0 ? null : signal ? `Killed by ${signal}` : null,
+    });
+  } catch (updateError) {
+    log(`[${Date.now()}][ERROR] Failed to update task status: ${updateError.message}\n`);
+  }
   process.exit(0);
 });
 
-child.on('error', (err) => {
+child.on('error', async (err) => {
   log(`\nError: ${err.message}\n`);
-  updateTask(taskId, { status: 'failed', error: err.message });
+  try {
+    await updateTask(taskId, { status: 'failed', error: err.message });
+  } catch (updateError) {
+    log(`[${Date.now()}][ERROR] Failed to update task status: ${updateError.message}\n`);
+  }
   process.exit(1);
 });