@the-open-engine/zeroshot 6.9.1 → 6.10.1

package/task-lib/store.js

@@ -12,6 +12,7 @@ import { TASKS_DIR, LOGS_DIR } from './config.js';
 import { generateName } from './name-generator.js';
 
 const DB_FILE = join(TASKS_DIR, 'store.db');
+export const TASK_STORE_SCHEMA_VERSION = 4;
 
 /** @type {Database.Database | null} */
 let db = null;
@@ -41,6 +42,9 @@ function getDb() {
       status TEXT NOT NULL DEFAULT 'pending',
       pid INTEGER,
       session_id TEXT,
+      session_id_conflict INTEGER NOT NULL DEFAULT 0,
+      requested_resume_session_id TEXT,
+      resume_identity_verified INTEGER NOT NULL DEFAULT 0,
       log_file TEXT,
       created_at TEXT NOT NULL,
       updated_at TEXT NOT NULL,
@@ -52,7 +56,10 @@ function getDb() {
       socket_path TEXT,
       attachable INTEGER DEFAULT 0,
       process_group_id INTEGER,
-      termination_strategy TEXT
+      termination_strategy TEXT,
+      command_cleanup TEXT,
+      cancel_requested INTEGER DEFAULT 0,
+      spawn_ownership_token TEXT
     );
 
     CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
@@ -77,8 +84,7 @@ function getDb() {
     CREATE INDEX IF NOT EXISTS idx_schedules_enabled ON schedules(enabled);
   `);
 
-  ensureTaskColumn(db, 'process_group_id', 'INTEGER');
-  ensureTaskColumn(db, 'termination_strategy', 'TEXT');
+  migrateTaskStore(db);
 
   return db;
 }
@@ -90,6 +96,62 @@ function ensureTaskColumn(database, name, definition) {
   }
 }
 
+function parseCommandCleanup(value) {
+  if (typeof value !== 'string' || value === '') return null;
+  try {
+    const parsed = JSON.parse(value);
+    return parsed && typeof parsed === 'object' ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+function serializeCommandCleanup(value) {
+  return value ? JSON.stringify(value) : null;
+}
+
+export function migrateTaskStore(database) {
+  ensureTaskColumn(database, 'process_group_id', 'INTEGER');
+  ensureTaskColumn(database, 'termination_strategy', 'TEXT');
+  ensureTaskColumn(database, 'command_cleanup', 'TEXT');
+  ensureTaskColumn(database, 'cancel_requested', 'INTEGER DEFAULT 0');
+  ensureTaskColumn(database, 'spawn_ownership_token', 'TEXT');
+  ensureTaskColumn(database, 'requested_resume_session_id', 'TEXT');
+  ensureTaskColumn(database, 'session_id_conflict', 'INTEGER NOT NULL DEFAULT 0');
+  ensureTaskColumn(database, 'resume_identity_verified', 'INTEGER NOT NULL DEFAULT 0');
+  database.exec(`
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_spawn_ownership_token
+      ON tasks(spawn_ownership_token)
+      WHERE spawn_ownership_token IS NOT NULL
+  `);
+
+  const version = database.pragma('user_version', { simple: true });
+  if (version >= TASK_STORE_SCHEMA_VERSION) return;
+
+  database.transaction(() => {
+    if (version < 2) {
+      database
+        .prepare(
+          `UPDATE tasks
+           SET requested_resume_session_id = COALESCE(requested_resume_session_id, session_id),
+               session_id = NULL`
+        )
+        .run();
+    }
+    if (version < 3) {
+      database.prepare('UPDATE tasks SET session_id_conflict = 0').run();
+    }
+    if (version < 4) {
+      database.prepare('UPDATE tasks SET resume_identity_verified = 0').run();
+    }
+    database.pragma(`user_version = ${TASK_STORE_SCHEMA_VERSION}`);
+  })();
+}
+
+function nullable(value) {
+  return value || null;
+}
+
 export function ensureDirs() {
   if (!existsSync(TASKS_DIR)) mkdirSync(TASKS_DIR, { recursive: true });
   if (!existsSync(LOGS_DIR)) mkdirSync(LOGS_DIR, { recursive: true });
@@ -112,6 +174,9 @@ function rowToTask(row) {
     status: row.status,
     pid: row.pid,
     sessionId: row.session_id,
+    sessionIdConflict: Boolean(row.session_id_conflict),
+    requestedResumeSessionId: row.requested_resume_session_id,
+    resumeIdentityVerified: Boolean(row.resume_identity_verified),
     logFile: row.log_file,
     createdAt: row.created_at,
     updatedAt: row.updated_at,
@@ -124,6 +189,9 @@ function rowToTask(row) {
     attachable: Boolean(row.attachable),
     processGroupId: row.process_group_id,
     terminationStrategy: row.termination_strategy,
+    commandCleanup: parseCommandCleanup(row.command_cleanup),
+    cancelRequested: Boolean(row.cancel_requested),
+    spawnOwnershipToken: row.spawn_ownership_token,
   };
 }
 
@@ -149,13 +217,15 @@ export function saveTasks(tasks) {
   const database = getDb();
   const insert = database.prepare(`
     INSERT OR REPLACE INTO tasks (
-      id, prompt, full_prompt, cwd, status, pid, session_id, log_file,
+      id, prompt, full_prompt, cwd, status, pid, session_id, session_id_conflict, requested_resume_session_id, resume_identity_verified, log_file,
       created_at, updated_at, exit_code, error, provider, model,
-      schedule_id, socket_path, attachable, process_group_id, termination_strategy
+      schedule_id, socket_path, attachable, process_group_id, termination_strategy,
+      command_cleanup, cancel_requested, spawn_ownership_token
     ) VALUES (
-      @id, @prompt, @fullPrompt, @cwd, @status, @pid, @sessionId, @logFile,
+      @id, @prompt, @fullPrompt, @cwd, @status, @pid, @sessionId, @sessionIdConflict, @requestedResumeSessionId, @resumeIdentityVerified, @logFile,
       @createdAt, @updatedAt, @exitCode, @error, @provider, @model,
-      @scheduleId, @socketPath, @attachable, @processGroupId, @terminationStrategy
+      @scheduleId, @socketPath, @attachable, @processGroupId, @terminationStrategy,
+      @commandCleanup, @cancelRequested, @spawnOwnershipToken
     )
   `);
 
@@ -172,6 +242,9 @@ export function saveTasks(tasks) {
         status: task.status || 'pending',
         pid: task.pid || null,
         sessionId: task.sessionId || null,
+        sessionIdConflict: task.sessionIdConflict ? 1 : 0,
+        requestedResumeSessionId: nullable(task.requestedResumeSessionId),
+        resumeIdentityVerified: task.resumeIdentityVerified ? 1 : 0,
         logFile: task.logFile || null,
         createdAt: task.createdAt || new Date().toISOString(),
         updatedAt: task.updatedAt || new Date().toISOString(),
@@ -184,6 +257,9 @@ export function saveTasks(tasks) {
         attachable: task.attachable ? 1 : 0,
         processGroupId: task.processGroupId || null,
         terminationStrategy: task.terminationStrategy || null,
+        commandCleanup: serializeCommandCleanup(task.commandCleanup),
+        cancelRequested: task.cancelRequested ? 1 : 0,
+        spawnOwnershipToken: task.spawnOwnershipToken || null,
       });
     }
   });
@@ -214,6 +290,24 @@ export function getTask(id) {
   return rowToTask(row);
 }
 
+export function getTaskBySpawnOwnershipToken(token) {
+  if (typeof token !== 'string' || token.length === 0) return null;
+  const row = getDb().prepare('SELECT * FROM tasks WHERE spawn_ownership_token = ?').get(token);
+  return rowToTask(row);
+}
+
+export function requestTaskCancellation(id) {
+  const now = new Date().toISOString();
+  getDb()
+    .prepare(
+      `UPDATE tasks
+       SET cancel_requested = 1, updated_at = ?, error = ?
+       WHERE id = ? AND status = 'running'`
+    )
+    .run(now, 'Cancellation requested before provider startup completed', id);
+  return getTask(id);
+}
+
 /**
  * Update a task
  * @param {string} id
@@ -240,6 +334,9 @@ export function updateTask(id, updates) {
       status = @status,
       pid = @pid,
       session_id = @sessionId,
+      session_id_conflict = @sessionIdConflict,
+      requested_resume_session_id = @requestedResumeSessionId,
+      resume_identity_verified = @resumeIdentityVerified,
       log_file = @logFile,
       updated_at = @updatedAt,
       exit_code = @exitCode,
@@ -250,7 +347,10 @@ export function updateTask(id, updates) {
       socket_path = @socketPath,
       attachable = @attachable,
       process_group_id = @processGroupId,
-      termination_strategy = @terminationStrategy
+      termination_strategy = @terminationStrategy,
+      command_cleanup = @commandCleanup,
+      cancel_requested =
+        CASE WHEN @hasCancelRequested = 1 THEN @cancelRequested ELSE cancel_requested END
     WHERE id = @id
   `
     )
@@ -262,6 +362,9 @@ export function updateTask(id, updates) {
       status: updated.status || 'pending',
       pid: updated.pid || null,
       sessionId: updated.sessionId || null,
+      sessionIdConflict: updated.sessionIdConflict ? 1 : 0,
+      requestedResumeSessionId: nullable(updated.requestedResumeSessionId),
+      resumeIdentityVerified: updated.resumeIdentityVerified ? 1 : 0,
       logFile: updated.logFile || null,
       updatedAt: updated.updatedAt,
       exitCode: updated.exitCode ?? null,
@@ -273,6 +376,9 @@ export function updateTask(id, updates) {
       attachable: updated.attachable ? 1 : 0,
       processGroupId: updated.processGroupId || null,
       terminationStrategy: updated.terminationStrategy || null,
+      commandCleanup: serializeCommandCleanup(updated.commandCleanup),
+      hasCancelRequested: Object.prototype.hasOwnProperty.call(updates, 'cancelRequested') ? 1 : 0,
+      cancelRequested: updated.cancelRequested ? 1 : 0,
     });
 
   return updated;
@@ -295,13 +401,15 @@ export function addTask(task) {
     .prepare(
       `
     INSERT INTO tasks (
-      id, prompt, full_prompt, cwd, status, pid, session_id, log_file,
+      id, prompt, full_prompt, cwd, status, pid, session_id, session_id_conflict, requested_resume_session_id, resume_identity_verified, log_file,
       created_at, updated_at, exit_code, error, provider, model,
-      schedule_id, socket_path, attachable, process_group_id, termination_strategy
+      schedule_id, socket_path, attachable, process_group_id, termination_strategy,
+      command_cleanup, cancel_requested, spawn_ownership_token
     ) VALUES (
-      @id, @prompt, @fullPrompt, @cwd, @status, @pid, @sessionId, @logFile,
+      @id, @prompt, @fullPrompt, @cwd, @status, @pid, @sessionId, @sessionIdConflict, @requestedResumeSessionId, @resumeIdentityVerified, @logFile,
       @createdAt, @updatedAt, @exitCode, @error, @provider, @model,
-      @scheduleId, @socketPath, @attachable, @processGroupId, @terminationStrategy
+      @scheduleId, @socketPath, @attachable, @processGroupId, @terminationStrategy,
+      @commandCleanup, @cancelRequested, @spawnOwnershipToken
     )
   `
     )
@@ -313,6 +421,9 @@ export function addTask(task) {
       status: fullTask.status || 'pending',
       pid: fullTask.pid || null,
       sessionId: fullTask.sessionId || null,
+      sessionIdConflict: fullTask.sessionIdConflict ? 1 : 0,
+      requestedResumeSessionId: nullable(fullTask.requestedResumeSessionId),
+      resumeIdentityVerified: fullTask.resumeIdentityVerified ? 1 : 0,
       logFile: fullTask.logFile || null,
       createdAt: fullTask.createdAt,
       updatedAt: fullTask.updatedAt,
@@ -325,6 +436,9 @@ export function addTask(task) {
       attachable: fullTask.attachable ? 1 : 0,
       processGroupId: fullTask.processGroupId || null,
       terminationStrategy: fullTask.terminationStrategy || null,
+      commandCleanup: serializeCommandCleanup(fullTask.commandCleanup),
+      cancelRequested: fullTask.cancelRequested ? 1 : 0,
+      spawnOwnershipToken: fullTask.spawnOwnershipToken || null,
     });
 
   return fullTask;

package/task-lib/watcher-output-runtime.js

@@ -0,0 +1,284 @@
+import { spawn } from 'child_process';
+import {
+  detectProviderFatalError,
+  detectProviderStreamingModeError,
+  recoverProviderStructuredOutput,
+  supportsProviderStructuredOutputRecovery,
+} from './provider-helper-runtime.js';
+import { terminateProcess } from './process-termination.js';
+
+export const COMMAND_CLEANUP_UNINITIALIZED = Symbol('command-cleanup-uninitialized');
+
+export function spawnWatcherProvider(command, finalArgs, options) {
+  return spawn(command, finalArgs, {
+    ...options,
+    windowsHide: true,
+  });
+}
+
+export async function terminateWatcherProvider(providerProcess, options = {}) {
+  const pid = providerProcess?.pid;
+  if (!pid) return true;
+  const platform = options.platform || process.platform;
+  const terminate = options.terminateProcessFn || terminateProcess;
+  const terminationStrategy = platform === 'win32' ? 'process-tree' : 'process-group';
+  const result = await terminate(pid, {
+    processGroupId: platform === 'win32' ? null : pid,
+    terminationStrategy,
+  });
+  if (terminationStrategy === 'process-tree' && result.alreadyDead && !options.exitObserved) {
+    return false;
+  }
+  return result.terminated;
+}
+
+function splitBufferLines(buffer, chunk) {
+  const nextBuffer = buffer + chunk;
+  const lines = nextBuffer.split('\n');
+  return { lines: lines.slice(0, -1), remaining: lines.at(-1) || '' };
+}
+
+export function resolveWatcherCommand(config, commandSpec, fallbackArgs, normalizeProviderName) {
+  return {
+    providerName: normalizeProviderName(config.provider || 'claude'),
+    env: { ...process.env, ...(commandSpec.env || {}) },
+    command: commandSpec.binary,
+    finalArgs: [...(commandSpec.args || fallbackArgs)],
+  };
+}
+
+export async function completeWatcherTask({
+  taskId,
+  completion,
+  commandCleanup,
+  terminateProvider,
+  updateTask,
+  emergencyLog,
+  terminalUpdates = {},
+}) {
+  let providerTerminal = false;
+  try {
+    providerTerminal = await terminateProvider();
+  } catch (error) {
+    emergencyLog(`[${Date.now()}][CLEANUP] Provider termination check failed: ${error.message}\n`);
+  }
+  if (!providerTerminal) {
+    emergencyLog(
+      `[${Date.now()}][CLEANUP] Provider termination boundary is still live; preserving command cleanup paths.\n`
+    );
+    try {
+      await updateTask(taskId, {
+        status: 'running',
+        error: completion.error
+          ? `${completion.error}; provider termination could not be confirmed`
+          : 'Provider termination could not be confirmed; retry and cleanup remain blocked',
+      });
+    } catch (error) {
+      emergencyLog(`[${Date.now()}][ERROR] Failed to preserve task ownership: ${error.message}\n`);
+    }
+    return false;
+  }
+
+  let cleanupSucceeded = false;
+  if (commandCleanup === COMMAND_CLEANUP_UNINITIALIZED) {
+    emergencyLog(
+      `[${Date.now()}][CLEANUP] Command cleanup ownership was not initialized; preserving the persisted receipt.\n`
+    );
+  } else if (commandCleanup?.run) {
+    try {
+      cleanupSucceeded = await commandCleanup.run();
+    } catch (error) {
+      emergencyLog(`[${Date.now()}][CLEANUP] Command cleanup failed: ${error.message}\n`);
+    }
+  }
+  try {
+    await updateTask(taskId, {
+      status: completion.status,
+      pid: null,
+      processGroupId: null,
+      exitCode: completion.resolvedCode,
+      error: completion.error,
+      cancelRequested: false,
+      ...terminalUpdates,
+      ...(completion.terminalUpdates || {}),
+      ...(cleanupSucceeded ? { commandCleanup: null } : {}),
+    });
+  } catch (error) {
+    emergencyLog(`[${Date.now()}][ERROR] Failed to update task status: ${error.message}\n`);
+  }
+  return true;
+}
+
+export async function completePendingWatcherCancellation({
+  taskId,
+  getTask,
+  ...completionOptions
+}) {
+  if (!getTask(taskId)?.cancelRequested) return false;
+  await completeWatcherTask({
+    taskId,
+    ...completionOptions,
+    completion: {
+      status: 'killed',
+      resolvedCode: 143,
+      error: 'Cancellation requested before provider startup completed',
+    },
+  });
+  return true;
+}
+
+export function completeWatcherFailure({ error, source, ...completionOptions }) {
+  const errorMessage = error instanceof Error ? error.stack || error.message : String(error);
+  completionOptions.emergencyLog(`\n[${Date.now()}][CRASH] ${source}: ${errorMessage}\n`);
+  return completeWatcherTask({
+    ...completionOptions,
+    completion: {
+      status: 'failed',
+      resolvedCode: 1,
+      error: `${source}: ${errorMessage}`,
+    },
+  });
+}
+
+export function createWatcherOutputRuntime({
+  config,
+  providerName,
+  log,
+  stopProvider,
+  providerSessionCapture = null,
+}) {
+  const enableRecovery = supportsProviderStructuredOutputRecovery(providerName);
+  const silentJsonMode =
+    config.outputFormat === 'json' &&
+    config.jsonSchema &&
+    config.silentJsonOutput &&
+    enableRecovery;
+  let finalResultJson = null;
+  let streamingModeError = null;
+  let fatalError = null;
+  const captureProviderSession = providerSessionCapture?.captureLine || (() => {});
+
+  function maybeHandleFatalError(line, timestamp) {
+    if (fatalError) return false;
+    const detected = detectProviderFatalError(providerName, line);
+    if (!detected) return false;
+    fatalError = detected;
+    if (silentJsonMode) log(`[${timestamp}]${line}\n`);
+    log(`[${timestamp}][FATAL] ${detected}\n`);
+    stopProvider(timestamp);
+    return true;
+  }
+
+  function captureStreamingError(line, timestamp) {
+    const detectedError = detectProviderStreamingModeError(providerName, line);
+    if (!detectedError) return false;
+    streamingModeError = { ...detectedError, timestamp };
+    return true;
+  }
+
+  function maybeCaptureStructuredOutput(line) {
+    try {
+      const json = JSON.parse(line);
+      if (json.structured_output) finalResultJson = line;
+    } catch {
+      // Not JSON, skip.
+    }
+  }
+
+  function handleOutputLine(line, timestamp) {
+    captureProviderSession(line);
+    if (silentJsonMode && !line.trim()) return;
+    maybeHandleFatalError(line, timestamp);
+    if (captureStreamingError(line, timestamp)) return;
+    if (silentJsonMode) {
+      maybeCaptureStructuredOutput(line);
+    } else {
+      log(`[${timestamp}]${line}\n`);
+    }
+  }
+
+  function consumeOutput(buffer, chunk) {
+    const timestamp = Date.now();
+    const { lines, remaining } = splitBufferLines(buffer, chunk.toString());
+    for (const line of lines) handleOutputLine(line, timestamp);
+    return remaining;
+  }
+
+  function consumeStderr(buffer, chunk) {
+    const timestamp = Date.now();
+    const { lines, remaining } = splitBufferLines(buffer, chunk.toString());
+    for (const line of lines) log(`[${timestamp}]${line}\n`);
+    return remaining;
+  }
+
+  function flushOutput(buffer, timestamp) {
+    if (!buffer.trim()) return;
+    captureProviderSession(buffer);
+    if (!enableRecovery) {
+      if (!silentJsonMode) log(`[${timestamp}]${buffer}\n`);
+      return;
+    }
+    maybeHandleFatalError(buffer, timestamp);
+    if (captureStreamingError(buffer, timestamp)) return;
+    if (silentJsonMode) {
+      maybeCaptureStructuredOutput(buffer);
+    } else {
+      log(`[${timestamp}]${buffer}\n`);
+    }
+  }
+
+  function flushStderr(buffer, timestamp) {
+    if (!buffer.trim()) return;
+    maybeHandleFatalError(buffer, timestamp);
+    log(`[${timestamp}]${buffer}\n`);
+  }
+
+  function attemptRecovery(code, timestamp) {
+    if (!(code !== 0 && streamingModeError?.sessionId)) return null;
+    const recovered = recoverProviderStructuredOutput(providerName, streamingModeError.sessionId);
+    if (recovered?.payload) {
+      const recoveredLine = JSON.stringify(recovered.payload);
+      if (silentJsonMode) {
+        finalResultJson = recoveredLine;
+      } else {
+        log(`[${timestamp}]${recoveredLine}\n`);
+      }
+    } else if (streamingModeError.line) {
+      const prefix = silentJsonMode ? '' : `[${streamingModeError.timestamp}]`;
+      log(`${prefix}${streamingModeError.line}\n`);
+    }
+    return recovered;
+  }
+
+  function complete({ code, signal, outputBuffer, stderrBuffer = null }) {
+    const timestamp = Date.now();
+    flushOutput(outputBuffer, timestamp);
+    if (stderrBuffer !== null) flushStderr(stderrBuffer, timestamp);
+    const recovered = attemptRecovery(code, timestamp);
+    const sessionIdentityError = providerSessionCapture?.getCompletionError() || null;
+    if (silentJsonMode && finalResultJson) log(`${finalResultJson}\n`);
+    if (config.outputFormat !== 'json') {
+      log(`\n${'='.repeat(50)}\n`);
+      log(`Finished: ${new Date().toISOString()}\n`);
+      log(`Exit code: ${code}, Signal: ${signal}\n`);
+    }
+    let resolvedCode = code;
+    if (recovered?.payload) {
+      resolvedCode = 0;
+    }
+    if (fatalError || sessionIdentityError) {
+      resolvedCode = 1;
+    }
+    return {
+      resolvedCode,
+      status: resolvedCode === 0 ? 'completed' : 'failed',
+      error:
+        fatalError ||
+        sessionIdentityError ||
+        (resolvedCode !== 0 && signal ? `Killed by ${signal}` : null),
+      terminalUpdates: providerSessionCapture?.getCompletionUpdate(resolvedCode) || {},
+    };
+  }
+
+  return { complete, consumeOutput, consumeStderr };
+}