@oar-sh/oar 0.9.13 → 0.9.15

package/server/services/session-worker-launch-service.mjs

@@ -14,6 +14,7 @@ import {
   isOarMcpServerDisabled,
 } from '../mcp/oar-mcp-server.mjs';
 import { WORKER_LOG_FILE_ENV } from '../../shared/worker-runtime/worker-log-file.mjs';
+import { missingWorkingDirectoryError } from '../../shared/worker-runtime/working-directory.mjs';
 import {
   describeSpawnEnvironment,
   envPathKey,
@@ -213,6 +214,9 @@ export function applyCursorProviderEnvironment(env = {}, {
   enabled = false,
   model = '',
   apiKey = '',
+  // The relay's resolved, absolute data dir: the worker keeps its agent stores
+  // there, and must not resolve a relative one or the config's own again.
+  dataDir = '',
 } = {}) {
   const next = { ...env };
   if (resolveWorkerKind(next) === 'cursor') delete next.COPILOT_WEB_RELAY_WORKER_KIND;
@@ -225,6 +229,8 @@ export function applyCursorProviderEnvironment(env = {}, {
   next.CURSOR_API_KEY = normalizedApiKey;
   const normalizedModel = normalizeText(model);
   if (normalizedModel) next.CURSOR_RELAY_MODEL = normalizedModel;
+  const normalizedDataDir = normalizeText(dataDir);
+  if (normalizedDataDir) next.COPILOT_WEB_RELAY_DATA_DIR = path.resolve(normalizedDataDir);
   return next;
 }
 
@@ -717,6 +723,9 @@ export function buildTmuxWorkerShellCommand(targetSessionId, env = {}, {
     'COPILOT_WEB_RELAY_CONFIG',
     'COPILOT_WEB_RELAY_TOOLS',
     'COPILOT_WEB_RELAY_LOG_DIR',
+    // The Cursor worker keeps its agent stores in the relay's data dir
+    // (applyCursorProviderEnvironment).
+    'COPILOT_WEB_RELAY_DATA_DIR',
     'COPILOT_WEB_RELAY_CLI_EXECUTABLE',
     'COPILOT_WEB_RELAY_EXTENSION_BOOTSTRAP_PATH',
     'COPILOT_SDK_PATH',
@@ -847,6 +856,19 @@ export function describeWindowsStartFailure(state, { spawnCommand = 'cmd.exe', p
   return null;
 }
 
+/**
+ * A spawn whose folder is gone fails with ENOENT (or ENOTDIR), which reads as
+ * a missing program (`node`, `cmd.exe`). Asked only after a spawn failed: the folder
+ * is named instead, or null when it is there. Exported for the tests.
+ * (A tmux launch never gets here: `new-session -c` falls back to the home
+ * folder, and the worker reports the folder on the turn.)
+ */
+export function describeLaunchFolderFailure(state, folder, { check = missingWorkingDirectoryError } = {}) {
+  if (state?.error?.code !== 'ENOENT' && state?.error?.code !== 'ENOTDIR') return null;
+  const missing = check(folder);
+  return missing ? `worker-spawn-failed: ${missing.message}` : null;
+}
+
 function buildWorkerLaunchEnv({ processCwd, workspaceRoot, env = process.env } = {}) {
   const launchProcessCwd = String(processCwd || '').trim();
   const launchWorkspaceRoot = String(workspaceRoot || '').trim() || launchProcessCwd;
@@ -900,6 +922,8 @@ export async function launchSessionCli({
   // Where a launch that went wrong explains itself (one line per call).
   log = null,
   locateExecutableImpl = locateExecutable,
+  // Asked after a failed spawn whether its folder is gone (the error, or null).
+  checkWorkingDirectoryImpl = missingWorkingDirectoryError,
 } = {}) {
   const target = String(targetSessionId || '').trim();
   if (!target) throw new Error('missing-target-session-id');
@@ -1154,7 +1178,8 @@ export async function launchSessionCli({
     // `cmd /c start` returns at once when the program started. Still running,
     // or ended with an error, means it did not: name the program, and log the
     // command and the PATH it was looked up on.
-    const failure = describeWindowsStartFailure(childState, { spawnCommand, program: startedProgram });
+    const failure = describeLaunchFolderFailure(childState, launchProcessCwd, { check: checkWorkingDirectoryImpl })
+      || describeWindowsStartFailure(childState, { spawnCommand, program: startedProgram });
     if (failure) {
       if (childState.tracked && !childState.exited) {
         try { child.kill?.(); } catch {}
@@ -1176,6 +1201,11 @@ export async function launchSessionCli({
     // A failed spawn reports its error on the next tick.
     await new Promise((resolve) => setImmediate(resolve));
     const reason = childState.error?.code || childState.error?.message || '';
+    const folderFailure = describeLaunchFolderFailure(childState, launchProcessCwd, { check: checkWorkingDirectoryImpl });
+    if (folderFailure) {
+      explain(`could not start ${spawnCommand}: ${folderFailure}`);
+      throw new Error(folderFailure);
+    }
     explain(`could not start ${spawnCommand}${reason ? ` (${reason})` : ''}; ${spawnEnvironmentText()}`);
     throw new Error(`worker-spawn-unhealthy:missing-pid${reason ? ` (${spawnCommand}: ${reason})` : ''}`);
   }

package/server/services/stale-turn-recovery.mjs

@@ -0,0 +1,99 @@
+'use strict';
+
+// The stale-turn sweep: a `processing` row whose worker went silent, or whose
+// turn ran past the max turn duration, goes back to the queue. Which rows
+// qualify is decided by the repository (listRecoverableProcessing); this
+// module decides what happens to them: continuations are torn down, consumed
+// rows fail terminally, rows out of retry budget fail, the rest are requeued
+// and announced.
+
+/**
+ * `failRecoveredRowTerminally(row, reason)`, `failConsumedRowTerminally(row,
+ * reason)` and `noteRestartTurnOutcome(row, outcome)` are the runtime's own;
+ * `emit(event, payload)` is the socket broadcast.
+ */
+export function createStaleTurnRecovery({
+  db,
+  stmts,
+  emit,
+  maxRequeueRetries,
+  failRecoveredRowTerminally,
+  failConsumedRowTerminally,
+  noteRestartTurnOutcome,
+  warn = (line) => console.warn(line),
+} = {}) {
+  if (!db) throw new Error('createStaleTurnRecovery requires db');
+  if (!stmts) throw new Error('createStaleTurnRecovery requires stmts');
+  if (typeof emit !== 'function') throw new Error('createStaleTurnRecovery requires emit');
+
+  // A Stop still pending for a requeued turn is settled: the attempt it was
+  // aimed at is gone.
+  const settleRecoveredAbortControls = db.prepare(`
+    UPDATE relay_control_requests
+    SET status = 'failed',
+        error = ?,
+        updated_at = ?,
+        completed_at = ?
+    WHERE queue_message_id = ?
+      AND type IN ('abort_turn', 'abort_subagent')
+      AND status IN ('pending', 'processing')
+  `);
+
+  function recoverProcessingOlderThan(cutoffIso, requeueAtIso, { ceilingBeforeIso = null } = {}) {
+    const params = { inactiveBefore: cutoffIso, ceilingBefore: ceilingBeforeIso || null };
+    // Stale background-continuation turns are torn down, never requeued —
+    // replaying one would deliver the CLI's own continuation as a user prompt.
+    const staleContinuations = stmts.listStaleProcessingContinuations?.all?.(params) || [];
+    for (const row of staleContinuations) {
+      stmts.dropStaleContinuation?.run?.(row.id);
+      emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'failed' });
+      noteRestartTurnOutcome(row, 'cut');
+      // A teardown is a turn ending with no reply; it must be visible in the
+      // console log (the only trace of a silent one was `ignored=already_failed`
+      // hours later, when the worker's reply arrived for the dead row).
+      const lastSeen = row.owner_last_claimed_at || row.processing_at || row.timestamp || null;
+      const reason = lastSeen && lastSeen < cutoffIso ? `inactive since ${lastSeen}` : `ceiling (started ${row.processing_at || row.timestamp})`;
+      warn(`STALE CONTINUATION ${String(row.id).slice(0, 8)} conv=${String(row.conversation_id).slice(0, 8)} dropped: ${reason}`);
+    }
+    const listedRows = stmts.listRecoverableProcessing.all(params);
+    for (const row of listedRows.filter((entry) => entry?.consumed_at)) {
+      failConsumedRowTerminally(row, 'stale-recovery');
+      noteRestartTurnOutcome(row, 'cut');
+    }
+    const allRows = listedRows.filter((entry) => !entry?.consumed_at);
+    if (!allRows.length) return [];
+    // Enforce the retry budget before requeueing: failing these rows first
+    // flips them out of 'processing' so the recovery UPDATE below skips them.
+    const overBudget = allRows.filter((row) => Number(row?.retry_count || 0) + 1 >= maxRequeueRetries);
+    for (const row of overBudget) {
+      failRecoveredRowTerminally(row, 'stale-recovery');
+      noteRestartTurnOutcome(row, 'failed');
+    }
+    const dueRows = allRows.filter((row) => Number(row?.retry_count || 0) + 1 < maxRequeueRetries);
+    if (!dueRows.length) return [];
+
+    // Requeue exactly the listed rows, and announce only the ones the
+    // UPDATE took (its re-check of the rule is a safeguard; nothing runs
+    // between the listing and here).
+    const rows = [];
+    const tx = db.transaction(() => {
+      const now = new Date().toISOString();
+      for (const row of dueRows) {
+        if (stmts.recoverProcessingBefore.run({ ...params, id: row.id, requeueAt: requeueAtIso }).changes === 0) continue;
+        settleRecoveredAbortControls.run('queue-recovered', now, now, row.id);
+        rows.push(row);
+      }
+    });
+    tx();
+    if (!rows.length) return [];
+
+    for (const row of rows) {
+      emit('message_status', { messageId: row.id, conversationId: row.conversation_id, status: 'pending' });
+      noteRestartTurnOutcome(row, 'requeued');
+    }
+    emit('queue_updated', { recovered: rows.length });
+    return rows;
+  }
+
+  return { recoverProcessingOlderThan };
+}

package/server/services/subagent-timeline-service.mjs

@@ -140,7 +140,10 @@ export function createSubagentTimelineService({ db, stmts, now = () => new Date(
     if (hasFields && headerSupported()) {
       stmts.updateSubagentRunHeader.run({ id: runId, updatedAt: at, ...fields });
     }
-    if (status) {
+    if (status === 'running' && typeof stmts.reopenSubagentRun?.run === 'function') {
+      // Running (again, for a resumed agent): the end of an earlier run goes.
+      stmts.reopenSubagentRun.run(at, runId);
+    } else if (status) {
       const terminal = status === 'completed' || status === 'failed' || status === 'cancelled';
       stmts.updateSubagentRunStatus?.run?.(status, at, terminal ? at : null, runId);
     }