@oar-sh/oar 0.9.5 → 0.9.7

package/server/cursor-worker/cursor-sdk-adapter.mjs

@@ -1,6 +1,14 @@
 import path from 'node:path';
 import fs from 'node:fs';
 
+import {
+  createTurnLiveness,
+  createTurnStalledError,
+  describeTurnStall,
+  isTurnStalledError,
+  resolveTurnStallWindows,
+} from '../../shared/worker-runtime/turn-liveness.mjs';
+
 /**
  * The only module in the repo that imports `@cursor/sdk` (and its
  * `@cursor/sdk/sqlite` entry). All SDK access is injectable and the real
@@ -169,10 +177,16 @@ export function startCursorRun({
   // LocalSendOptions passthrough — `{ force: true }` is the SDK's documented
   // recovery for a local agent left wedged by a crashed process.
   sendLocal = null,
-  // Turn backstop: with no SDK traffic for this long (and no client-hosted
+  // Turn backstop: with no SDK traffic for too long (and no client-hosted
   // work pending, e.g. an open ask_user card) the run is failed as stalled
-  // instead of parking the merged iterator forever. 0 disables.
-  stallTimeoutMs = 120_000,
+  // instead of parking the merged iterator forever. How long depends on what
+  // the run is doing (shared/worker-runtime/turn-liveness.mjs):
+  // `stallTimeoutMs` until the run exists, the model window while it works,
+  // the tool window while a tool call is open. 0 disables.
+  stallTimeoutMs = undefined,
+  stallModelTimeoutMs = undefined,
+  stallToolTimeoutMs = undefined,
+  env = process.env,
   hasPendingClientWork = () => false,
   dbg = () => {},
 } = {}) {
@@ -181,8 +195,11 @@ export function startCursorRun({
   let failure = null;
   let notify = null;
   let run = null;
-  let lastEventAt = Date.now();
   let stallTimer = null;
+  const stallWindows = resolveTurnStallWindows({
+    env, idleMs: stallTimeoutMs, modelMs: stallModelTimeoutMs, toolMs: stallToolTimeoutMs,
+  });
+  const liveness = createTurnLiveness({ windows: stallWindows });
 
   const wake = () => {
     if (!notify) return;
@@ -190,9 +207,25 @@ export function startCursorRun({
     notify = null;
     resolve();
   };
+  // A tool call is reported on both surfaces (the delta hook and the message
+  // stream) under the same id; either one opens and closes it.
+  const observe = (event) => {
+    if (event.source === 'delta') {
+      const update = event.update || {};
+      liveness.traffic(update.type);
+      if (update.type === 'tool-call-started') liveness.begin('tool', update.callId, update.toolCall?.type);
+      else if (update.type === 'tool-call-completed') liveness.end('tool', update.callId);
+      return;
+    }
+    const message = event.message || {};
+    liveness.traffic(message.type);
+    if (message.type !== 'tool_call') return;
+    if (message.status === 'running') liveness.begin('tool', message.call_id, message.name);
+    else if (message.status) liveness.end('tool', message.call_id);
+  };
   const push = (event) => {
     if (ended) return;
-    lastEventAt = Date.now();
+    observe(event);
     queue.push(event);
     wake();
   };
@@ -216,21 +249,31 @@ export function startCursorRun({
     abortSignal.addEventListener('abort', onAbort, { once: true });
   }
 
-  if (!ended && Number(stallTimeoutMs) > 0) {
+  if (!ended && stallWindows.idle > 0) {
     stallTimer = setInterval(() => {
       if (ended) return;
       if (hasPendingClientWork()) {
         // A human is thinking about a question card; the clock restarts when
         // the tool settles and traffic resumes.
-        lastEventAt = Date.now();
+        liveness.traffic('client-work');
         return;
       }
-      if (Date.now() - lastEventAt >= Number(stallTimeoutMs)) {
-        failure = failure || new Error(`cursor turn stalled: no SDK traffic for ${Math.round(Number(stallTimeoutMs) / 1000)}s`);
-        dbg('cursor run stalled, ending merged stream');
-        end();
-      }
-    }, Math.min(15_000, Math.max(1_000, Math.floor(Number(stallTimeoutMs) / 4))));
+      const verdict = liveness.check();
+      if (!verdict.stalled) return;
+      failure = failure || createTurnStalledError({
+        agentLabel: 'Cursor',
+        quietMs: verdict.quietMs,
+        phase: verdict.phase,
+        tools: liveness.toolsInFlight(),
+      });
+      dbg('cursor run stalled, ending merged stream', liveness.describe());
+      // The run is still the agent's active one; left alone it made the next
+      // message wait for the busy retry.
+      Promise.resolve().then(() => run?.cancel?.()).catch((error) => {
+        dbg('cursor run cancel after a stall failed', error?.message || String(error));
+      });
+      end();
+    }, Math.min(15_000, Math.max(5, Math.floor(stallWindows.idle / 4))));
     stallTimer.unref?.();
   }
 
@@ -247,6 +290,17 @@ export function startCursorRun({
         ...(sendLocal && typeof sendLocal === 'object' ? { local: sendLocal } : {}),
         onDelta: ({ update }) => push({ source: 'delta', update }),
       });
+      if (ended) {
+        // Stalled (or aborted) before the run existed: nothing cancelled it
+        // then, and left active it makes the next message wait for it.
+        Promise.resolve().then(() => run?.cancel?.()).catch((error) => {
+          dbg('cursor run cancel after a late start failed', error?.message || String(error));
+        });
+        return;
+      }
+      // From here on the run is either waiting for the model or for a tool.
+      liveness.traffic('run-started');
+      liveness.begin('model', 'run');
       for await (const sdkMessage of run.stream()) {
         if (ended) break;
         push({ source: 'stream', message: sdkMessage });
@@ -314,6 +368,19 @@ export function classifyCursorError(error) {
   const message = String(error?.message || error);
   const isRetryable = error?.isRetryable === true;
   const name = String(error?.name || '');
+  if (isTurnStalledError(error)) {
+    return {
+      isAuth: false,
+      isBusy: false,
+      isStalled: true,
+      isRetryable,
+      code: 'turn-stalled',
+      stableCode: 'cursor.turn-stalled',
+      message,
+      // What the user reads; `message` is the log's wording.
+      text: describeTurnStall(error),
+    };
+  }
   if (name === 'AuthenticationError' || isCursorAuthErrorMessage(message)) {
     return {
       isAuth: true,

package/server/cursor-worker/cursor-turn-runner.mjs

@@ -778,13 +778,16 @@ export function createCursorTurnRunner({
       await publishResponse(message, {
         text: classified.isAuth
           ? `System note: the Cursor runtime could not authenticate (${classified.message}). Set or renew the Cursor API key in provider settings, then retry.`
-          : `System note: the Cursor turn failed (${classified.message}).`,
+          : classified.text || `System note: the Cursor turn failed (${classified.message}).`,
         model: null,
         terminalError: {
           kind: 'cursor-turn-failed',
           code: classified.code,
           stableCode: classified.stableCode,
-          message: classified.message,
+          // A stall is told in the user's terms, and with its own advice: the
+          // run was cancelled, so sending the message again is all it takes.
+          message: classified.text || classified.message,
+          ...(classified.isStalled ? { guidance: 'Send the message again to retry.' } : {}),
           failedAt: new Date().toISOString(),
           queueMessageId: String(message.id || '') || null,
         },

package/server/db-schema.mjs

@@ -11,6 +11,8 @@ import { rebuildRecentWorkspaceRootsTable } from './migrations/0002-recent-works
 import { ensurePushSubscriptionsTable } from './migrations/0003-push-subscriptions.mjs';
 import { migrateSteerSettleMarkers } from './migrations/0004-steer-settle-markers.mjs';
 import { migrateRemoteRelays } from './migrations/0005-remote-relays.mjs';
+import { migrateWorkspaceRootSettings } from './migrations/0006-workspace-root-settings.mjs';
+import { migrateClaudeCloud } from './migrations/0007-claude-cloud.mjs';
 import { migrateImageConversationSchema } from './repositories/image-conversation-repository.mjs';
 
 // Mirrors DEFAULT_RELAY_MODE in server-runtime.mjs; used only by the one-time
@@ -665,6 +667,13 @@ if (!queueColumns.includes('continuation_op_id')) {
   // after a lost response must find the row it already created, not mint a twin.
   db.exec(`ALTER TABLE queue ADD COLUMN continuation_op_id TEXT`);
 }
+if (!queueColumns.includes('usage_limit_pause')) {
+  // Set on the follow-up the relay queues when a Claude turn is refused at the
+  // subscription's usage limit (JSON: which window, its reset, when to send).
+  // While such a row is pending with next_attempt_at ahead, the turn is
+  // paused: it is held back, keeps no worker warm and blocks no restart.
+  db.exec(`ALTER TABLE queue ADD COLUMN usage_limit_pause TEXT`);
+}
 
 // recent_workspace_roots gained a case-normalized primary key (path_key). The
 // CREATE TABLE IF NOT EXISTS above only covers fresh databases, so upgrade an
@@ -710,6 +719,21 @@ try {
 } catch (error) {
   console.warn(`[remote-relays] migration 0005 failed; retrying on next boot: ${error?.message || error}`);
 }
+// Per-folder settings (commit attribution override). Readers tolerate the
+// table's absence, so a failure is logged and retried next boot like 0005.
+try {
+  migrateWorkspaceRootSettings(db);
+} catch (error) {
+  console.warn(`[workspace-root-settings] migration 0006 failed; retrying on next boot: ${error?.message || error}`);
+}
+// Claude Cloud conversations: the cloud session binding and what it clones.
+// Additive columns whose readers tolerate their absence, so a failure is
+// logged and retried next boot like 0006.
+try {
+  migrateClaudeCloud(db);
+} catch (error) {
+  console.warn(`[claude-cloud] migration 0007 failed; retrying on next boot: ${error?.message || error}`);
+}
 
 const runtimeSessionColumns = db.prepare(`PRAGMA table_info(runtime_sessions)`).all().map((c) => c.name);
 if (runtimeSessionColumns.length) {

package/server/grok-worker/acp-client.mjs

@@ -7,14 +7,24 @@ import { EventEmitter } from 'node:events';
 import { spawn } from 'node:child_process';
 import readline from 'node:readline';
 
+import {
+  createTurnLiveness,
+  createTurnStalledError,
+  resolveTurnStallWindows,
+} from '../../shared/worker-runtime/turn-liveness.mjs';
+
 function whichCommand(command = 'grok') {
   const value = String(command || 'grok').trim() || 'grok';
   return value;
 }
 
-// Prompt watchdog defaults: a turn that produces no ACP traffic for the
-// inactivity window (while no client-side work is pending) is stalled; the
-// ceiling bounds a single prompt absolutely. Both are overridable per call.
+// Prompt watchdog defaults: a turn that produces no ACP traffic for too long
+// (while no client-side work is pending) is stalled; the ceiling bounds a
+// single prompt absolutely. Both are overridable per call. How long "too
+// long" is depends on what the agent is doing
+// (shared/worker-runtime/turn-liveness.mjs): a prompt in flight is a model
+// request unless one of the agent's own tool calls is open, and the
+// inactivity window below is the base the two are derived from.
 export const ACP_PROMPT_INACTIVITY_MS = 120_000;
 export const ACP_PROMPT_MAX_TURN_MS = 1_800_000;
 // How long the watchdog waits for its `holdStall` question before treating
@@ -352,12 +362,31 @@ export class AcpClient extends EventEmitter {
    * `holdCheckTimeoutMs` counts as `false`.
    *
    * @param {object} [extra] additional session/prompt params (e.g. `_meta`)
-   * @param {object} [watchdog] { inactivityMs, maxTurnMs, hasPendingWork, holdStall, holdCheckTimeoutMs }
+   * @param {object} [watchdog] { inactivityMs, modelMs, toolMs, env, maxTurnMs, hasPendingWork, holdStall, holdCheckTimeoutMs }
    */
   async sessionPrompt(sessionId, prompt, onUpdate, extra = {}, watchdog = {}) {
-    const inactivityMs = Number(watchdog?.inactivityMs) > 0
-      ? Number(watchdog.inactivityMs)
-      : ACP_PROMPT_INACTIVITY_MS;
+    const resolved = resolveTurnStallWindows({
+      env: watchdog?.env ?? process.env,
+      idleMs: Number(watchdog?.inactivityMs) > 0 ? Number(watchdog.inactivityMs) : undefined,
+      modelMs: watchdog?.modelMs,
+      toolMs: watchdog?.toolMs,
+    });
+    // An environment that turns the shared watchdog off does not turn this
+    // one off: it is the only thing that notices a dead transport. An idle
+    // window of 0 takes the two others to 0 with it, so all three are
+    // rebuilt from the default. A window the caller named stays the
+    // caller's.
+    const stallWindows = resolved.idle > 0
+      ? resolved
+      : resolveTurnStallWindows({
+        env: {},
+        idleMs: ACP_PROMPT_INACTIVITY_MS,
+        modelMs: watchdog?.modelMs,
+        toolMs: watchdog?.toolMs,
+      });
+    const inactivityMs = stallWindows.idle;
+    const liveness = createTurnLiveness({ windows: stallWindows });
+    liveness.begin('model', 'prompt');
     // An explicit 0 means NO absolute ceiling (the user's "No limit"
     // setting); only an absent/invalid value falls back to the default. The
     // inactivity watchdog still catches dead transports either way.
@@ -375,7 +404,22 @@ export class AcpClient extends EventEmitter {
     const handler = (msg) => {
       if (msg.method !== 'session/update') return;
       if (msg.params?.sessionId && msg.params.sessionId !== sessionId) return;
-      onUpdate?.(msg.params?.update || msg.params);
+      const update = msg.params?.update || msg.params;
+      noteToolCall(update);
+      onUpdate?.(update);
+    };
+    // The agent's own tool calls (the ones it runs itself; client-hosted
+    // terminals are `hasPendingWork`). One that prints nothing sends nothing.
+    const noteToolCall = (update) => {
+      const kind = String(update?.sessionUpdate || update?.type || '').trim();
+      if (kind !== 'tool_call' && kind !== 'tool_call_update') return;
+      const id = String(update.toolCallId || update.tool_call_id || update.id || '').trim();
+      const status = String(update.status || (kind === 'tool_call' ? 'pending' : '')).trim().toLowerCase();
+      if (status === 'pending' || status === 'in_progress' || status === 'running') {
+        liveness.begin('tool', id, String(update.title || update.name || update.kind || '').trim());
+      } else if (status) {
+        liveness.end('tool', id);
+      }
     };
     this.on('notification', handler);
     let watchdogTimer = null;
@@ -386,10 +430,17 @@ export class AcpClient extends EventEmitter {
     const quietSince = () => Math.max(this.lastActivityAt, heldAt);
     const startedAt = Date.now();
     const stallPromise = new Promise((_, reject) => {
+      const quiet = () => liveness.check({ quietMs: Date.now() - quietSince() });
       const stall = () => {
         if (settled) return;
         try { this.sessionCancel(sessionId); } catch { /* ignore */ }
-        reject(new Error(`grok turn stalled: no ACP activity for ${Math.round(inactivityMs / 1000)}s`));
+        const verdict = quiet();
+        reject(createTurnStalledError({
+          agentLabel: 'Grok',
+          quietMs: verdict.quietMs,
+          phase: verdict.phase,
+          tools: liveness.toolsInFlight(),
+        }));
       };
       const askForHold = () => {
         let timer = null;
@@ -410,10 +461,15 @@ export class AcpClient extends EventEmitter {
               return;
             }
             // Traffic that arrived while the question ran re-armed it anyway.
-            if (Date.now() - quietSince() >= inactivityMs) stall();
+            if (quiet().stalled) stall();
           });
       };
-      const pollMs = Math.max(250, Math.min(5000, Math.floor(inactivityMs / 4)));
+      // Often enough for the shortest window that is set, not only for the
+      // idle one.
+      const shortestWindowMs = Math.min(
+        ...[stallWindows.idle, stallWindows.model, stallWindows.tool].filter((ms) => Number(ms) > 0),
+      );
+      const pollMs = Math.max(250, Math.min(5000, Math.floor(shortestWindowMs / 4)));
       watchdogTimer = setInterval(() => {
         const now = Date.now();
         if (maxTurnMs > 0 && now - startedAt >= maxTurnMs) {
@@ -422,7 +478,7 @@ export class AcpClient extends EventEmitter {
           return;
         }
         if (hasPendingWork()) return;
-        if (now - quietSince() < inactivityMs) return;
+        if (!quiet().stalled) return;
         if (!holdStall) {
           stall();
           return;

package/server/grok-worker/grok-sdk-adapter.mjs

@@ -5,6 +5,7 @@
 import { AcpClient, extractGrokModelsFromInitialize, pickAllowOnceOptionId } from './acp-client.mjs';
 import { createAcpHostServices } from './acp-host-services.mjs';
 import { createSdkMessageNormalizer } from './sdk-message-normalizer.mjs';
+import { describeTurnStall, isTurnStalledError } from '../../shared/worker-runtime/turn-liveness.mjs';
 
 export { extractGrokModelsFromInitialize };
 
@@ -44,6 +45,9 @@ export function classifyGrokError(error) {
     return {
       code: 'grok.turn-stalled',
       message: message || 'Grok turn stalled',
+      // The inactivity trip in the user's terms: how long, and what the agent
+      // was doing. The ceiling's own message already is.
+      ...(isTurnStalledError(error) ? { text: describeTurnStall(error) } : {}),
       isAuth: false,
       isBusy: false,
       isStalled: true,

package/server/grok-worker/grok-turn-runner.mjs

@@ -63,7 +63,10 @@ function buildGrokTerminalError(classified, message) {
     kind: 'grok-turn-failed',
     code: classified.code,
     stableCode: classified.code,
-    message: classified.message,
+    message: classified.text || classified.message,
+    // A stalled turn was cancelled and its agent is restarted for the next
+    // message, so sending the message again is all it takes.
+    ...(classified.isStalled ? { guidance: 'Send the message again to retry.' } : {}),
     failedAt: new Date().toISOString(),
     queueMessageId: String(message?.id || '') || null,
   };
@@ -584,7 +587,8 @@ export function createGrokTurnRunner({
       const noteText = classified.isAuth
         ? `System note: the Grok agent could not authenticate (${classified.message})`
         : classified.isStalled
-          ? `System note: the Grok turn stalled and was stopped (${classified.message}). Send a new message to retry.`
+          ? classified.text
+            || `System note: the Grok turn stalled and was stopped (${classified.message}). Send a new message to retry.`
           : `System note: the Grok turn failed (${classified.message}).`;
       await publishResponse(message, {
         text: noteText,

package/server/migrations/0006-workspace-root-settings.mjs

@@ -0,0 +1,67 @@
+'use strict';
+
+/**
+ * Migration: per-folder settings (`workspace_root_settings`).
+ *
+ * The first setting a repo folder can carry is the commit attribution mode
+ * of its Claude sessions (shared/claude-attribution.mjs): a repo that forbids
+ * co-author trailers, or wants its own, overrides the provider setting for
+ * every session that runs in it. Keyed like `recent_workspace_roots`
+ * (`normalizeWorkspaceRootKey`), but its own table: the MRU list is pruned.
+ *
+ * Additive and idempotent. Safe to run on every boot.
+ *
+ * Usage: node server/migrations/0006-workspace-root-settings.mjs [path/to/copilot.db]
+ */
+
+import { fileURLToPath } from 'url';
+import path from 'path';
+import process from 'process';
+import Database from 'better-sqlite3';
+
+function tableExists(db, name) {
+  return !!db
+    .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`)
+    .get(name);
+}
+
+export function migrateWorkspaceRootSettings(db) {
+  const existed = tableExists(db, 'workspace_root_settings');
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS workspace_root_settings (
+      path_key         TEXT PRIMARY KEY,
+      path             TEXT NOT NULL,
+      attribution_mode TEXT,
+      updated_at       TEXT NOT NULL
+    )
+  `);
+  return { applied: !existed, tableCreated: !existed };
+}
+
+function defaultDbPath() {
+  const here = path.dirname(fileURLToPath(import.meta.url));
+  return path.resolve(here, '..', 'data', 'copilot.db');
+}
+
+export function migrate(dbPath, options = {}) {
+  const db = new Database(dbPath, options);
+  try {
+    return migrateWorkspaceRootSettings(db);
+  } finally {
+    db.close();
+  }
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  const dbPath = process.argv[2] ? path.resolve(process.argv[2]) : defaultDbPath();
+  try {
+    const result = migrate(dbPath);
+    console.log(`Migration 0006 on ${dbPath}`);
+    console.log(result.applied ? '  workspace_root_settings table created' : '  skipped: nothing to change');
+    process.exit(0);
+  } catch (err) {
+    console.error(`Migration 0006 failed: ${err?.message || err}`);
+    process.exit(1);
+  }
+}

package/server/migrations/0007-claude-cloud.mjs

@@ -0,0 +1,82 @@
+'use strict';
+
+/**
+ * Migration: storage for Claude Cloud conversations (provider `claude-cloud`).
+ *
+ * 1. `runtime_sessions.claude_cloud_session_id` — the cloud session a
+ *    conversation runs in, reported by its worker after the first message
+ *    (the per-provider native id, next to `claude_native_session_id`,
+ *    `cursor_agent_id` and `grok_native_session_id`).
+ * 2. `runtime_sessions.claude_cloud_last_sequence` — the last event sequence
+ *    the worker has handled: where it resumes the event stream after a restart.
+ * 3. `runtime_sessions.claude_cloud_cost_usd` — what the session has cost so
+ *    far, as the cloud reports it.
+ * 4. `conversations.cloud_source_json` — what the session clones and where it
+ *    is shown: `{ repoUrl, branch, environmentId, sessionUrl,
+ *    pushedBranches: [{ branch, at }] }`.
+ *
+ * Additive and idempotent. Safe to run on every boot.
+ *
+ * Usage: node server/migrations/0007-claude-cloud.mjs [path/to/copilot.db]
+ */
+
+import { fileURLToPath } from 'url';
+import path from 'path';
+import process from 'process';
+import Database from 'better-sqlite3';
+
+const COLUMNS = Object.freeze([
+  Object.freeze({ table: 'runtime_sessions', name: 'claude_cloud_session_id', type: 'TEXT' }),
+  Object.freeze({ table: 'runtime_sessions', name: 'claude_cloud_last_sequence', type: 'TEXT' }),
+  Object.freeze({ table: 'runtime_sessions', name: 'claude_cloud_cost_usd', type: 'REAL' }),
+  Object.freeze({ table: 'conversations', name: 'cloud_source_json', type: 'TEXT' }),
+]);
+
+function tableExists(db, name) {
+  return !!db
+    .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`)
+    .get(name);
+}
+
+function columnNames(db, table) {
+  return new Set(db.prepare(`PRAGMA table_info(${table})`).all().map((column) => String(column.name)));
+}
+
+export function migrateClaudeCloud(db) {
+  let columnsAdded = 0;
+  for (const column of COLUMNS) {
+    if (!tableExists(db, column.table)) continue;
+    if (columnNames(db, column.table).has(column.name)) continue;
+    db.exec(`ALTER TABLE ${column.table} ADD COLUMN ${column.name} ${column.type}`);
+    columnsAdded += 1;
+  }
+  return { applied: columnsAdded > 0, columnsAdded };
+}
+
+function defaultDbPath() {
+  const here = path.dirname(fileURLToPath(import.meta.url));
+  return path.resolve(here, '..', 'data', 'copilot.db');
+}
+
+export function migrate(dbPath, options = {}) {
+  const db = new Database(dbPath, options);
+  try {
+    return migrateClaudeCloud(db);
+  } finally {
+    db.close();
+  }
+}
+
+const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+if (isMain) {
+  const dbPath = process.argv[2] ? path.resolve(process.argv[2]) : defaultDbPath();
+  try {
+    const result = migrate(dbPath);
+    console.log(`Migration 0007 on ${dbPath}`);
+    console.log(result.applied ? `  claude cloud columns added: ${result.columnsAdded}` : '  skipped: nothing to change');
+    process.exit(0);
+  } catch (err) {
+    console.error(`Migration 0007 failed: ${err?.message || err}`);
+    process.exit(1);
+  }
+}