@oar-sh/oar 0.9.5 → 0.9.6

package/server/cursor-worker/cursor-sdk-adapter.mjs

@@ -1,6 +1,14 @@
 import path from 'node:path';
 import fs from 'node:fs';
 
+import {
+  createTurnLiveness,
+  createTurnStalledError,
+  describeTurnStall,
+  isTurnStalledError,
+  resolveTurnStallWindows,
+} from '../../shared/worker-runtime/turn-liveness.mjs';
+
 /**
  * The only module in the repo that imports `@cursor/sdk` (and its
  * `@cursor/sdk/sqlite` entry). All SDK access is injectable and the real
@@ -169,10 +177,16 @@ export function startCursorRun({
   // LocalSendOptions passthrough — `{ force: true }` is the SDK's documented
   // recovery for a local agent left wedged by a crashed process.
   sendLocal = null,
-  // Turn backstop: with no SDK traffic for this long (and no client-hosted
+  // Turn backstop: with no SDK traffic for too long (and no client-hosted
   // work pending, e.g. an open ask_user card) the run is failed as stalled
-  // instead of parking the merged iterator forever. 0 disables.
-  stallTimeoutMs = 120_000,
+  // instead of parking the merged iterator forever. How long depends on what
+  // the run is doing (shared/worker-runtime/turn-liveness.mjs):
+  // `stallTimeoutMs` until the run exists, the model window while it works,
+  // the tool window while a tool call is open. 0 disables.
+  stallTimeoutMs = undefined,
+  stallModelTimeoutMs = undefined,
+  stallToolTimeoutMs = undefined,
+  env = process.env,
   hasPendingClientWork = () => false,
   dbg = () => {},
 } = {}) {
@@ -181,8 +195,11 @@ export function startCursorRun({
   let failure = null;
   let notify = null;
   let run = null;
-  let lastEventAt = Date.now();
   let stallTimer = null;
+  const stallWindows = resolveTurnStallWindows({
+    env, idleMs: stallTimeoutMs, modelMs: stallModelTimeoutMs, toolMs: stallToolTimeoutMs,
+  });
+  const liveness = createTurnLiveness({ windows: stallWindows });
 
   const wake = () => {
     if (!notify) return;
@@ -190,9 +207,25 @@ export function startCursorRun({
     notify = null;
     resolve();
   };
+  // A tool call is reported on both surfaces (the delta hook and the message
+  // stream) under the same id; either one opens and closes it.
+  const observe = (event) => {
+    if (event.source === 'delta') {
+      const update = event.update || {};
+      liveness.traffic(update.type);
+      if (update.type === 'tool-call-started') liveness.begin('tool', update.callId, update.toolCall?.type);
+      else if (update.type === 'tool-call-completed') liveness.end('tool', update.callId);
+      return;
+    }
+    const message = event.message || {};
+    liveness.traffic(message.type);
+    if (message.type !== 'tool_call') return;
+    if (message.status === 'running') liveness.begin('tool', message.call_id, message.name);
+    else if (message.status) liveness.end('tool', message.call_id);
+  };
   const push = (event) => {
     if (ended) return;
-    lastEventAt = Date.now();
+    observe(event);
     queue.push(event);
     wake();
   };
@@ -216,21 +249,31 @@ export function startCursorRun({
     abortSignal.addEventListener('abort', onAbort, { once: true });
   }
 
-  if (!ended && Number(stallTimeoutMs) > 0) {
+  if (!ended && stallWindows.idle > 0) {
     stallTimer = setInterval(() => {
       if (ended) return;
       if (hasPendingClientWork()) {
         // A human is thinking about a question card; the clock restarts when
         // the tool settles and traffic resumes.
-        lastEventAt = Date.now();
+        liveness.traffic('client-work');
         return;
       }
-      if (Date.now() - lastEventAt >= Number(stallTimeoutMs)) {
-        failure = failure || new Error(`cursor turn stalled: no SDK traffic for ${Math.round(Number(stallTimeoutMs) / 1000)}s`);
-        dbg('cursor run stalled, ending merged stream');
-        end();
-      }
-    }, Math.min(15_000, Math.max(1_000, Math.floor(Number(stallTimeoutMs) / 4))));
+      const verdict = liveness.check();
+      if (!verdict.stalled) return;
+      failure = failure || createTurnStalledError({
+        agentLabel: 'Cursor',
+        quietMs: verdict.quietMs,
+        phase: verdict.phase,
+        tools: liveness.toolsInFlight(),
+      });
+      dbg('cursor run stalled, ending merged stream', liveness.describe());
+      // The run is still the agent's active one; left alone it made the next
+      // message wait for the busy retry.
+      Promise.resolve().then(() => run?.cancel?.()).catch((error) => {
+        dbg('cursor run cancel after a stall failed', error?.message || String(error));
+      });
+      end();
+    }, Math.min(15_000, Math.max(5, Math.floor(stallWindows.idle / 4))));
     stallTimer.unref?.();
   }
 
@@ -247,6 +290,17 @@ export function startCursorRun({
         ...(sendLocal && typeof sendLocal === 'object' ? { local: sendLocal } : {}),
         onDelta: ({ update }) => push({ source: 'delta', update }),
       });
+      if (ended) {
+        // Stalled (or aborted) before the run existed: nothing cancelled it
+        // then, and left active it makes the next message wait for it.
+        Promise.resolve().then(() => run?.cancel?.()).catch((error) => {
+          dbg('cursor run cancel after a late start failed', error?.message || String(error));
+        });
+        return;
+      }
+      // From here on the run is either waiting for the model or for a tool.
+      liveness.traffic('run-started');
+      liveness.begin('model', 'run');
       for await (const sdkMessage of run.stream()) {
         if (ended) break;
         push({ source: 'stream', message: sdkMessage });
@@ -314,6 +368,19 @@ export function classifyCursorError(error) {
   const message = String(error?.message || error);
   const isRetryable = error?.isRetryable === true;
   const name = String(error?.name || '');
+  if (isTurnStalledError(error)) {
+    return {
+      isAuth: false,
+      isBusy: false,
+      isStalled: true,
+      isRetryable,
+      code: 'turn-stalled',
+      stableCode: 'cursor.turn-stalled',
+      message,
+      // What the user reads; `message` is the log's wording.
+      text: describeTurnStall(error),
+    };
+  }
   if (name === 'AuthenticationError' || isCursorAuthErrorMessage(message)) {
     return {
       isAuth: true,

package/server/cursor-worker/cursor-turn-runner.mjs

@@ -778,13 +778,16 @@ export function createCursorTurnRunner({
       await publishResponse(message, {
         text: classified.isAuth
           ? `System note: the Cursor runtime could not authenticate (${classified.message}). Set or renew the Cursor API key in provider settings, then retry.`
-          : `System note: the Cursor turn failed (${classified.message}).`,
+          : classified.text || `System note: the Cursor turn failed (${classified.message}).`,
         model: null,
         terminalError: {
           kind: 'cursor-turn-failed',
           code: classified.code,
           stableCode: classified.stableCode,
-          message: classified.message,
+          // A stall is told in the user's terms, and with its own advice: the
+          // run was cancelled, so sending the message again is all it takes.
+          message: classified.text || classified.message,
+          ...(classified.isStalled ? { guidance: 'Send the message again to retry.' } : {}),
           failedAt: new Date().toISOString(),
           queueMessageId: String(message.id || '') || null,
         },

package/server/db-schema.mjs

@@ -665,6 +665,13 @@ if (!queueColumns.includes('continuation_op_id')) {
   // after a lost response must find the row it already created, not mint a twin.
   db.exec(`ALTER TABLE queue ADD COLUMN continuation_op_id TEXT`);
 }
+if (!queueColumns.includes('usage_limit_pause')) {
+  // Set on the follow-up the relay queues when a Claude turn is refused at the
+  // subscription's usage limit (JSON: which window, its reset, when to send).
+  // While such a row is pending with next_attempt_at ahead, the turn is
+  // paused: it is held back, keeps no worker warm and blocks no restart.
+  db.exec(`ALTER TABLE queue ADD COLUMN usage_limit_pause TEXT`);
+}
 
 // recent_workspace_roots gained a case-normalized primary key (path_key). The
 // CREATE TABLE IF NOT EXISTS above only covers fresh databases, so upgrade an

package/server/grok-worker/acp-client.mjs

@@ -7,14 +7,24 @@ import { EventEmitter } from 'node:events';
 import { spawn } from 'node:child_process';
 import readline from 'node:readline';
 
+import {
+  createTurnLiveness,
+  createTurnStalledError,
+  resolveTurnStallWindows,
+} from '../../shared/worker-runtime/turn-liveness.mjs';
+
 function whichCommand(command = 'grok') {
   const value = String(command || 'grok').trim() || 'grok';
   return value;
 }
 
-// Prompt watchdog defaults: a turn that produces no ACP traffic for the
-// inactivity window (while no client-side work is pending) is stalled; the
-// ceiling bounds a single prompt absolutely. Both are overridable per call.
+// Prompt watchdog defaults: a turn that produces no ACP traffic for too long
+// (while no client-side work is pending) is stalled; the ceiling bounds a
+// single prompt absolutely. Both are overridable per call. How long "too
+// long" is depends on what the agent is doing
+// (shared/worker-runtime/turn-liveness.mjs): a prompt in flight is a model
+// request unless one of the agent's own tool calls is open, and the
+// inactivity window below is the base the two are derived from.
 export const ACP_PROMPT_INACTIVITY_MS = 120_000;
 export const ACP_PROMPT_MAX_TURN_MS = 1_800_000;
 // How long the watchdog waits for its `holdStall` question before treating
@@ -352,12 +362,31 @@ export class AcpClient extends EventEmitter {
    * `holdCheckTimeoutMs` counts as `false`.
    *
    * @param {object} [extra] additional session/prompt params (e.g. `_meta`)
-   * @param {object} [watchdog] { inactivityMs, maxTurnMs, hasPendingWork, holdStall, holdCheckTimeoutMs }
+   * @param {object} [watchdog] { inactivityMs, modelMs, toolMs, env, maxTurnMs, hasPendingWork, holdStall, holdCheckTimeoutMs }
    */
   async sessionPrompt(sessionId, prompt, onUpdate, extra = {}, watchdog = {}) {
-    const inactivityMs = Number(watchdog?.inactivityMs) > 0
-      ? Number(watchdog.inactivityMs)
-      : ACP_PROMPT_INACTIVITY_MS;
+    const resolved = resolveTurnStallWindows({
+      env: watchdog?.env ?? process.env,
+      idleMs: Number(watchdog?.inactivityMs) > 0 ? Number(watchdog.inactivityMs) : undefined,
+      modelMs: watchdog?.modelMs,
+      toolMs: watchdog?.toolMs,
+    });
+    // An environment that turns the shared watchdog off does not turn this
+    // one off: it is the only thing that notices a dead transport. An idle
+    // window of 0 takes the two others to 0 with it, so all three are
+    // rebuilt from the default. A window the caller named stays the
+    // caller's.
+    const stallWindows = resolved.idle > 0
+      ? resolved
+      : resolveTurnStallWindows({
+        env: {},
+        idleMs: ACP_PROMPT_INACTIVITY_MS,
+        modelMs: watchdog?.modelMs,
+        toolMs: watchdog?.toolMs,
+      });
+    const inactivityMs = stallWindows.idle;
+    const liveness = createTurnLiveness({ windows: stallWindows });
+    liveness.begin('model', 'prompt');
     // An explicit 0 means NO absolute ceiling (the user's "No limit"
     // setting); only an absent/invalid value falls back to the default. The
     // inactivity watchdog still catches dead transports either way.
@@ -375,7 +404,22 @@ export class AcpClient extends EventEmitter {
     const handler = (msg) => {
       if (msg.method !== 'session/update') return;
       if (msg.params?.sessionId && msg.params.sessionId !== sessionId) return;
-      onUpdate?.(msg.params?.update || msg.params);
+      const update = msg.params?.update || msg.params;
+      noteToolCall(update);
+      onUpdate?.(update);
+    };
+    // The agent's own tool calls (the ones it runs itself; client-hosted
+    // terminals are `hasPendingWork`). One that prints nothing sends nothing.
+    const noteToolCall = (update) => {
+      const kind = String(update?.sessionUpdate || update?.type || '').trim();
+      if (kind !== 'tool_call' && kind !== 'tool_call_update') return;
+      const id = String(update.toolCallId || update.tool_call_id || update.id || '').trim();
+      const status = String(update.status || (kind === 'tool_call' ? 'pending' : '')).trim().toLowerCase();
+      if (status === 'pending' || status === 'in_progress' || status === 'running') {
+        liveness.begin('tool', id, String(update.title || update.name || update.kind || '').trim());
+      } else if (status) {
+        liveness.end('tool', id);
+      }
     };
     this.on('notification', handler);
     let watchdogTimer = null;
@@ -386,10 +430,17 @@ export class AcpClient extends EventEmitter {
     const quietSince = () => Math.max(this.lastActivityAt, heldAt);
     const startedAt = Date.now();
     const stallPromise = new Promise((_, reject) => {
+      const quiet = () => liveness.check({ quietMs: Date.now() - quietSince() });
       const stall = () => {
         if (settled) return;
         try { this.sessionCancel(sessionId); } catch { /* ignore */ }
-        reject(new Error(`grok turn stalled: no ACP activity for ${Math.round(inactivityMs / 1000)}s`));
+        const verdict = quiet();
+        reject(createTurnStalledError({
+          agentLabel: 'Grok',
+          quietMs: verdict.quietMs,
+          phase: verdict.phase,
+          tools: liveness.toolsInFlight(),
+        }));
       };
       const askForHold = () => {
         let timer = null;
@@ -410,10 +461,15 @@ export class AcpClient extends EventEmitter {
               return;
             }
             // Traffic that arrived while the question ran re-armed it anyway.
-            if (Date.now() - quietSince() >= inactivityMs) stall();
+            if (quiet().stalled) stall();
           });
       };
-      const pollMs = Math.max(250, Math.min(5000, Math.floor(inactivityMs / 4)));
+      // Often enough for the shortest window that is set, not only for the
+      // idle one.
+      const shortestWindowMs = Math.min(
+        ...[stallWindows.idle, stallWindows.model, stallWindows.tool].filter((ms) => Number(ms) > 0),
+      );
+      const pollMs = Math.max(250, Math.min(5000, Math.floor(shortestWindowMs / 4)));
       watchdogTimer = setInterval(() => {
         const now = Date.now();
         if (maxTurnMs > 0 && now - startedAt >= maxTurnMs) {
@@ -422,7 +478,7 @@ export class AcpClient extends EventEmitter {
           return;
         }
         if (hasPendingWork()) return;
-        if (now - quietSince() < inactivityMs) return;
+        if (!quiet().stalled) return;
         if (!holdStall) {
           stall();
           return;

package/server/grok-worker/grok-sdk-adapter.mjs

@@ -5,6 +5,7 @@
 import { AcpClient, extractGrokModelsFromInitialize, pickAllowOnceOptionId } from './acp-client.mjs';
 import { createAcpHostServices } from './acp-host-services.mjs';
 import { createSdkMessageNormalizer } from './sdk-message-normalizer.mjs';
+import { describeTurnStall, isTurnStalledError } from '../../shared/worker-runtime/turn-liveness.mjs';
 
 export { extractGrokModelsFromInitialize };
 
@@ -44,6 +45,9 @@ export function classifyGrokError(error) {
     return {
       code: 'grok.turn-stalled',
       message: message || 'Grok turn stalled',
+      // The inactivity trip in the user's terms: how long, and what the agent
+      // was doing. The ceiling's own message already is.
+      ...(isTurnStalledError(error) ? { text: describeTurnStall(error) } : {}),
       isAuth: false,
       isBusy: false,
       isStalled: true,

package/server/grok-worker/grok-turn-runner.mjs

@@ -63,7 +63,10 @@ function buildGrokTerminalError(classified, message) {
     kind: 'grok-turn-failed',
     code: classified.code,
     stableCode: classified.code,
-    message: classified.message,
+    message: classified.text || classified.message,
+    // A stalled turn was cancelled and its agent is restarted for the next
+    // message, so sending the message again is all it takes.
+    ...(classified.isStalled ? { guidance: 'Send the message again to retry.' } : {}),
     failedAt: new Date().toISOString(),
     queueMessageId: String(message?.id || '') || null,
   };
@@ -584,7 +587,8 @@ export function createGrokTurnRunner({
       const noteText = classified.isAuth
         ? `System note: the Grok agent could not authenticate (${classified.message})`
         : classified.isStalled
-          ? `System note: the Grok turn stalled and was stopped (${classified.message}). Send a new message to retry.`
+          ? classified.text
+            || `System note: the Grok turn stalled and was stopped (${classified.message}). Send a new message to retry.`
           : `System note: the Grok turn failed (${classified.message}).`;
       await publishResponse(message, {
         text: noteText,

package/server/public/app/api-client.js

@@ -972,6 +972,20 @@ export async function cancelHostSuspend(body = {}) {
   });
 }
 
+export async function resumeUsageLimitPause(conversationId) {
+  return apiFetch('/api/usage-limit/resume', {
+    method: 'POST',
+    body: JSON.stringify({ conversationId }),
+  });
+}
+
+export async function cancelUsageLimitPause(conversationId) {
+  return apiFetch('/api/usage-limit/cancel', {
+    method: 'POST',
+    body: JSON.stringify({ conversationId }),
+  });
+}
+
 export async function cancelRelayRestart(body = {}) {
   return apiFetch('/api/relay/shutdown/cancel', {
     method: 'POST',
@@ -1091,8 +1105,9 @@ export async function answerRelayQuestionStructured(questionId, structuredAnswer
   }
 }
 
-export async function loadRelayBoards(status = 'pending') {
-  return apiFetch(`/api/relay-boards?status=${encodeURIComponent(status)}`);
+export async function loadRelayBoards(status = 'pending', conversationId = '') {
+  const conversation = String(conversationId || '').trim();
+  return apiFetch(`/api/relay-boards?status=${encodeURIComponent(status)}${conversation ? `&conversationId=${encodeURIComponent(conversation)}` : ''}`);
 }
 
 export async function loadRelayBoard(boardId) {

package/server/public/app/bootstrap.js

@@ -312,6 +312,7 @@ import {
   cancelQueuedHostSuspend,
 } from './action-confirmations.js';
 import { applyPendingActionsFromStatus, initHostSuspendUi } from './host-suspend-ui.js';
+import { applyUsageLimitFromStatus, initUsageLimitUi } from './usage-limit-ui.js';
 import {
   startClaudeRelogin,
   submitClaudeLoginCodeFromInput,
@@ -3250,6 +3251,7 @@ async function refreshSessionWorkerStatus() {
   if (!status) return;
   syncQueueStatusMenuEntry(status);
   applyPendingActionsFromStatus(status);
+  applyUsageLimitFromStatus(status);
   if (setSessionWorkerStatesFromStatusPayload(status.sessionWorker)) {
     renderConvList();
     // A worker steering snapshot may have changed; the composer's Steer
@@ -3959,6 +3961,7 @@ async function initApp() {
   if (!sharedMode) {
     syncSuspendHostVisibility();
     initHostSuspendUi();
+    initUsageLimitUi();
   }
   setupViewportTracking();
   bindChatSelectionGuard();

package/server/public/app/composer-control-state.mjs

@@ -27,6 +27,7 @@ const STEERING_HOLD_TITLES = Object.freeze({
   compaction: 'Will steer in once the conversation finishes compacting',
   adoption: 'Will steer in once the compaction recovery finishes',
   delivery: 'Will steer in once the current delivery lands',
+  tool: 'Will steer in once the running tool call finishes',
 });
 
 export function deriveComposerControlState({
@@ -41,9 +42,9 @@ export function deriveComposerControlState({
   // not "Queue". Workers that do not advertise it keep the queue wording.
   steeringSupported = false,
   // The worker reports steering as momentarily held (open question card or
-  // plan approval, compaction, post-compaction adoption). A send stays allowed
-  // and reads "Queue": the relay holds the message and it steers in once the
-  // hold clears.
+  // plan approval, compaction, post-compaction adoption, a running tool call
+  // that a steer would cut short). A send stays allowed and reads "Queue":
+  // the relay holds the message and it steers in once the hold clears.
   steeringHeld = false,
   steeringHoldReason = null,
 } = {}) {