@oar-sh/oar 0.9.23 → 0.9.24

package/server/services/worker-launch-note-service.mjs

@@ -18,6 +18,7 @@
 import { randomUUID } from 'node:crypto';
 
 export const WORKER_LAUNCH_NOTE_KINDS = Object.freeze({
+  installing: 'worker-launch-installing',
   failed: 'worker-launch-failed',
   stopped: 'worker-launch-stopped',
   started: 'worker-launch-started',
@@ -47,6 +48,11 @@ export function buildWorkerLaunchNoteText({ type, lifecycle = {}, retryEveryMs =
   const tries = Math.max(0, Number(lifecycle?.retryCount || 0));
   const triesText = `${tries} ${tries === 1 ? 'try' : 'tries'}`;
   const cause = clipCause(lifecycle?.lastError);
+  if (type === 'installing') {
+    // The first chat of a provider whose agent SDK is not installed yet
+    // (agent-sdk-store.mjs): the relay installs it before the worker starts.
+    return `⏳ Installing the ${lifecycle?.sdkLabel || 'agent SDK'} on the relay host, once for this version. Your message is delivered when it is ready.`;
+  }
   if (type === 'launched') {
     return `✅ The worker for this conversation started at ${formatTime(lifecycle?.launchAt || new Date().toISOString())}. The queued message is being delivered.`;
   }
@@ -77,6 +83,7 @@ export function createWorkerLaunchNoteService({
   const episodes = new Map();
 
   function kindFor(type) {
+    if (type === 'installing') return WORKER_LAUNCH_NOTE_KINDS.installing;
     if (type === 'launched') return WORKER_LAUNCH_NOTE_KINDS.started;
     if (type === 'stopped') return WORKER_LAUNCH_NOTE_KINDS.stopped;
     return WORKER_LAUNCH_NOTE_KINDS.failed;
@@ -105,7 +112,7 @@ export function createWorkerLaunchNoteService({
       stmts.updateConvTime?.run?.(at, conversationId);
     });
     write();
-    episodes.set(sdkSessionId, { messageId, conversationId, timestamp: at });
+    episodes.set(sdkSessionId, { messageId, conversationId, timestamp: at, kind, text });
     try { emit('assistant_message', { conversationId, messageId, message: messagePayload({ messageId, text, kind, timestamp: at }) }); } catch {}
     try { logger.log?.(`[worker-launch] conv=${conversationId.slice(0, 8)} note opened (${type}): ${clipCause(lifecycle?.lastError)}`); } catch {}
   }
@@ -113,6 +120,11 @@ export function createWorkerLaunchNoteService({
   function update(episode, sdkSessionId, { type, lifecycle }) {
     const kind = kindFor(type);
     const text = buildWorkerLaunchNoteText({ type, lifecycle, retryEveryMs, retryWindowMs, formatTime });
+    // The same note again (a launch deferred on every prime while an agent
+    // SDK installs): no write, no broadcast.
+    if (episode.kind === kind && episode.text === text) return;
+    episode.kind = kind;
+    episode.text = text;
     updateNote.run(text, kind, episode.messageId);
     try {
       emit('message_updated', {
@@ -130,7 +142,7 @@ export function createWorkerLaunchNoteService({
   /** The supervisor's onLaunchState listener. */
   function handle({ type, sdkSessionId, lifecycle = null } = {}) {
     const sid = String(sdkSessionId || '').trim();
-    if (!sid || !['exhausted', 'retry-failed', 'stopped', 'launched'].includes(type)) return false;
+    if (!sid || !['installing', 'exhausted', 'retry-failed', 'stopped', 'launched'].includes(type)) return false;
     const episode = episodes.get(sid) || null;
     if (episode) {
       update(episode, sid, { type, lifecycle });

package/server/services/worker-recycle-service.mjs

@@ -27,6 +27,31 @@
  * `quietMs` without a delivery or a turn end for the session, the runners'
  * own idle time.
  *
+ * A stale worker on an agent SDK the store installs (`isAgentSdkWorker`: the
+ * claude, claude-api, deepseek and cursor kinds, unless a git checkout runs
+ * that SDK from its own node_modules) is the exception. An update moves the
+ * SDKs out of the package, so the files such a worker would start its next
+ * CLI child from may be gone, and a message it gets once its child has idled
+ * out fails. So:
+ *
+ * - it skips the 'cli' and 'quiet' holds and is recycled as soon as its turn,
+ *   question and compaction holds (and the processing row) allow;
+ * - its background hold stays a hold, and is the one hold it is not fenced
+ *   for: background tasks run in the CLI child, so while they do the child is
+ *   up and a delivery goes to it, never to a new child from the gone files;
+ * - from the first sweep that sees it stale until its recycle, the
+ *   replacement fence (below) keeps every delivery away from it, so what is
+ *   queued waits for the new worker's cold start rather than failing on the
+ *   old one;
+ * - a relay that has just started knows its holds only from its first
+ *   heartbeat's steering snapshot: until that comes it waits ('steering'),
+ *   fenced, for at most `steeringWaitMs`; one that still has sent none by
+ *   then is judged as any other worker, unfenced.
+ *
+ * Besides the periodic sweep, a sweep runs at start and on every worker hello
+ * (`sweepNow`), so a stale worker that reconnects to a new relay is fenced
+ * before the hello's own request for work is answered.
+ *
  * From the moment a recycle passes its holds until its launch is done, the
  * replacement fence (worker-replacement.mjs) keeps every delivery away from
  * the process being replaced, and the stop and the launch run under the
@@ -61,9 +86,10 @@ const LOG_TAG = '[worker-recycle]';
  * the worker's own release holds (workerReleaseHold; 'turn' also for a
  * processing row the session owns), 'cli' while its CLI child runs, and
  * 'quiet' while a worker that does not report its CLI child has had a
- * delivery or a turn end within `quietMs`.
+ * delivery or a turn end within `quietMs`, and 'steering' while a worker on
+ * an agent SDK has not yet sent the snapshot that names its holds.
  */
-export const RECYCLE_HOLDS = Object.freeze(['turn', 'question', 'compaction', 'background', 'cli', 'quiet']);
+export const RECYCLE_HOLDS = Object.freeze(['turn', 'question', 'compaction', 'background', 'cli', 'quiet', 'steering']);
 const RECYCLABLE_STATUSES = new Set(['ready', 'processing']);
 
 function sid8(sid) {
@@ -155,6 +181,15 @@ export function createWorkerRecycleService({
   // without a delivery or a turn end: the Claude and Copilot runners'
   // idleShutdownMs, after which they stop the child themselves.
   quietMs = 10 * 60_000,
+  // worker → whether its kind runs on an agent SDK (agentSdkForWorkerKind):
+  // such a worker, once stale, skips the cli and quiet holds and is fenced
+  // until its recycle (see the header).
+  isAgentSdkWorker = () => false,
+  // How long a stale worker on an agent SDK may go without a steering
+  // snapshot (from when this service first saw its process) before it is
+  // judged as any other worker. Its heartbeat sends one within a second of
+  // a relay's hello.
+  steeringWaitMs = 30_000,
   logger = console,
 } = {}) {
   if (typeof stopWorker !== 'function' || typeof launchWorker !== 'function') {
@@ -214,8 +249,13 @@ export function createWorkerRecycleService({
         pendingLaunch: null,
         lastWaitLogAt: -Infinity,
         // When this service first saw the session's current pid: the quiet
-        // window's start for a worker with no start time and no queue rows.
+        // window's start for a worker with no start time and no queue rows,
+        // and the steering wait's start.
         firstSeen: null,
+        // The pid this service fenced as a stale worker on an agent SDK
+        // waiting for its recycle (sdkFenceUp), or null. A recycle admitted
+        // takes the fence over and clears this.
+        sdkFence: null,
       };
       bySession.set(sid, state);
     }
@@ -394,8 +434,18 @@ export function createWorkerRecycleService({
       });
     } finally {
       // A stop that did not go through (or never ran) leaves the old worker
-      // serving its session: it takes work again at once.
-      if (!stopped) fence.release(sid, pid);
+      // serving its session: it takes work again at once. Not one on an
+      // agent SDK the update took away: its fence stays up through the
+      // stop's backoff, until the worker goes or the tries reach stuckAfter
+      // (the sweep keeps it, sdkFenced), so no queued row goes to it.
+      if (!stopped) {
+        if (isAgentSdkWorker(worker) === true && state.failures > 0 && state.failures < stuckAfter && stillSameProcess(sid, pid)) {
+          state.sdkFence = pid;
+          log(`fence kept ${sid8(sid)} pid ${pid}: its stop failed, deliveries wait for the retry`);
+        } else {
+          fence.release(sid, pid);
+        }
+      }
     }
   }
 
@@ -454,17 +504,68 @@ export function createWorkerRecycleService({
    * worker's start, else when this service first saw the process. Throws
    * when a probe does; the caller counts that as a probe failure.
    */
-  function holdFor(sid, worker, state) {
+  function holdFor(sid, worker, state, sdkMode = null) {
     if (hasRunningTurn(sid) === true) return 'turn';
+    if (sdkMode === 'steering') return 'steering';
     const releaseHold = workerReleaseHold(worker);
     if (releaseHold) return releaseHold;
+    // A stale worker on an agent SDK goes as soon as its own holds allow: its
+    // next CLI child may have no files left to start from.
+    if (sdkMode === 'fast') return null;
     const cliAlive = worker?.steering?.cliAlive;
     if (cliAlive === true) return 'cli';
     if (cliAlive === false) return null;
+    const since = toTimeMs(quietSince(sid)) ?? toTimeMs(worker.startedAt) ?? firstSeenAt(state, worker);
+    return now() - since < quietMs ? 'quiet' : null;
+  }
+
+  // When this service first saw the worker's current process.
+  function firstSeenAt(state, worker) {
     const pid = toPid(worker.pid);
     if (state.firstSeen?.pid !== pid) state.firstSeen = { pid, at: now() };
-    const since = toTimeMs(quietSince(sid)) ?? toTimeMs(worker.startedAt) ?? state.firstSeen.at;
-    return now() - since < quietMs ? 'quiet' : null;
+    return state.firstSeen.at;
+  }
+
+  /**
+   * How a stale worker on an agent SDK is judged: 'fast' (its steering
+   * snapshot names its holds: no cli or quiet hold, fenced until the
+   * recycle), 'steering' (no snapshot yet, within `steeringWaitMs` of when
+   * this service first saw the process: wait for it, fenced), or null (not on
+   * an agent SDK, or a snapshot never came: as any other worker). Throws
+   * when the predicate does; the caller counts that as a probe failure.
+   */
+  function sdkModeFor(worker, state) {
+    if (isAgentSdkWorker(worker) !== true) return null;
+    if (worker?.steering && typeof worker.steering === 'object') return 'fast';
+    return now() - firstSeenAt(state, worker) < steeringWaitMs ? 'steering' : null;
+  }
+
+  // Raises the fence for a stale worker on an agent SDK until its recycle:
+  // whatever is queued for the session waits for the new worker.
+  function sdkFenceUp(sid, pid, state, confirmed) {
+    confirmed.add(sid);
+    if (state.sdkFence === pid && fence.isHeld(sid, pid)) return;
+    fence.hold(sid, pid);
+    if (state.sdkFence !== pid) log(`fenced ${sid8(sid)} pid ${pid}: stale on an agent SDK, deliveries wait for the recycle`);
+    state.sdkFence = pid;
+  }
+
+  // A recycle (or an archived stop) admitted for the session owns the fence
+  // from here: it lowers it when it is done.
+  function sdkFenceHandOver(state) {
+    state.sdkFence = null;
+  }
+
+  // Lowers the fences this service raised for workers no longer judged
+  // fenced by this sweep: gone, recycled, in a background hold, a probe that
+  // failed, a snapshot that never came.
+  function sdkFencesDown(confirmed) {
+    for (const [sid, state] of bySession) {
+      if (state.sdkFence === null || confirmed.has(sid)) continue;
+      const pid = state.sdkFence;
+      state.sdkFence = null;
+      if (fence.release(sid, pid)) log(`unfenced ${sid8(sid)} pid ${pid}`);
+    }
   }
 
   function noteWaiting(sid, state, hold, nextWaiting) {
@@ -509,6 +610,8 @@ export function createWorkerRecycleService({
       return;
     }
     const running = new Set();
+    // The sessions whose agent-SDK fence this sweep keeps up (sdkFenceUp).
+    const sdkFenced = new Set();
     for (const worker of workers) {
       const sid = String(worker?.sdkSessionId || '').trim();
       if (!sid) continue;
@@ -527,36 +630,51 @@ export function createWorkerRecycleService({
         continue;
       }
       const state = stateFor(sid);
-      if (state.inFlight || now() < state.nextAttemptAt) continue;
+      if (state.inFlight) continue;
+      if (now() < state.nextAttemptAt) {
+        // A failed stop's agent-SDK fence stays up through the backoff, up to
+        // stuckAfter tries; then the old worker serves again.
+        if (state.sdkFence !== null && state.sdkFence === pid && state.failures < stuckAfter) sdkFenced.add(sid);
+        continue;
+      }
       if (state.recycledPid && state.recycledPid === toPid(worker.pid)) continue;
       let hold = null;
       let archived = false;
+      let sdkMode = null;
       try {
-        hold = holdFor(sid, worker, state);
+        sdkMode = sdkModeFor(worker, state);
+        hold = holdFor(sid, worker, state, sdkMode);
         if (!hold) archived = isArchived(sid) === true;
       } catch (error) {
         // The relay cannot tell whether the session is busy or archived: the
         // worker is not stopped on a guess, and the probe is retried later.
+        // Nor is it fenced on a guess: an agent-SDK fence comes down.
         noteFailure(sid, state, 'probe', errorText(error, 'unknown'));
         continue;
       }
       if (hold) {
+        // Background tasks keep the CLI child up, so a delivery goes to it:
+        // the one hold such a worker waits out unfenced.
+        if (sdkMode && hold !== 'background') sdkFenceUp(sid, pid, state, sdkFenced);
         noteWaiting(sid, state, hold, nextWaiting);
         continue;
       }
       if (inFlight >= maxInFlight) {
         deferred = true;
+        if (sdkMode) sdkFenceUp(sid, pid, state, sdkFenced);
         continue;
       }
       if (archived) {
         // Fenced from here: the holds just passed, and a delivery to the
         // process being stopped would start a turn on it.
         fence.hold(sid, pid);
+        sdkFenceHandOver(state);
         startedSteps.push(begin(state, () => stopArchived(sid, worker, state)));
         continue;
       }
       if (!canLaunchNow()) {
         deferred = true;
+        if (sdkMode) sdkFenceUp(sid, pid, state, sdkFenced);
         continue;
       }
       lastAdmitAt = now();
@@ -564,8 +682,10 @@ export function createWorkerRecycleService({
       // await: no delivery can reach the old process between this check and
       // its stop, nor between the stop and the launch.
       fence.hold(sid, pid);
+      sdkFenceHandOver(state);
       startedSteps.push(begin(state, () => recycle(sid, worker, state)));
     }
+    sdkFencesDown(sdkFenced);
     // Launches owed after a stop went through: the registry lists no new
     // process for those sessions.
     for (const [sid, state] of bySession) {
@@ -639,6 +759,20 @@ export function createWorkerRecycleService({
     }
     started = true;
     scheduleNext();
+    // The start's own sweep: workers left from before the restart need not
+    // wait a whole interval.
+    runSweep();
+  }
+
+  /**
+   * A sweep now, outside the interval, while the service runs (a worker's
+   * hello). Its judging, the fence included, is done synchronously before
+   * this returns; the promise resolves once the recycles it started are done.
+   * Never rejects.
+   */
+  function sweepNow() {
+    if (!started) return Promise.resolve();
+    return sweep().catch(() => {});
   }
 
   function stop() {
@@ -667,7 +801,7 @@ export function createWorkerRecycleService({
     };
   }
 
-  return { start, stop, sweep, isStale, status };
+  return { start, stop, sweep, sweepNow, isStale, status };
 }
 
 /**

package/shared/model-refresh-schedule.mjs

@@ -0,0 +1,120 @@
+// "Check for new models" (Settings → General → Models): the relay asks each
+// enabled provider for its model list on a timer, skipping a provider whose
+// list was refreshed recently by anything (the Refresh button, a settings
+// save, a login, an install, the boot). Relay-wide app_settings keys:
+//
+//   model_refresh_auto            'true' / 'false'; absent = on
+//   model_refresh_interval_hours  6 / 12 / 24 / 72 / 168; absent = 24
+//   model_refresh_excluded        JSON list of provider ids left out; absent = none
+//   model_refresh_last            JSON { <provider>: { at, ok, error, lastOkAt, lastFailureAt } }
+
+export const MODEL_REFRESH_AUTO_SETTING_KEY = 'model_refresh_auto';
+export const MODEL_REFRESH_INTERVAL_HOURS_SETTING_KEY = 'model_refresh_interval_hours';
+export const MODEL_REFRESH_EXCLUDED_SETTING_KEY = 'model_refresh_excluded';
+export const MODEL_REFRESH_LAST_SETTING_KEY = 'model_refresh_last';
+
+export const DEFAULT_MODEL_REFRESH_AUTO = true;
+export const MODEL_REFRESH_INTERVAL_HOURS_CHOICES = Object.freeze([6, 12, 24, 72, 168]);
+export const DEFAULT_MODEL_REFRESH_INTERVAL_HOURS = 24;
+// A failed check is tried again at most this often, whatever the interval.
+export const MODEL_REFRESH_FAILURE_RETRY_MS = 60 * 60 * 1000;
+// The first look a minute after boot (overdue providers are checked then),
+// then every quarter of an hour.
+export const MODEL_REFRESH_FIRST_TICK_MS = 60 * 1000;
+export const MODEL_REFRESH_TICK_MS = 15 * 60 * 1000;
+
+// The providers in the order Settings and Select Models list them; `github`
+// is Copilot. `tab` is the provider's Select Models tab.
+export const MODEL_REFRESH_PROVIDERS = Object.freeze([
+  Object.freeze({ id: 'github', label: 'Copilot', tab: 'copilot' }),
+  Object.freeze({ id: 'openai', label: 'OpenAI', tab: 'openai' }),
+  Object.freeze({ id: 'claude', label: 'Claude', tab: 'anthropic' }),
+  Object.freeze({ id: 'claude-api', label: 'Claude API', tab: 'claude-api' }),
+  Object.freeze({ id: 'cursor', label: 'Cursor', tab: 'cursor' }),
+  Object.freeze({ id: 'grok', label: 'Grok', tab: 'grok' }),
+  Object.freeze({ id: 'deepseek', label: 'DeepSeek', tab: 'deepseek' }),
+]);
+const PROVIDER_IDS = new Set(MODEL_REFRESH_PROVIDERS.map((provider) => provider.id));
+
+export function modelRefreshProvider(id) {
+  return MODEL_REFRESH_PROVIDERS.find((provider) => provider.id === String(id || '').trim().toLowerCase()) || null;
+}
+
+/** An unset or unreadable setting is on. */
+export function readModelRefreshAutoSetting(storedValue) {
+  const text = String(storedValue ?? '').trim().toLowerCase();
+  if (text === 'false') return false;
+  if (text === 'true') return true;
+  return DEFAULT_MODEL_REFRESH_AUTO;
+}
+
+export function parseModelRefreshAutoUpdate(value) {
+  if (typeof value !== 'boolean') return { ok: false, error: 'auto must be true or false' };
+  return { ok: true, auto: value };
+}
+
+/** One of the choices; an unset or unknown value is the default. */
+export function readModelRefreshIntervalHoursSetting(storedValue) {
+  const hours = Number(String(storedValue ?? '').trim());
+  return MODEL_REFRESH_INTERVAL_HOURS_CHOICES.includes(hours) ? hours : DEFAULT_MODEL_REFRESH_INTERVAL_HOURS;
+}
+
+export function parseModelRefreshIntervalHoursUpdate(value) {
+  const hours = typeof value === 'number' ? value : (typeof value === 'string' && value.trim() ? Number(value.trim()) : NaN);
+  if (!MODEL_REFRESH_INTERVAL_HOURS_CHOICES.includes(hours)) {
+    return { ok: false, error: `intervalHours must be one of ${MODEL_REFRESH_INTERVAL_HOURS_CHOICES.join(', ')}` };
+  }
+  return { ok: true, intervalHours: hours };
+}
+
+function knownProviderIds(values) {
+  return [...new Set((Array.isArray(values) ? values : [])
+    .map((value) => String(value || '').trim().toLowerCase())
+    .filter((value) => PROVIDER_IDS.has(value)))];
+}
+
+/** The providers left out of the check; unknown ids are dropped. */
+export function readModelRefreshExcludedSetting(storedValue) {
+  try {
+    return knownProviderIds(JSON.parse(String(storedValue || '[]')));
+  } catch {
+    return [];
+  }
+}
+
+export function parseModelRefreshExcludedUpdate(value) {
+  if (!Array.isArray(value)) return { ok: false, error: 'excluded must be a list of provider ids' };
+  const unknown = value.map((entry) => String(entry || '').trim().toLowerCase()).filter((entry) => entry && !PROVIDER_IDS.has(entry));
+  if (unknown.length) return { ok: false, error: `Unknown provider: ${unknown[0]}` };
+  return { ok: true, excluded: knownProviderIds(value) };
+}
+
+function isoOrNull(value) {
+  const text = String(value ?? '').trim();
+  return text && Number.isFinite(Date.parse(text)) ? new Date(text).toISOString() : null;
+}
+
+/** The last refresh of each provider; entries that cannot be read are dropped. */
+export function readModelRefreshLastSetting(storedValue) {
+  let parsed;
+  try {
+    parsed = JSON.parse(String(storedValue || '{}'));
+  } catch {
+    return {};
+  }
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) return {};
+  const out = {};
+  for (const [id, entry] of Object.entries(parsed)) {
+    if (!PROVIDER_IDS.has(id) || !entry || typeof entry !== 'object') continue;
+    const at = isoOrNull(entry.at);
+    if (!at) continue;
+    out[id] = {
+      at,
+      ok: entry.ok === true,
+      error: entry.ok === true ? null : (String(entry.error || '').trim().slice(0, 300) || null),
+      lastOkAt: isoOrNull(entry.lastOkAt),
+      lastFailureAt: isoOrNull(entry.lastFailureAt),
+    };
+  }
+  return out;
+}

package/shared/worker-runtime/direct-openai-image.mjs

@@ -99,11 +99,15 @@ export async function runDirectOpenAIImageTurn({
       attachments: Array.isArray(message?.attachments) ? message.attachments : [],
     }, requestOptions);
   } catch (error) {
-    if (isDirectOpenAIImageTerminalStatus(error?.status)) {
-      const terminalError = new Error(
-        `OpenAI image request failed: ${error?.detail || `HTTP ${error?.status || 'error'}`}`,
-      );
+    // `relayProcessing`: the provider answered 2xx and the relay could not
+    // process the reply; the generation is billed, so no requeue either.
+    const relayProcessing = error?.body?.relayProcessing === true;
+    if (isDirectOpenAIImageTerminalStatus(error?.status) || relayProcessing) {
+      const terminalError = new Error(relayProcessing
+        ? `The relay could not process the image OpenAI returned (${error?.detail || 'unknown error'}). The generation was billed; the message was not retried.`
+        : `OpenAI image request failed: ${error?.detail || `HTTP ${error?.status || 'error'}`}`);
       terminalError.code = OPENAI_IMAGE_TERMINAL_CODE;
+      terminalError.relayProcessing = relayProcessing;
       terminalError.status = error?.status;
       terminalError.detail = error?.detail || null;
       throw terminalError;