@oar-sh/oar 0.9.12 → 0.9.14

package/shared/worker-runtime/config-loader.mjs

@@ -2,6 +2,9 @@ import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
 
+import { RESCUE_DIR_NAME } from "../../server/services/earlier-database-service.mjs";
+import { resolveRelayStateDirs, resolveStateLayout } from "../../server/services/oar-state-layout.mjs";
+
 function toAbsolutePath(value) {
   if (!value) return "";
   return path.isAbsolute(value) ? value : path.resolve(process.cwd(), value);
@@ -12,15 +15,21 @@ function looksLikeServerDir(serverDir) {
   return fs.existsSync(path.join(serverDir, "server.js")) && fs.existsSync(path.join(serverDir, "config.json"));
 }
 
-export function resolveRelayPaths(importMetaUrl) {
+/**
+ * Where the extension finds the relay, its config and logs. A global install
+ * uses the state root's folders as `oar start` does, and `RELAY_ENV` pins
+ * them for the managed relay (not `server/data`, which `npm i -g` deletes).
+ * The environment wins; a checkout keeps its `server/` folders.
+ */
+export function resolveRelayPaths(importMetaUrl, { env = process.env, platform = process.platform } = {}) {
   const __dirname = path.dirname(fileURLToPath(importMetaUrl));
   const projectServerDir = path.resolve(__dirname, "../../../server");
   const cwdServerDir = path.resolve(process.cwd(), "server");
-  const envRoot = toAbsolutePath(process.env.COPILOT_WEB_RELAY_ROOT || "");
-  const envServerDir = toAbsolutePath(process.env.COPILOT_WEB_RELAY_SERVER_DIR || "");
-  const envConfigPath = toAbsolutePath(process.env.COPILOT_WEB_RELAY_CONFIG || "");
-  const envToolsPath = toAbsolutePath(process.env.COPILOT_WEB_RELAY_TOOLS || "");
-  const envLogDir = toAbsolutePath(process.env.COPILOT_WEB_RELAY_LOG_DIR || "");
+  const envRoot = toAbsolutePath(env.COPILOT_WEB_RELAY_ROOT || "");
+  const envServerDir = toAbsolutePath(env.COPILOT_WEB_RELAY_SERVER_DIR || "");
+  const envConfigPath = toAbsolutePath(env.COPILOT_WEB_RELAY_CONFIG || "");
+  const envToolsPath = toAbsolutePath(env.COPILOT_WEB_RELAY_TOOLS || "");
+  const envLogDir = toAbsolutePath(env.COPILOT_WEB_RELAY_LOG_DIR || "");
 
   const resolvedServerDir =
     (looksLikeServerDir(envServerDir) && envServerDir) ||
@@ -30,11 +39,23 @@ export function resolveRelayPaths(importMetaUrl) {
     projectServerDir;
 
   const SERVER_DIR = resolvedServerDir;
-  const CONFIG_PATH = envConfigPath || path.resolve(SERVER_DIR, "config.json");
+  const packageRoot = path.dirname(SERVER_DIR);
+  const layout = resolveStateLayout({ packageRoot, env, platform });
+  const stateDirs = layout.checkout ? null : resolveRelayStateDirs({ packageRoot, env, platform, layout });
+  const CONFIG_PATH = envConfigPath
+    || (stateDirs ? path.join(layout.configDir, "config.json") : path.resolve(SERVER_DIR, "config.json"));
   const RELAY_TOOLS_PATH = envToolsPath || path.resolve(SERVER_DIR, "relay-tools.md");
-  const LOG_DIR = envLogDir || path.resolve(SERVER_DIR, "logs");
+  const LOG_DIR = envLogDir || stateDirs?.logDir || path.resolve(SERVER_DIR, "logs");
   const SERVER_LOG_PATH = path.resolve(LOG_DIR, "server.log");
   const SERVER_ERR_PATH = path.resolve(LOG_DIR, "server-err.log");
+  const RELAY_ENV = stateDirs
+    ? {
+      COPILOT_WEB_RELAY_ROOT: packageRoot,
+      COPILOT_WEB_RELAY_CONFIG: CONFIG_PATH,
+      COPILOT_WEB_RELAY_DATA_DIR: stateDirs.dataDir,
+      COPILOT_WEB_RELAY_LOG_DIR: LOG_DIR,
+    }
+    : {};
 
   return {
     __dirname,
@@ -44,9 +65,34 @@ export function resolveRelayPaths(importMetaUrl) {
     LOG_DIR,
     SERVER_LOG_PATH,
     SERVER_ERR_PATH,
+    RELAY_ENV,
   };
 }
 
+/**
+ * The package's `server/config.json` (or the newest rescued one) copied to the
+ * state root when it has none: copy only, never over one. Returns the source.
+ */
+export function carryOverPackageConfig({ serverDir, configPath, dataDir, fsImpl = fs } = {}) {
+  if (!configPath || !dataDir || fsImpl.existsSync(configPath)) return null;
+  const sources = [path.join(serverDir, "config.json")];
+  try {
+    const rescueRoot = path.join(dataDir, RESCUE_DIR_NAME);
+    for (const name of fsImpl.readdirSync(rescueRoot).filter((entry) => !entry.endsWith(".partial")).sort().reverse()) {
+      sources.push(path.join(rescueRoot, name, "config.json"));
+    }
+  } catch {}
+  const source = sources.find((file) => fsImpl.existsSync(file));
+  if (!source) return null;
+  try {
+    fsImpl.mkdirSync(path.dirname(configPath), { recursive: true });
+    fsImpl.copyFileSync(source, configPath, fs.constants.COPYFILE_EXCL);
+    return source;
+  } catch {
+    return null;
+  }
+}
+
 export function loadTokenFromConfig(configPath) {
   try {
     const cfg = loadRelayConfig(configPath);

package/shared/worker-runtime/prompt-context.mjs

@@ -99,7 +99,9 @@ export function createRelayPromptBuilder({
       // Guidance is advisory: a preview lookup that fails must cost the block,
       // never the turn.
       const previewBlock = typeof getPreviewInstructions === "function"
-        ? await Promise.resolve().then(getPreviewInstructions).catch(() => "")
+        ? await Promise.resolve()
+          .then(() => getPreviewInstructions({ conversationId: String(message?.conversationId || "") }))
+          .catch(() => "")
         : "";
       instructions = applyPreviewInstructions(toolInstructions, previewBlock);
     }

package/shared/worker-runtime/worker-websocket-link.mjs

@@ -69,6 +69,15 @@ export function createWorkerWebSocketLink({
   minBackoffMs = 1000,
   maxBackoffMs = 8_000,
   jitterMs = 250,
+  // A connection counts as made only at the relay's first server.hello (the
+  // relay sends it as soon as it accepts the socket). One still connecting,
+  // or open but silent, when this runs out is given up and retried: a relay
+  // that hangs in its shutdown can hold a socket in either state without
+  // ever ending it.
+  connectTimeoutMs = 15_000,
+  // Failed attempts after the first few are logged at most once per this
+  // interval, with a count of the ones in between.
+  failureLogIntervalMs = 60_000,
   readyRefreshMs = 10_000,
   staleConnectionMs = 30_000,
   WebSocketImpl = globalThis.WebSocket,
@@ -84,6 +93,10 @@ export function createWorkerWebSocketLink({
   let readyRefreshTimer = null;
   let reconnectAttempt = 0;
   let reconnectDelayMs = 0;
+  // The current socket's connect deadline (see connectTimeoutMs).
+  let connectDeadlineTimer = null;
+  let lastFailureLoggedAt = null;
+  let failuresNotLogged = 0;
   // Ends the current socket's connection (see connect).
   let endCurrentSocket = null;
   // Insertion-ordered: the first entry is the oldest in-flight delivery, which
@@ -239,6 +252,12 @@ export function createWorkerWebSocketLink({
     reconnectTimer = null;
   }
 
+  function clearConnectDeadline() {
+    if (!connectDeadlineTimer) return;
+    clearTimeoutImpl(connectDeadlineTimer);
+    connectDeadlineTimer = null;
+  }
+
   function clearReadyRefreshTimer() {
     if (!readyRefreshTimer) return;
     clearIntervalImpl(readyRefreshTimer);
@@ -290,7 +309,23 @@ export function createWorkerWebSocketLink({
     if (typeof readyRefreshTimer?.unref === "function") readyRefreshTimer.unref();
   }
 
-  function scheduleReconnect() {
+  // One line per failed attempt for the first few, then at most one per
+  // failureLogIntervalMs: a relay that stays away for an hour must not fill
+  // the worker log with a line every 8 seconds.
+  function logFailedAttempt(reason) {
+    const nowMs = getNowMs();
+    const intervalMs = Math.max(0, Number(failureLogIntervalMs) || 0);
+    if (reconnectAttempt > 3 && lastFailureLoggedAt !== null && (nowMs - lastFailureLoggedAt) < intervalMs) {
+      failuresNotLogged += 1;
+      return;
+    }
+    const notLogged = failuresNotLogged ? [`notLogged=${failuresNotLogged}`] : [];
+    dbg("worker ws disconnected", reason, `attempt=${reconnectAttempt}`, `retryInMs=${reconnectDelayMs}`, ...notLogged);
+    lastFailureLoggedAt = nowMs;
+    failuresNotLogged = 0;
+  }
+
+  function scheduleReconnect(reason) {
     if (stopped) return;
     // One timer at a time: a second end reported for the same outage must not
     // stack another attempt or push the backoff up.
@@ -299,7 +334,7 @@ export function createWorkerWebSocketLink({
     const backoff = Math.min(maxBackoffMs, minBackoffMs * (2 ** Math.max(0, reconnectAttempt - 1)));
     const jitter = Math.floor(Math.random() * Math.max(0, jitterMs));
     reconnectDelayMs = Math.min(maxBackoffMs, backoff + jitter);
-    dbg("worker ws reconnect scheduled", `attempt=${reconnectAttempt}`, `delayMs=${reconnectDelayMs}`);
+    logFailedAttempt(reason);
     reconnectTimer = setTimeoutImpl(() => {
       reconnectTimer = null;
       connect();
@@ -312,42 +347,70 @@ export function createWorkerWebSocketLink({
       const openState = resolveSocketStateValue(ws, "OPEN", 1);
       const connectingState = resolveSocketStateValue(ws, "CONNECTING", 0);
       if (ws.readyState === openState || ws.readyState === connectingState) return;
+      // A socket left closing whose end was never reported: give it up
+      // quietly, this connect is its reconnect.
+      endCurrentSocket?.("replaced", { reconnect: false });
     }
     if (typeof WebSocketImpl !== "function") {
       dbg("worker ws unavailable: global WebSocket is missing");
       return;
     }
     clearReconnectTimer();
-    const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
-    const socket = new WebSocketImpl(url);
+    let socket = null;
+    try {
+      const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
+      socket = new WebSocketImpl(url);
+    } catch (error) {
+      // A socket that fails in its constructor raises no events: nothing but
+      // this would ever try again.
+      scheduleReconnect(`connect-failed: ${error?.message || String(error)}`);
+      return;
+    }
     ws = socket;
     let serverHelloSeen = false;
-    // Set once this socket's end has been handled, by whichever of error and
-    // close comes first; every later event from it is ignored. Node 22 fires
-    // no close for a socket closed while still connecting, and closing it
-    // from its own error handler raised the error again, so the reconnect
-    // has to be scheduled from the error as well, and only once.
+    // Set once this socket's end has been handled, by whichever of error,
+    // close, the connect deadline or a stale/draining close comes first;
+    // every later event from it is ignored. Node 22 fires no close for a
+    // socket closed while still connecting, and closing it from its own
+    // error handler raised the error again, so the reconnect has to be
+    // scheduled from the error as well, and only once.
     let ended = false;
-    function endSocket(reason) {
+    function endSocket(reason, { reconnect = true } = {}) {
       if (ended) return false;
       ended = true;
+      if (endCurrentSocket === endSocket) {
+        endCurrentSocket = null;
+        clearConnectDeadline();
+      }
       if (ws === socket) {
         ws = null;
         clearReadyRefreshTimer();
         // The next connection may reach a different (restarted) relay.
         serverCapabilities = new Set();
       }
-      if (stopped) return true;
-      dbg("worker ws disconnected", reason);
-      scheduleReconnect();
+      if (stopped || !reconnect) return true;
+      // A newer socket is already under way and owns the reconnect.
+      if (ws) return true;
+      scheduleReconnect(reason);
       return true;
     }
     endCurrentSocket = endSocket;
+    clearConnectDeadline();
+    const deadlineMs = Math.max(0, Number(connectTimeoutMs) || 0);
+    if (deadlineMs > 0) {
+      connectDeadlineTimer = setTimeoutImpl(() => {
+        connectDeadlineTimer = null;
+        if (ended) return;
+        const opened = socket.readyState === resolveSocketStateValue(socket, "OPEN", 1);
+        // Ended before the close call, as in the error handler below.
+        endSocket(opened ? "no-server-hello" : "connect-timeout");
+        try { socket.close?.(); } catch {}
+      }, deadlineMs);
+      if (typeof connectDeadlineTimer?.unref === "function") connectDeadlineTimer.unref();
+    }
     socket.addEventListener("open", () => {
       if (ended) return;
       const openedAt = getNowMs();
-      reconnectAttempt = 0;
-      reconnectDelayMs = 0;
       lastOpenAt = openedAt;
       lastMessageAt = openedAt;
       dbg("worker ws connected");
@@ -373,6 +436,14 @@ export function createWorkerWebSocketLink({
         serverCapabilities = new Set(Array.isArray(payload.capabilities) ? payload.capabilities.map(String) : []);
         if (!serverHelloSeen) {
           serverHelloSeen = true;
+          // The handshake is complete: only now does the backoff start over.
+          // A relay that accepts the socket and drops it before its hello
+          // keeps backing off instead of being retried every second.
+          clearConnectDeadline();
+          reconnectAttempt = 0;
+          reconnectDelayMs = 0;
+          lastFailureLoggedAt = null;
+          failuresNotLogged = 0;
           try {
             onServerHello(String(payload.reason || "server-hello"));
           } catch (error) {
@@ -388,6 +459,9 @@ export function createWorkerWebSocketLink({
       }
       if (payload?.type === "server.draining") {
         dbg("worker ws draining", payload.reason || "relay-shutdown");
+        // A relay stuck in its shutdown may never complete the close
+        // handshake: the reconnect does not wait for the close event.
+        endSocket("draining");
         try { socket.close?.(); } catch {}
         return;
       }
@@ -408,15 +482,16 @@ export function createWorkerWebSocketLink({
         dbg("worker ws blocked", payload.reason || "blocked");
       }
     });
-    socket.addEventListener("close", () => {
-      endSocket("close");
+    socket.addEventListener("close", (event) => {
+      const code = Number(event?.code);
+      endSocket(Number.isFinite(code) && code > 0 ? `close code=${code}` : "close");
     });
-    socket.addEventListener("error", (error) => {
+    socket.addEventListener("error", (event) => {
       if (ended) return;
-      dbg("worker ws error", error?.message || String(error));
+      const detail = event?.message || event?.error?.message || "unknown";
       // Ended before the close call, so an error or close it raises
       // (synchronously on some runtimes) finds the socket already handled.
-      endSocket("error");
+      endSocket(`error: ${detail}`);
       try { socket.close?.(); } catch {}
     });
   }
@@ -429,6 +504,7 @@ export function createWorkerWebSocketLink({
   function stop() {
     stopped = true;
     clearReconnectTimer();
+    clearConnectDeadline();
     clearReadyRefreshTimer();
     if (ws) {
       try { ws.close(); } catch {}

package/shared/worker-runtime/working-directory.mjs

@@ -0,0 +1,65 @@
+// The session's working directory, checked before a runtime is started in it.
+//
+// A session runs in the folder it was given (COPILOT_WORKSPACE_ROOT). When that
+// folder is gone (deleted, renamed, a drive unmounted), spawning a process in
+// it fails with ENOENT, the code a missing executable has too, and each SDK
+// takes it for that: the Claude Agent SDK reports a native binary that "exists
+// but failed to launch" and blames the libc, Grok's classifier a CLI that is
+// not installed. The workers check the folder before they start anything in
+// it, and put a launch error that happens while the folder is missing down to
+// the folder. A tmux launch does not catch it earlier: `tmux new-session -c`
+// falls back to the home folder, so the worker starts and only the folder in
+// its environment is wrong.
+
+import fs from 'node:fs';
+
+export const WORKING_DIRECTORY_MISSING_CODE = 'working-directory-missing';
+
+export const WORKING_DIRECTORY_GUIDANCE =
+  'Restore the folder, or choose another working directory for this conversation (⋯ → Change CWD), '
+  + 'then send the message again.';
+
+// What a failed launch says, in the words of the SDKs: Node's spawn error, the
+// Claude Agent SDK's own wording for it.
+const LAUNCH_FAILURE_TEXT = /\bENOENT\b|\bspawn\b|failed to launch|failed to spawn|not found at/i;
+
+/**
+ * The error a turn fails with when `cwd` cannot be worked in, or null when it
+ * can. Only an answer that the folder is not there counts: a folder that
+ * cannot be read (EACCES, a network drive that does not answer) is left to the
+ * launch, which says more about it than a guess would.
+ */
+export function missingWorkingDirectoryError(cwd, { statSync = fs.statSync } = {}) {
+  const folder = String(cwd || '').trim();
+  if (!folder) return null;
+  let stats = null;
+  try {
+    stats = statSync(folder);
+  } catch (error) {
+    if (error?.code !== 'ENOENT' && error?.code !== 'ENOTDIR') return null;
+  }
+  if (stats?.isDirectory()) return null;
+  const error = new Error(stats
+    ? `The working directory of this conversation is not a folder: ${folder}.`
+    : `The working directory of this conversation no longer exists: ${folder}.`);
+  error.code = WORKING_DIRECTORY_MISSING_CODE;
+  error.workingDirectory = folder;
+  error.guidance = WORKING_DIRECTORY_GUIDANCE;
+  return error;
+}
+
+export function isWorkingDirectoryMissingError(error) {
+  return error?.code === WORKING_DIRECTORY_MISSING_CODE;
+}
+
+/**
+ * A launch that failed while the folder is missing failed because of it: the
+ * folder's error in its place. Any other error, or one that does not look like
+ * a failed launch, is returned as it is.
+ */
+export function explainLaunchFailure(error, cwd, { check = missingWorkingDirectoryError } = {}) {
+  if (isWorkingDirectoryMissingError(error)) return error;
+  const text = `${error?.code || ''} ${error?.message || error || ''}`;
+  if (!LAUNCH_FAILURE_TEXT.test(text)) return error;
+  return check(cwd) || error;
+}