@oar-sh/oar 0.9.11 → 0.9.13

package/shared/worker-runtime/task-output-spool.mjs

@@ -0,0 +1,181 @@
+// The spool a session worker writes a background task's output into when the
+// provider keeps none in a file of its own (Copilot's attached shells, Grok's
+// terminals): `<os.tmpdir()>/oar-task-output/<sessionId>/<taskId>.log`,
+// append-only. The worker names the file as the task row's `outputPath`
+// (`outputKind: 'spool'`) and the relay reads it like any other output file,
+// so the console works the same for every provider and outlives a worker
+// restart.
+//
+// Bounded: a file stops growing at 50 MB with one "output capped" line, and
+// `sweep()` (at worker start and hourly) deletes files that ended more than an
+// hour ago. Nothing is deleted at exit: the relay keeps showing an ended
+// task's output for about an hour.
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export const TASK_OUTPUT_SPOOL_DIR = 'oar-task-output';
+export const TASK_OUTPUT_SPOOL_MAX_BYTES = 50 * 1024 * 1024;
+export const TASK_OUTPUT_SPOOL_MAX_AGE_MS = 60 * 60 * 1000;
+// A dead worker's session folder is left for its own next start to sweep;
+// another session's worker removes it only once it is a day old.
+export const TASK_OUTPUT_SPOOL_FOREIGN_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
+const CAPPED_LINE = '\n[output capped at 50 MB: later output is not kept]\n';
+
+/** A file-name-safe form of an id: [A-Za-z0-9._-], at most 128 characters, never `.` or `..`. */
+export function spoolSafeName(value) {
+  const text = String(value ?? '').trim().replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 128);
+  return /^\.+$/.test(text) ? text.replace(/\./g, '_') : text;
+}
+
+/**
+ * `createTaskOutputSpool({ sessionId })` →
+ * `{ root, pathFor(taskId), append(taskId, text), end(taskId), sweep(maxAgeMs) }`.
+ *
+ *  - `pathFor(taskId)`: the absolute file path for the task's row.
+ *  - `append(taskId, text)`: `{ ok, bytes, capped }`; never throws. The cap
+ *    writes one marker line and drops what comes after it.
+ *  - `end(taskId)`: the task finished; its file stays for the relay until a
+ *    sweep finds it older than the age limit.
+ *  - `sweep(maxAgeMs = 1 h)`: deletes this session's files not written to for
+ *    that long (a task still appending is never swept), and other sessions'
+ *    folders a day old. Returns the number of files removed.
+ *
+ * `baseDir` (default the OS temp dir) and `now` are for tests.
+ */
+export function createTaskOutputSpool({
+  sessionId,
+  now = () => Date.now(),
+  baseDir = os.tmpdir(),
+  maxBytes = TASK_OUTPUT_SPOOL_MAX_BYTES,
+  fsImpl = fs,
+} = {}) {
+  const sessionName = spoolSafeName(sessionId);
+  if (!sessionName) throw new Error('createTaskOutputSpool requires sessionId');
+  const top = path.join(path.resolve(baseDir), TASK_OUTPUT_SPOOL_DIR);
+  const root = path.join(top, sessionName);
+  // taskId → { bytes, capped }; a task listed here is live (still appending).
+  const live = new Map();
+  let rootReady = false;
+
+  // The shared folder is ours to write and sweep only when it is a real
+  // folder (not a symlink someone else put into a shared temp dir) owned by
+  // this user where the platform says who owns it.
+  function topIsOurs() {
+    let stats;
+    try { stats = fsImpl.lstatSync(top); } catch { return false; }
+    if (stats.isSymbolicLink() || !stats.isDirectory()) return false;
+    const uid = typeof process.getuid === 'function' ? process.getuid() : null;
+    return uid === null || typeof stats.uid !== 'number' || stats.uid === uid;
+  }
+
+  function ensureRoot() {
+    if (rootReady) return;
+    // Owner-only: the temp dir is shared on a multi-user host.
+    try { fsImpl.mkdirSync(top, { mode: 0o700 }); } catch (error) { if (error?.code !== 'EEXIST') throw error; }
+    if (!topIsOurs()) throw new Error(`${top} is not a folder of this user`);
+    fsImpl.mkdirSync(root, { recursive: true, mode: 0o700 });
+    rootReady = true;
+  }
+
+  function pathFor(taskId) {
+    const name = spoolSafeName(taskId);
+    if (!name) throw new Error('task id is empty');
+    return path.join(root, `${name}.log`);
+  }
+
+  function stateOf(taskId, file) {
+    let state = live.get(taskId);
+    if (!state) {
+      let bytes = 0;
+      try { bytes = fsImpl.statSync(file).size; } catch {}
+      state = { bytes, capped: bytes >= maxBytes };
+      live.set(taskId, state);
+    }
+    return state;
+  }
+
+  function append(taskId, text) {
+    const id = String(taskId ?? '').trim();
+    const value = typeof text === 'string' ? text : (Buffer.isBuffer(text) ? text.toString('utf8') : String(text ?? ''));
+    if (!id) return { ok: false, bytes: 0, capped: false };
+    try {
+      ensureRoot();
+      const file = pathFor(id);
+      const state = stateOf(id, file);
+      if (state.capped || !value) return { ok: true, bytes: state.bytes, capped: state.capped };
+      let data = Buffer.from(value, 'utf8');
+      const room = maxBytes - state.bytes;
+      if (data.length > room) {
+        // Cut on a character boundary, then say so once.
+        let end = Math.max(0, room);
+        while (end > 0 && (data[end] & 0xc0) === 0x80) end -= 1;
+        data = Buffer.concat([data.subarray(0, end), Buffer.from(CAPPED_LINE, 'utf8')]);
+        state.capped = true;
+      }
+      fsImpl.appendFileSync(file, data, { mode: 0o600 });
+      state.bytes += data.length;
+      return { ok: true, bytes: state.bytes, capped: state.capped };
+    } catch {
+      return { ok: false, bytes: live.get(id)?.bytes || 0, capped: !!live.get(id)?.capped };
+    }
+  }
+
+  function end(taskId) {
+    live.delete(String(taskId ?? '').trim());
+  }
+
+  function liveNames() {
+    return new Set([...live.keys()].map((id) => `${spoolSafeName(id)}.log`));
+  }
+
+  function sweepDir(dir, maxAge, skip = new Set()) {
+    let removed = 0;
+    let left = 0;
+    let entries = [];
+    try { entries = fsImpl.readdirSync(dir, { withFileTypes: true }); } catch { return { removed, left: 0, missing: true }; }
+    const cutoff = now() - maxAge;
+    for (const entry of entries) {
+      if (!entry.isFile() || skip.has(entry.name)) {
+        left += 1;
+        continue;
+      }
+      const file = path.join(dir, entry.name);
+      try {
+        if (fsImpl.statSync(file).mtimeMs < cutoff) {
+          fsImpl.unlinkSync(file);
+          removed += 1;
+        } else {
+          left += 1;
+        }
+      } catch {
+        left += 1;
+      }
+    }
+    return { removed, left, missing: false };
+  }
+
+  function sweep(maxAgeMs = TASK_OUTPUT_SPOOL_MAX_AGE_MS) {
+    const maxAge = Number.isFinite(Number(maxAgeMs)) && Number(maxAgeMs) >= 0 ? Number(maxAgeMs) : TASK_OUTPUT_SPOOL_MAX_AGE_MS;
+    // Never through a symlinked or foreign shared folder: its "session
+    // folders" could be anything.
+    if (!topIsOurs()) return 0;
+    let removed = sweepDir(root, maxAge, liveNames()).removed;
+    let folders = [];
+    try { folders = fsImpl.readdirSync(top, { withFileTypes: true }); } catch {}
+    for (const folder of folders) {
+      if (!folder.isDirectory() || folder.name === sessionName) continue;
+      const dir = path.join(top, folder.name);
+      const result = sweepDir(dir, Math.max(maxAge, TASK_OUTPUT_SPOOL_FOREIGN_MAX_AGE_MS));
+      removed += result.removed;
+      if (!result.missing && result.left === 0) {
+        try { fsImpl.rmdirSync(dir); } catch {}
+      }
+    }
+    return removed;
+  }
+
+  return { root, pathFor, append, end, sweep };
+}

package/shared/worker-runtime/worker-websocket-link.mjs

@@ -69,6 +69,15 @@ export function createWorkerWebSocketLink({
   minBackoffMs = 1000,
   maxBackoffMs = 8_000,
   jitterMs = 250,
+  // A connection counts as made only at the relay's first server.hello (the
+  // relay sends it as soon as it accepts the socket). One still connecting,
+  // or open but silent, when this runs out is given up and retried: a relay
+  // that hangs in its shutdown can hold a socket in either state without
+  // ever ending it.
+  connectTimeoutMs = 15_000,
+  // Failed attempts after the first few are logged at most once per this
+  // interval, with a count of the ones in between.
+  failureLogIntervalMs = 60_000,
   readyRefreshMs = 10_000,
   staleConnectionMs = 30_000,
   WebSocketImpl = globalThis.WebSocket,
@@ -84,6 +93,10 @@ export function createWorkerWebSocketLink({
   let readyRefreshTimer = null;
   let reconnectAttempt = 0;
   let reconnectDelayMs = 0;
+  // The current socket's connect deadline (see connectTimeoutMs).
+  let connectDeadlineTimer = null;
+  let lastFailureLoggedAt = null;
+  let failuresNotLogged = 0;
   // Ends the current socket's connection (see connect).
   let endCurrentSocket = null;
   // Insertion-ordered: the first entry is the oldest in-flight delivery, which
@@ -239,6 +252,12 @@ export function createWorkerWebSocketLink({
     reconnectTimer = null;
   }
 
+  function clearConnectDeadline() {
+    if (!connectDeadlineTimer) return;
+    clearTimeoutImpl(connectDeadlineTimer);
+    connectDeadlineTimer = null;
+  }
+
   function clearReadyRefreshTimer() {
     if (!readyRefreshTimer) return;
     clearIntervalImpl(readyRefreshTimer);
@@ -290,7 +309,23 @@ export function createWorkerWebSocketLink({
     if (typeof readyRefreshTimer?.unref === "function") readyRefreshTimer.unref();
   }
 
-  function scheduleReconnect() {
+  // One line per failed attempt for the first few, then at most one per
+  // failureLogIntervalMs: a relay that stays away for an hour must not fill
+  // the worker log with a line every 8 seconds.
+  function logFailedAttempt(reason) {
+    const nowMs = getNowMs();
+    const intervalMs = Math.max(0, Number(failureLogIntervalMs) || 0);
+    if (reconnectAttempt > 3 && lastFailureLoggedAt !== null && (nowMs - lastFailureLoggedAt) < intervalMs) {
+      failuresNotLogged += 1;
+      return;
+    }
+    const notLogged = failuresNotLogged ? [`notLogged=${failuresNotLogged}`] : [];
+    dbg("worker ws disconnected", reason, `attempt=${reconnectAttempt}`, `retryInMs=${reconnectDelayMs}`, ...notLogged);
+    lastFailureLoggedAt = nowMs;
+    failuresNotLogged = 0;
+  }
+
+  function scheduleReconnect(reason) {
     if (stopped) return;
     // One timer at a time: a second end reported for the same outage must not
     // stack another attempt or push the backoff up.
@@ -299,7 +334,7 @@ export function createWorkerWebSocketLink({
     const backoff = Math.min(maxBackoffMs, minBackoffMs * (2 ** Math.max(0, reconnectAttempt - 1)));
     const jitter = Math.floor(Math.random() * Math.max(0, jitterMs));
     reconnectDelayMs = Math.min(maxBackoffMs, backoff + jitter);
-    dbg("worker ws reconnect scheduled", `attempt=${reconnectAttempt}`, `delayMs=${reconnectDelayMs}`);
+    logFailedAttempt(reason);
     reconnectTimer = setTimeoutImpl(() => {
       reconnectTimer = null;
       connect();
@@ -312,42 +347,70 @@ export function createWorkerWebSocketLink({
       const openState = resolveSocketStateValue(ws, "OPEN", 1);
       const connectingState = resolveSocketStateValue(ws, "CONNECTING", 0);
       if (ws.readyState === openState || ws.readyState === connectingState) return;
+      // A socket left closing whose end was never reported: give it up
+      // quietly, this connect is its reconnect.
+      endCurrentSocket?.("replaced", { reconnect: false });
     }
     if (typeof WebSocketImpl !== "function") {
       dbg("worker ws unavailable: global WebSocket is missing");
       return;
     }
     clearReconnectTimer();
-    const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
-    const socket = new WebSocketImpl(url);
+    let socket = null;
+    try {
+      const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
+      socket = new WebSocketImpl(url);
+    } catch (error) {
+      // A socket that fails in its constructor raises no events: nothing but
+      // this would ever try again.
+      scheduleReconnect(`connect-failed: ${error?.message || String(error)}`);
+      return;
+    }
     ws = socket;
     let serverHelloSeen = false;
-    // Set once this socket's end has been handled, by whichever of error and
-    // close comes first; every later event from it is ignored. Node 22 fires
-    // no close for a socket closed while still connecting, and closing it
-    // from its own error handler raised the error again, so the reconnect
-    // has to be scheduled from the error as well, and only once.
+    // Set once this socket's end has been handled, by whichever of error,
+    // close, the connect deadline or a stale/draining close comes first;
+    // every later event from it is ignored. Node 22 fires no close for a
+    // socket closed while still connecting, and closing it from its own
+    // error handler raised the error again, so the reconnect has to be
+    // scheduled from the error as well, and only once.
     let ended = false;
-    function endSocket(reason) {
+    function endSocket(reason, { reconnect = true } = {}) {
       if (ended) return false;
       ended = true;
+      if (endCurrentSocket === endSocket) {
+        endCurrentSocket = null;
+        clearConnectDeadline();
+      }
       if (ws === socket) {
         ws = null;
         clearReadyRefreshTimer();
         // The next connection may reach a different (restarted) relay.
         serverCapabilities = new Set();
       }
-      if (stopped) return true;
-      dbg("worker ws disconnected", reason);
-      scheduleReconnect();
+      if (stopped || !reconnect) return true;
+      // A newer socket is already under way and owns the reconnect.
+      if (ws) return true;
+      scheduleReconnect(reason);
       return true;
     }
     endCurrentSocket = endSocket;
+    clearConnectDeadline();
+    const deadlineMs = Math.max(0, Number(connectTimeoutMs) || 0);
+    if (deadlineMs > 0) {
+      connectDeadlineTimer = setTimeoutImpl(() => {
+        connectDeadlineTimer = null;
+        if (ended) return;
+        const opened = socket.readyState === resolveSocketStateValue(socket, "OPEN", 1);
+        // Ended before the close call, as in the error handler below.
+        endSocket(opened ? "no-server-hello" : "connect-timeout");
+        try { socket.close?.(); } catch {}
+      }, deadlineMs);
+      if (typeof connectDeadlineTimer?.unref === "function") connectDeadlineTimer.unref();
+    }
     socket.addEventListener("open", () => {
       if (ended) return;
       const openedAt = getNowMs();
-      reconnectAttempt = 0;
-      reconnectDelayMs = 0;
       lastOpenAt = openedAt;
       lastMessageAt = openedAt;
       dbg("worker ws connected");
@@ -373,6 +436,14 @@ export function createWorkerWebSocketLink({
         serverCapabilities = new Set(Array.isArray(payload.capabilities) ? payload.capabilities.map(String) : []);
         if (!serverHelloSeen) {
           serverHelloSeen = true;
+          // The handshake is complete: only now does the backoff start over.
+          // A relay that accepts the socket and drops it before its hello
+          // keeps backing off instead of being retried every second.
+          clearConnectDeadline();
+          reconnectAttempt = 0;
+          reconnectDelayMs = 0;
+          lastFailureLoggedAt = null;
+          failuresNotLogged = 0;
           try {
             onServerHello(String(payload.reason || "server-hello"));
           } catch (error) {
@@ -388,6 +459,9 @@ export function createWorkerWebSocketLink({
       }
       if (payload?.type === "server.draining") {
         dbg("worker ws draining", payload.reason || "relay-shutdown");
+        // A relay stuck in its shutdown may never complete the close
+        // handshake: the reconnect does not wait for the close event.
+        endSocket("draining");
         try { socket.close?.(); } catch {}
         return;
       }
@@ -408,15 +482,16 @@ export function createWorkerWebSocketLink({
         dbg("worker ws blocked", payload.reason || "blocked");
       }
     });
-    socket.addEventListener("close", () => {
-      endSocket("close");
+    socket.addEventListener("close", (event) => {
+      const code = Number(event?.code);
+      endSocket(Number.isFinite(code) && code > 0 ? `close code=${code}` : "close");
     });
-    socket.addEventListener("error", (error) => {
+    socket.addEventListener("error", (event) => {
       if (ended) return;
-      dbg("worker ws error", error?.message || String(error));
+      const detail = event?.message || event?.error?.message || "unknown";
       // Ended before the close call, so an error or close it raises
       // (synchronously on some runtimes) finds the socket already handled.
-      endSocket("error");
+      endSocket(`error: ${detail}`);
       try { socket.close?.(); } catch {}
     });
   }
@@ -429,6 +504,7 @@ export function createWorkerWebSocketLink({
   function stop() {
     stopped = true;
     clearReconnectTimer();
+    clearConnectDeadline();
     clearReadyRefreshTimer();
     if (ws) {
       try { ws.close(); } catch {}