@oar-sh/oar 0.9.10 → 0.9.12

package/shared/windows-system-tools.mjs

@@ -0,0 +1,106 @@
+// Windows system programs by absolute path, and the bits of a process
+// environment that decide whether a child can start at all.
+//
+// A relay started by the boot-time Scheduled Task (S4U, before anyone signs
+// in) gets an environment built without a full profile: user PATH entries
+// that refer to other variables can arrive unexpanded, and a bare `node`
+// that a signed-in shell finds was not found there. Node looks a bare name
+// up on PATH only, so the programs Windows ships are called by their full
+// path under %SystemRoot%; one that is not there falls back to the bare
+// name, which is what every call site did before.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const WINDOWS_DEFAULT_ROOT = 'C:\\Windows';
+
+function text(value) {
+  return String(value == null ? '' : value).trim();
+}
+
+/**
+ * The value of an environment variable, matched without regard to case.
+ * Windows names are case-insensitive, and a copied environment (`{ ...env }`)
+ * keeps whatever case it came with: `Path`, not `PATH`, on most machines.
+ */
+export function readEnvValue(env, name) {
+  if (!env || typeof env !== 'object') return '';
+  if (Object.prototype.hasOwnProperty.call(env, name)) return text(env[name]);
+  const wanted = String(name).toLowerCase();
+  for (const key of Object.keys(env)) {
+    if (key.toLowerCase() === wanted) return text(env[key]);
+  }
+  return '';
+}
+
+/** The key PATH is stored under in `env` (`Path` on most Windows machines). */
+export function envPathKey(env, platform = process.platform) {
+  if (env && typeof env === 'object') {
+    if (Object.prototype.hasOwnProperty.call(env, 'PATH')) return 'PATH';
+    if (platform === 'win32') {
+      const found = Object.keys(env).find((key) => key.toLowerCase() === 'path');
+      if (found) return found;
+    }
+  }
+  return platform === 'win32' ? 'Path' : 'PATH';
+}
+
+export function windowsSystemRoot(env = process.env) {
+  return readEnvValue(env, 'SystemRoot') || readEnvValue(env, 'windir') || WINDOWS_DEFAULT_ROOT;
+}
+
+const TOOL_SUBPATHS = Object.freeze({
+  'powershell.exe': ['System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'],
+});
+
+/**
+ * Full path of a program that ships with Windows (`schtasks.exe`,
+ * `powershell.exe`, `taskkill.exe`, ...), or the bare name when the file is
+ * not where Windows keeps it.
+ */
+export function windowsSystemToolPath(name, {
+  env = process.env,
+  existsImpl = fs.existsSync,
+} = {}) {
+  const bare = text(name);
+  const fileName = /\.exe$/i.test(bare) ? bare : `${bare}.exe`;
+  const parts = TOOL_SUBPATHS[fileName.toLowerCase()] || ['System32', fileName];
+  const full = path.win32.join(windowsSystemRoot(env), ...parts);
+  try {
+    if (existsImpl(full)) return full;
+  } catch {}
+  return bare;
+}
+
+/** The command interpreter: %ComSpec%, else cmd.exe under %SystemRoot%. */
+export function windowsCmdPath({ env = process.env, existsImpl = fs.existsSync } = {}) {
+  return readEnvValue(env, 'ComSpec') || windowsSystemToolPath('cmd.exe', { env, existsImpl });
+}
+
+/**
+ * One log line's worth of the environment a child was started with: PATH in
+ * full (the entries are what a failed lookup needs), SystemRoot and ComSpec.
+ */
+export function describeSpawnEnvironment(env = process.env, platform = process.platform) {
+  const pathValue = readEnvValue(env, 'PATH');
+  const parts = [`PATH=${pathValue || '(not set)'}`];
+  if (platform === 'win32') {
+    parts.push(`SystemRoot=${readEnvValue(env, 'SystemRoot') || '(not set)'}`);
+    parts.push(`ComSpec=${readEnvValue(env, 'ComSpec') || '(not set)'}`);
+  }
+  return parts.join(' ');
+}
+
+/** What a failed execFile/execFileSync reports, on one line. */
+export function describeExecError(error) {
+  if (!error) return 'unknown error';
+  const parts = [];
+  const code = error.code ?? error.status;
+  if (code !== undefined && code !== null) parts.push(`code=${code}`);
+  if (error.signal) parts.push(`signal=${error.signal}`);
+  const stderr = text(Buffer.isBuffer(error.stderr) ? error.stderr.toString('utf8') : error.stderr);
+  const message = text(error.message).split(/\r?\n/)[0];
+  if (message) parts.push(message);
+  if (stderr) parts.push(`stderr=${stderr.replace(/\s+/g, ' ').slice(0, 500)}`);
+  return parts.join(' ') || 'unknown error';
+}

package/shared/worker-runtime/background-task-digest.mjs

@@ -0,0 +1,30 @@
+import { createHash } from "node:crypto";
+
+/**
+ * The digest of a background-task set, as a worker's heartbeat carries it and
+ * the relay computes it for the set it holds: `{ count, digest }` over the
+ * sorted task ids only. Descriptions, summaries and token counts change every
+ * few seconds and are no reason to send the set again; membership is.
+ */
+export function backgroundTaskDigest(taskIds) {
+  const ids = [...new Set((Array.isArray(taskIds) ? taskIds : [])
+    .map((id) => String(id ?? "").trim())
+    .filter(Boolean))].sort();
+  return {
+    count: ids.length,
+    digest: createHash("sha256").update(ids.join("\n")).digest("hex").slice(0, 16),
+  };
+}
+
+/** A digest off the wire, or null when the sender gave none (it has no opinion). */
+export function normalizeBackgroundTaskDigest(raw) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const digest = String(raw.digest ?? "").trim();
+  const count = Number(raw.count);
+  if (!/^[0-9a-f]{16}$/.test(digest) || !Number.isInteger(count) || count < 0) return null;
+  return { count, digest };
+}
+
+export function sameBackgroundTaskDigest(a, b) {
+  return !!a && !!b && a.count === b.count && a.digest === b.digest;
+}

package/shared/worker-runtime/heartbeat.mjs

@@ -1,3 +1,5 @@
+import { backgroundTaskDigest } from "./background-task-digest.mjs";
+
 export function createHeartbeatController({
   api,
   pollMs,
@@ -19,6 +21,18 @@ export function createHeartbeatController({
   // `settleFailedHandled`, which `onSettleFailedHandled` receives.
   getSettleFailed,
   onSettleFailedHandled = () => {},
+  // Optional: the ids of the worker's live background tasks (an array, empty
+  // for none). The body then carries their digest, and a relay whose own set
+  // for this worker differs (it restarted, or a post was lost) answers
+  // `resyncBackgroundTasks`: `onResyncBackgroundTasks` publishes the full
+  // set again. It is asked again by every beat until the sets agree, so a
+  // failed publish needs no retry of its own. Workers that publish no task
+  // sets leave both unset; the relay reads a missing digest as no opinion.
+  getBackgroundTaskIds,
+  onResyncBackgroundTasks = () => {},
+  // Optional: called after every beat the relay answered, for work that
+  // waits for a reachable relay (posts kept after a failure).
+  onAnswered = () => {},
   // requestPulse's window: one turn transition can flip the steering
   // snapshot several times in quick succession, and each flip only needs to
   // reach the relay once.
@@ -43,11 +57,15 @@ export function createHeartbeatController({
       const settleFailed = typeof getSettleFailed === "function"
         ? (getSettleFailed() || []).filter((entry) => entry && String(entry.id || "").trim())
         : [];
+      const backgroundTaskIds = typeof getBackgroundTaskIds === "function"
+        ? getBackgroundTaskIds()
+        : null;
       const body = {
         ...(activeQueueMessageId ? { activeQueueMessageId } : {}),
         ...(activeQueueMessageIds.length ? { activeQueueMessageIds } : {}),
         ...(steering && typeof steering === "object" ? { steering } : {}),
         ...(settleFailed.length ? { settleFailed } : {}),
+        ...(Array.isArray(backgroundTaskIds) ? { backgroundTasks: backgroundTaskDigest(backgroundTaskIds) } : {}),
       };
       const response = await api("POST", "/api/heartbeat", body);
       // Failed by the relay, or skipped because this worker has no claim on
@@ -59,6 +77,12 @@ export function createHeartbeatController({
       if (handled.length) {
         try { onSettleFailedHandled(handled); } catch {}
       }
+      // Kept posts first: an end the relay has not heard of yet goes before
+      // the set that no longer lists the task.
+      try { await onAnswered(); } catch {}
+      if (response?.resyncBackgroundTasks === true) {
+        try { await onResyncBackgroundTasks(); } catch {}
+      }
       return true;
     } catch {
       return false;

package/shared/worker-runtime/kept-posts.mjs

@@ -0,0 +1,83 @@
+/**
+ * Posts that must not be lost to a relay that is away for a moment: a post
+ * that fails is kept and sent again once the relay answers (the worker's
+ * heartbeat calls `flush`). Bounded both ways, the newest `max` posts for
+ * `ttlMs`, because what they report is only worth telling for a while and a
+ * relay that stays away must not grow the worker.
+ *
+ * A post the relay refused (a 4xx answer) is not kept: sending it again
+ * would be refused again.
+ */
+export const KEPT_POSTS_MAX = 20;
+export const KEPT_POSTS_TTL_MS = 10 * 60_000;
+// A post that gets no answer is given up after this long and kept: fetch
+// alone waits five minutes, and the heartbeat's resync waits for the flush.
+export const KEPT_POSTS_TIMEOUT_MS = 10_000;
+
+export function createKeptPosts({
+  api,
+  max = KEPT_POSTS_MAX,
+  ttlMs = KEPT_POSTS_TTL_MS,
+  timeoutMs = KEPT_POSTS_TIMEOUT_MS,
+  now = () => Date.now(),
+  dbg = () => {},
+} = {}) {
+  let kept = [];
+  let flushing = null;
+
+  function worthKeeping(error) {
+    const status = Number(error?.status);
+    return !(status >= 400 && status < 500);
+  }
+
+  function send(routePath, body) {
+    return api("POST", routePath, body, timeoutMs > 0 ? { signal: AbortSignal.timeout(timeoutMs) } : {});
+  }
+
+  function keep(entry) {
+    kept.push(entry);
+    if (kept.length > max) kept = kept.slice(-max);
+  }
+
+  /** Post now; on a failure keep it for `flush`. Never rejects. */
+  async function post(routePath, body) {
+    try {
+      await send(routePath, body);
+      return true;
+    } catch (error) {
+      dbg("post failed", routePath, error?.message || String(error));
+      if (worthKeeping(error)) keep({ routePath, body, at: now() });
+      return false;
+    }
+  }
+
+  /** Send the kept posts again, oldest first; what fails again stays. */
+  function flush() {
+    if (flushing) return flushing;
+    if (!kept.length) return Promise.resolve(0);
+    flushing = (async () => {
+      const pending = kept;
+      kept = [];
+      let sent = 0;
+      for (let index = 0; index < pending.length; index += 1) {
+        const entry = pending[index];
+        if (now() - entry.at > ttlMs) continue;
+        try {
+          await send(entry.routePath, entry.body);
+          sent += 1;
+        } catch (error) {
+          dbg("kept post failed again", entry.routePath, error?.message || String(error));
+          if (!worthKeeping(error)) continue;
+          // The relay is still away: the rest waits too, in order, ahead of
+          // whatever was kept meanwhile.
+          kept = [...pending.slice(index).filter((later) => now() - later.at <= ttlMs), ...kept].slice(-max);
+          break;
+        }
+      }
+      return sent;
+    })().finally(() => { flushing = null; });
+    return flushing;
+  }
+
+  return { post, flush, size: () => kept.length };
+}

package/shared/worker-runtime/task-output-spool.mjs

@@ -0,0 +1,181 @@
+// The spool a session worker writes a background task's output into when the
+// provider keeps none in a file of its own (Copilot's attached shells, Grok's
+// terminals): `<os.tmpdir()>/oar-task-output/<sessionId>/<taskId>.log`,
+// append-only. The worker names the file as the task row's `outputPath`
+// (`outputKind: 'spool'`) and the relay reads it like any other output file,
+// so the console works the same for every provider and outlives a worker
+// restart.
+//
+// Bounded: a file stops growing at 50 MB with one "output capped" line, and
+// `sweep()` (at worker start and hourly) deletes files that ended more than an
+// hour ago. Nothing is deleted at exit: the relay keeps showing an ended
+// task's output for about an hour.
+
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+export const TASK_OUTPUT_SPOOL_DIR = 'oar-task-output';
+export const TASK_OUTPUT_SPOOL_MAX_BYTES = 50 * 1024 * 1024;
+export const TASK_OUTPUT_SPOOL_MAX_AGE_MS = 60 * 60 * 1000;
+// A dead worker's session folder is left for its own next start to sweep;
+// another session's worker removes it only once it is a day old.
+export const TASK_OUTPUT_SPOOL_FOREIGN_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
+const CAPPED_LINE = '\n[output capped at 50 MB: later output is not kept]\n';
+
+/** A file-name-safe form of an id: [A-Za-z0-9._-], at most 128 characters, never `.` or `..`. */
+export function spoolSafeName(value) {
+  const text = String(value ?? '').trim().replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 128);
+  return /^\.+$/.test(text) ? text.replace(/\./g, '_') : text;
+}
+
+/**
+ * `createTaskOutputSpool({ sessionId })` →
+ * `{ root, pathFor(taskId), append(taskId, text), end(taskId), sweep(maxAgeMs) }`.
+ *
+ *  - `pathFor(taskId)`: the absolute file path for the task's row.
+ *  - `append(taskId, text)`: `{ ok, bytes, capped }`; never throws. The cap
+ *    writes one marker line and drops what comes after it.
+ *  - `end(taskId)`: the task finished; its file stays for the relay until a
+ *    sweep finds it older than the age limit.
+ *  - `sweep(maxAgeMs = 1 h)`: deletes this session's files not written to for
+ *    that long (a task still appending is never swept), and other sessions'
+ *    folders a day old. Returns the number of files removed.
+ *
+ * `baseDir` (default the OS temp dir) and `now` are for tests.
+ */
+export function createTaskOutputSpool({
+  sessionId,
+  now = () => Date.now(),
+  baseDir = os.tmpdir(),
+  maxBytes = TASK_OUTPUT_SPOOL_MAX_BYTES,
+  fsImpl = fs,
+} = {}) {
+  const sessionName = spoolSafeName(sessionId);
+  if (!sessionName) throw new Error('createTaskOutputSpool requires sessionId');
+  const top = path.join(path.resolve(baseDir), TASK_OUTPUT_SPOOL_DIR);
+  const root = path.join(top, sessionName);
+  // taskId → { bytes, capped }; a task listed here is live (still appending).
+  const live = new Map();
+  let rootReady = false;
+
+  // The shared folder is ours to write and sweep only when it is a real
+  // folder (not a symlink someone else put into a shared temp dir) owned by
+  // this user where the platform says who owns it.
+  function topIsOurs() {
+    let stats;
+    try { stats = fsImpl.lstatSync(top); } catch { return false; }
+    if (stats.isSymbolicLink() || !stats.isDirectory()) return false;
+    const uid = typeof process.getuid === 'function' ? process.getuid() : null;
+    return uid === null || typeof stats.uid !== 'number' || stats.uid === uid;
+  }
+
+  function ensureRoot() {
+    if (rootReady) return;
+    // Owner-only: the temp dir is shared on a multi-user host.
+    try { fsImpl.mkdirSync(top, { mode: 0o700 }); } catch (error) { if (error?.code !== 'EEXIST') throw error; }
+    if (!topIsOurs()) throw new Error(`${top} is not a folder of this user`);
+    fsImpl.mkdirSync(root, { recursive: true, mode: 0o700 });
+    rootReady = true;
+  }
+
+  function pathFor(taskId) {
+    const name = spoolSafeName(taskId);
+    if (!name) throw new Error('task id is empty');
+    return path.join(root, `${name}.log`);
+  }
+
+  function stateOf(taskId, file) {
+    let state = live.get(taskId);
+    if (!state) {
+      let bytes = 0;
+      try { bytes = fsImpl.statSync(file).size; } catch {}
+      state = { bytes, capped: bytes >= maxBytes };
+      live.set(taskId, state);
+    }
+    return state;
+  }
+
+  function append(taskId, text) {
+    const id = String(taskId ?? '').trim();
+    const value = typeof text === 'string' ? text : (Buffer.isBuffer(text) ? text.toString('utf8') : String(text ?? ''));
+    if (!id) return { ok: false, bytes: 0, capped: false };
+    try {
+      ensureRoot();
+      const file = pathFor(id);
+      const state = stateOf(id, file);
+      if (state.capped || !value) return { ok: true, bytes: state.bytes, capped: state.capped };
+      let data = Buffer.from(value, 'utf8');
+      const room = maxBytes - state.bytes;
+      if (data.length > room) {
+        // Cut on a character boundary, then say so once.
+        let end = Math.max(0, room);
+        while (end > 0 && (data[end] & 0xc0) === 0x80) end -= 1;
+        data = Buffer.concat([data.subarray(0, end), Buffer.from(CAPPED_LINE, 'utf8')]);
+        state.capped = true;
+      }
+      fsImpl.appendFileSync(file, data, { mode: 0o600 });
+      state.bytes += data.length;
+      return { ok: true, bytes: state.bytes, capped: state.capped };
+    } catch {
+      return { ok: false, bytes: live.get(id)?.bytes || 0, capped: !!live.get(id)?.capped };
+    }
+  }
+
+  function end(taskId) {
+    live.delete(String(taskId ?? '').trim());
+  }
+
+  function liveNames() {
+    return new Set([...live.keys()].map((id) => `${spoolSafeName(id)}.log`));
+  }
+
+  function sweepDir(dir, maxAge, skip = new Set()) {
+    let removed = 0;
+    let left = 0;
+    let entries = [];
+    try { entries = fsImpl.readdirSync(dir, { withFileTypes: true }); } catch { return { removed, left: 0, missing: true }; }
+    const cutoff = now() - maxAge;
+    for (const entry of entries) {
+      if (!entry.isFile() || skip.has(entry.name)) {
+        left += 1;
+        continue;
+      }
+      const file = path.join(dir, entry.name);
+      try {
+        if (fsImpl.statSync(file).mtimeMs < cutoff) {
+          fsImpl.unlinkSync(file);
+          removed += 1;
+        } else {
+          left += 1;
+        }
+      } catch {
+        left += 1;
+      }
+    }
+    return { removed, left, missing: false };
+  }
+
+  function sweep(maxAgeMs = TASK_OUTPUT_SPOOL_MAX_AGE_MS) {
+    const maxAge = Number.isFinite(Number(maxAgeMs)) && Number(maxAgeMs) >= 0 ? Number(maxAgeMs) : TASK_OUTPUT_SPOOL_MAX_AGE_MS;
+    // Never through a symlinked or foreign shared folder: its "session
+    // folders" could be anything.
+    if (!topIsOurs()) return 0;
+    let removed = sweepDir(root, maxAge, liveNames()).removed;
+    let folders = [];
+    try { folders = fsImpl.readdirSync(top, { withFileTypes: true }); } catch {}
+    for (const folder of folders) {
+      if (!folder.isDirectory() || folder.name === sessionName) continue;
+      const dir = path.join(top, folder.name);
+      const result = sweepDir(dir, Math.max(maxAge, TASK_OUTPUT_SPOOL_FOREIGN_MAX_AGE_MS));
+      removed += result.removed;
+      if (!result.missing && result.left === 0) {
+        try { fsImpl.rmdirSync(dir); } catch {}
+      }
+    }
+    return removed;
+  }
+
+  return { root, pathFor, append, end, sweep };
+}

package/shared/worker-runtime/worker-websocket-link.mjs

@@ -84,6 +84,8 @@ export function createWorkerWebSocketLink({
   let readyRefreshTimer = null;
   let reconnectAttempt = 0;
   let reconnectDelayMs = 0;
+  // Ends the current socket's connection (see connect).
+  let endCurrentSocket = null;
   // Insertion-ordered: the first entry is the oldest in-flight delivery, which
   // is what a legacy worker's coalesced delivery waits on.
   const deliveriesInFlight = new Set();
@@ -246,7 +248,11 @@ export function createWorkerWebSocketLink({
   function closeStaleSocket(reason) {
     if (!isSocketOpen()) return false;
     dbg("worker ws stale connection closing", reason);
-    try { ws?.close?.(); } catch {}
+    const socket = ws;
+    // A dead peer may never complete the close handshake: the reconnect does
+    // not wait for the close event.
+    endCurrentSocket?.(`stale:${reason}`);
+    try { socket?.close?.(); } catch {}
     return true;
   }
 
@@ -286,7 +292,9 @@ export function createWorkerWebSocketLink({
 
   function scheduleReconnect() {
     if (stopped) return;
-    clearReconnectTimer();
+    // One timer at a time: a second end reported for the same outage must not
+    // stack another attempt or push the backoff up.
+    if (reconnectTimer) return;
     reconnectAttempt += 1;
     const backoff = Math.min(maxBackoffMs, minBackoffMs * (2 ** Math.max(0, reconnectAttempt - 1)));
     const jitter = Math.floor(Math.random() * Math.max(0, jitterMs));
@@ -309,10 +317,34 @@ export function createWorkerWebSocketLink({
       dbg("worker ws unavailable: global WebSocket is missing");
       return;
     }
+    clearReconnectTimer();
     const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
-    ws = new WebSocketImpl(url);
+    const socket = new WebSocketImpl(url);
+    ws = socket;
     let serverHelloSeen = false;
-    ws.addEventListener("open", () => {
+    // Set once this socket's end has been handled, by whichever of error and
+    // close comes first; every later event from it is ignored. Node 22 fires
+    // no close for a socket closed while still connecting, and closing it
+    // from its own error handler raised the error again, so the reconnect
+    // has to be scheduled from the error as well, and only once.
+    let ended = false;
+    function endSocket(reason) {
+      if (ended) return false;
+      ended = true;
+      if (ws === socket) {
+        ws = null;
+        clearReadyRefreshTimer();
+        // The next connection may reach a different (restarted) relay.
+        serverCapabilities = new Set();
+      }
+      if (stopped) return true;
+      dbg("worker ws disconnected", reason);
+      scheduleReconnect();
+      return true;
+    }
+    endCurrentSocket = endSocket;
+    socket.addEventListener("open", () => {
+      if (ended) return;
       const openedAt = getNowMs();
       reconnectAttempt = 0;
       reconnectDelayMs = 0;
@@ -323,7 +355,8 @@ export function createWorkerWebSocketLink({
       void notifyReady("ws-open");
       startReadyRefreshTimer();
     });
-    ws.addEventListener("message", (event) => {
+    socket.addEventListener("message", (event) => {
+      if (ended) return;
       lastMessageAt = getNowMs();
       let payload = null;
       try {
@@ -355,7 +388,7 @@ export function createWorkerWebSocketLink({
       }
       if (payload?.type === "server.draining") {
         dbg("worker ws draining", payload.reason || "relay-shutdown");
-        try { ws?.close?.(); } catch {}
+        try { socket.close?.(); } catch {}
         return;
       }
       if (payload?.type === "queue.changed") {
@@ -375,17 +408,16 @@ export function createWorkerWebSocketLink({
         dbg("worker ws blocked", payload.reason || "blocked");
       }
     });
-    ws.addEventListener("close", () => {
-      clearReadyRefreshTimer();
-      // The next connection may reach a different (restarted) relay.
-      serverCapabilities = new Set();
-      if (stopped) return;
-      dbg("worker ws disconnected");
-      scheduleReconnect();
+    socket.addEventListener("close", () => {
+      endSocket("close");
     });
-    ws.addEventListener("error", (error) => {
+    socket.addEventListener("error", (error) => {
+      if (ended) return;
       dbg("worker ws error", error?.message || String(error));
-      try { ws?.close?.(); } catch {}
+      // Ended before the close call, so an error or close it raises
+      // (synchronously on some runtimes) finds the socket already handled.
+      endSocket("error");
+      try { socket.close?.(); } catch {}
     });
   }