@oar-sh/oar 0.9.10 → 0.9.11

package/shared/windows-system-tools.mjs

@@ -0,0 +1,106 @@
+// Windows system programs by absolute path, and the bits of a process
+// environment that decide whether a child can start at all.
+//
+// A relay started by the boot-time Scheduled Task (S4U, before anyone signs
+// in) gets an environment built without a full profile: user PATH entries
+// that refer to other variables can arrive unexpanded, and a bare `node`
+// that a signed-in shell finds was not found there. Node looks a bare name
+// up on PATH only, so the programs Windows ships are called by their full
+// path under %SystemRoot%; one that is not there falls back to the bare
+// name, which is what every call site did before.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const WINDOWS_DEFAULT_ROOT = 'C:\\Windows';
+
+function text(value) {
+  return String(value == null ? '' : value).trim();
+}
+
+/**
+ * The value of an environment variable, matched without regard to case.
+ * Windows names are case-insensitive, and a copied environment (`{ ...env }`)
+ * keeps whatever case it came with: `Path`, not `PATH`, on most machines.
+ */
+export function readEnvValue(env, name) {
+  if (!env || typeof env !== 'object') return '';
+  if (Object.prototype.hasOwnProperty.call(env, name)) return text(env[name]);
+  const wanted = String(name).toLowerCase();
+  for (const key of Object.keys(env)) {
+    if (key.toLowerCase() === wanted) return text(env[key]);
+  }
+  return '';
+}
+
+/** The key PATH is stored under in `env` (`Path` on most Windows machines). */
+export function envPathKey(env, platform = process.platform) {
+  if (env && typeof env === 'object') {
+    if (Object.prototype.hasOwnProperty.call(env, 'PATH')) return 'PATH';
+    if (platform === 'win32') {
+      const found = Object.keys(env).find((key) => key.toLowerCase() === 'path');
+      if (found) return found;
+    }
+  }
+  return platform === 'win32' ? 'Path' : 'PATH';
+}
+
+export function windowsSystemRoot(env = process.env) {
+  return readEnvValue(env, 'SystemRoot') || readEnvValue(env, 'windir') || WINDOWS_DEFAULT_ROOT;
+}
+
+const TOOL_SUBPATHS = Object.freeze({
+  'powershell.exe': ['System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'],
+});
+
+/**
+ * Full path of a program that ships with Windows (`schtasks.exe`,
+ * `powershell.exe`, `taskkill.exe`, ...), or the bare name when the file is
+ * not where Windows keeps it.
+ */
+export function windowsSystemToolPath(name, {
+  env = process.env,
+  existsImpl = fs.existsSync,
+} = {}) {
+  const bare = text(name);
+  const fileName = /\.exe$/i.test(bare) ? bare : `${bare}.exe`;
+  const parts = TOOL_SUBPATHS[fileName.toLowerCase()] || ['System32', fileName];
+  const full = path.win32.join(windowsSystemRoot(env), ...parts);
+  try {
+    if (existsImpl(full)) return full;
+  } catch {}
+  return bare;
+}
+
+/** The command interpreter: %ComSpec%, else cmd.exe under %SystemRoot%. */
+export function windowsCmdPath({ env = process.env, existsImpl = fs.existsSync } = {}) {
+  return readEnvValue(env, 'ComSpec') || windowsSystemToolPath('cmd.exe', { env, existsImpl });
+}
+
+/**
+ * One log line's worth of the environment a child was started with: PATH in
+ * full (the entries are what a failed lookup needs), SystemRoot and ComSpec.
+ */
+export function describeSpawnEnvironment(env = process.env, platform = process.platform) {
+  const pathValue = readEnvValue(env, 'PATH');
+  const parts = [`PATH=${pathValue || '(not set)'}`];
+  if (platform === 'win32') {
+    parts.push(`SystemRoot=${readEnvValue(env, 'SystemRoot') || '(not set)'}`);
+    parts.push(`ComSpec=${readEnvValue(env, 'ComSpec') || '(not set)'}`);
+  }
+  return parts.join(' ');
+}
+
+/** What a failed execFile/execFileSync reports, on one line. */
+export function describeExecError(error) {
+  if (!error) return 'unknown error';
+  const parts = [];
+  const code = error.code ?? error.status;
+  if (code !== undefined && code !== null) parts.push(`code=${code}`);
+  if (error.signal) parts.push(`signal=${error.signal}`);
+  const stderr = text(Buffer.isBuffer(error.stderr) ? error.stderr.toString('utf8') : error.stderr);
+  const message = text(error.message).split(/\r?\n/)[0];
+  if (message) parts.push(message);
+  if (stderr) parts.push(`stderr=${stderr.replace(/\s+/g, ' ').slice(0, 500)}`);
+  return parts.join(' ') || 'unknown error';
+}

package/shared/worker-runtime/background-task-digest.mjs

@@ -0,0 +1,30 @@
+import { createHash } from "node:crypto";
+
+/**
+ * The digest of a background-task set, as a worker's heartbeat carries it and
+ * the relay computes it for the set it holds: `{ count, digest }` over the
+ * sorted task ids only. Descriptions, summaries and token counts change every
+ * few seconds and are no reason to send the set again; membership is.
+ */
+export function backgroundTaskDigest(taskIds) {
+  const ids = [...new Set((Array.isArray(taskIds) ? taskIds : [])
+    .map((id) => String(id ?? "").trim())
+    .filter(Boolean))].sort();
+  return {
+    count: ids.length,
+    digest: createHash("sha256").update(ids.join("\n")).digest("hex").slice(0, 16),
+  };
+}
+
+/** A digest off the wire, or null when the sender gave none (it has no opinion). */
+export function normalizeBackgroundTaskDigest(raw) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const digest = String(raw.digest ?? "").trim();
+  const count = Number(raw.count);
+  if (!/^[0-9a-f]{16}$/.test(digest) || !Number.isInteger(count) || count < 0) return null;
+  return { count, digest };
+}
+
+export function sameBackgroundTaskDigest(a, b) {
+  return !!a && !!b && a.count === b.count && a.digest === b.digest;
+}

package/shared/worker-runtime/heartbeat.mjs

@@ -1,3 +1,5 @@
+import { backgroundTaskDigest } from "./background-task-digest.mjs";
+
 export function createHeartbeatController({
   api,
   pollMs,
@@ -19,6 +21,18 @@ export function createHeartbeatController({
   // `settleFailedHandled`, which `onSettleFailedHandled` receives.
   getSettleFailed,
   onSettleFailedHandled = () => {},
+  // Optional: the ids of the worker's live background tasks (an array, empty
+  // for none). The body then carries their digest, and a relay whose own set
+  // for this worker differs (it restarted, or a post was lost) answers
+  // `resyncBackgroundTasks`: `onResyncBackgroundTasks` publishes the full
+  // set again. It is asked again by every beat until the sets agree, so a
+  // failed publish needs no retry of its own. Workers that publish no task
+  // sets leave both unset; the relay reads a missing digest as no opinion.
+  getBackgroundTaskIds,
+  onResyncBackgroundTasks = () => {},
+  // Optional: called after every beat the relay answered, for work that
+  // waits for a reachable relay (posts kept after a failure).
+  onAnswered = () => {},
   // requestPulse's window: one turn transition can flip the steering
   // snapshot several times in quick succession, and each flip only needs to
   // reach the relay once.
@@ -43,11 +57,15 @@ export function createHeartbeatController({
       const settleFailed = typeof getSettleFailed === "function"
         ? (getSettleFailed() || []).filter((entry) => entry && String(entry.id || "").trim())
         : [];
+      const backgroundTaskIds = typeof getBackgroundTaskIds === "function"
+        ? getBackgroundTaskIds()
+        : null;
       const body = {
         ...(activeQueueMessageId ? { activeQueueMessageId } : {}),
         ...(activeQueueMessageIds.length ? { activeQueueMessageIds } : {}),
         ...(steering && typeof steering === "object" ? { steering } : {}),
         ...(settleFailed.length ? { settleFailed } : {}),
+        ...(Array.isArray(backgroundTaskIds) ? { backgroundTasks: backgroundTaskDigest(backgroundTaskIds) } : {}),
       };
       const response = await api("POST", "/api/heartbeat", body);
       // Failed by the relay, or skipped because this worker has no claim on
@@ -59,6 +77,12 @@ export function createHeartbeatController({
       if (handled.length) {
         try { onSettleFailedHandled(handled); } catch {}
       }
+      // Kept posts first: an end the relay has not heard of yet goes before
+      // the set that no longer lists the task.
+      try { await onAnswered(); } catch {}
+      if (response?.resyncBackgroundTasks === true) {
+        try { await onResyncBackgroundTasks(); } catch {}
+      }
       return true;
     } catch {
       return false;

package/shared/worker-runtime/kept-posts.mjs

@@ -0,0 +1,83 @@
+/**
+ * Posts that must not be lost to a relay that is away for a moment: a post
+ * that fails is kept and sent again once the relay answers (the worker's
+ * heartbeat calls `flush`). Bounded both ways, the newest `max` posts for
+ * `ttlMs`, because what they report is only worth telling for a while and a
+ * relay that stays away must not grow the worker.
+ *
+ * A post the relay refused (a 4xx answer) is not kept: sending it again
+ * would be refused again.
+ */
+export const KEPT_POSTS_MAX = 20;
+export const KEPT_POSTS_TTL_MS = 10 * 60_000;
+// A post that gets no answer is given up after this long and kept: fetch
+// alone waits five minutes, and the heartbeat's resync waits for the flush.
+export const KEPT_POSTS_TIMEOUT_MS = 10_000;
+
+export function createKeptPosts({
+  api,
+  max = KEPT_POSTS_MAX,
+  ttlMs = KEPT_POSTS_TTL_MS,
+  timeoutMs = KEPT_POSTS_TIMEOUT_MS,
+  now = () => Date.now(),
+  dbg = () => {},
+} = {}) {
+  let kept = [];
+  let flushing = null;
+
+  function worthKeeping(error) {
+    const status = Number(error?.status);
+    return !(status >= 400 && status < 500);
+  }
+
+  function send(routePath, body) {
+    return api("POST", routePath, body, timeoutMs > 0 ? { signal: AbortSignal.timeout(timeoutMs) } : {});
+  }
+
+  function keep(entry) {
+    kept.push(entry);
+    if (kept.length > max) kept = kept.slice(-max);
+  }
+
+  /** Post now; on a failure keep it for `flush`. Never rejects. */
+  async function post(routePath, body) {
+    try {
+      await send(routePath, body);
+      return true;
+    } catch (error) {
+      dbg("post failed", routePath, error?.message || String(error));
+      if (worthKeeping(error)) keep({ routePath, body, at: now() });
+      return false;
+    }
+  }
+
+  /** Send the kept posts again, oldest first; what fails again stays. */
+  function flush() {
+    if (flushing) return flushing;
+    if (!kept.length) return Promise.resolve(0);
+    flushing = (async () => {
+      const pending = kept;
+      kept = [];
+      let sent = 0;
+      for (let index = 0; index < pending.length; index += 1) {
+        const entry = pending[index];
+        if (now() - entry.at > ttlMs) continue;
+        try {
+          await send(entry.routePath, entry.body);
+          sent += 1;
+        } catch (error) {
+          dbg("kept post failed again", entry.routePath, error?.message || String(error));
+          if (!worthKeeping(error)) continue;
+          // The relay is still away: the rest waits too, in order, ahead of
+          // whatever was kept meanwhile.
+          kept = [...pending.slice(index).filter((later) => now() - later.at <= ttlMs), ...kept].slice(-max);
+          break;
+        }
+      }
+      return sent;
+    })().finally(() => { flushing = null; });
+    return flushing;
+  }
+
+  return { post, flush, size: () => kept.length };
+}

package/shared/worker-runtime/worker-websocket-link.mjs

@@ -84,6 +84,8 @@ export function createWorkerWebSocketLink({
   let readyRefreshTimer = null;
   let reconnectAttempt = 0;
   let reconnectDelayMs = 0;
+  // Ends the current socket's connection (see connect).
+  let endCurrentSocket = null;
   // Insertion-ordered: the first entry is the oldest in-flight delivery, which
   // is what a legacy worker's coalesced delivery waits on.
   const deliveriesInFlight = new Set();
@@ -246,7 +248,11 @@ export function createWorkerWebSocketLink({
   function closeStaleSocket(reason) {
     if (!isSocketOpen()) return false;
     dbg("worker ws stale connection closing", reason);
-    try { ws?.close?.(); } catch {}
+    const socket = ws;
+    // A dead peer may never complete the close handshake: the reconnect does
+    // not wait for the close event.
+    endCurrentSocket?.(`stale:${reason}`);
+    try { socket?.close?.(); } catch {}
     return true;
   }
 
@@ -286,7 +292,9 @@ export function createWorkerWebSocketLink({
 
   function scheduleReconnect() {
     if (stopped) return;
-    clearReconnectTimer();
+    // One timer at a time: a second end reported for the same outage must not
+    // stack another attempt or push the backoff up.
+    if (reconnectTimer) return;
     reconnectAttempt += 1;
     const backoff = Math.min(maxBackoffMs, minBackoffMs * (2 ** Math.max(0, reconnectAttempt - 1)));
     const jitter = Math.floor(Math.random() * Math.max(0, jitterMs));
@@ -309,10 +317,34 @@ export function createWorkerWebSocketLink({
       dbg("worker ws unavailable: global WebSocket is missing");
       return;
     }
+    clearReconnectTimer();
     const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
-    ws = new WebSocketImpl(url);
+    const socket = new WebSocketImpl(url);
+    ws = socket;
     let serverHelloSeen = false;
-    ws.addEventListener("open", () => {
+    // Set once this socket's end has been handled, by whichever of error and
+    // close comes first; every later event from it is ignored. Node 22 fires
+    // no close for a socket closed while still connecting, and closing it
+    // from its own error handler raised the error again, so the reconnect
+    // has to be scheduled from the error as well, and only once.
+    let ended = false;
+    function endSocket(reason) {
+      if (ended) return false;
+      ended = true;
+      if (ws === socket) {
+        ws = null;
+        clearReadyRefreshTimer();
+        // The next connection may reach a different (restarted) relay.
+        serverCapabilities = new Set();
+      }
+      if (stopped) return true;
+      dbg("worker ws disconnected", reason);
+      scheduleReconnect();
+      return true;
+    }
+    endCurrentSocket = endSocket;
+    socket.addEventListener("open", () => {
+      if (ended) return;
       const openedAt = getNowMs();
       reconnectAttempt = 0;
       reconnectDelayMs = 0;
@@ -323,7 +355,8 @@ export function createWorkerWebSocketLink({
       void notifyReady("ws-open");
       startReadyRefreshTimer();
     });
-    ws.addEventListener("message", (event) => {
+    socket.addEventListener("message", (event) => {
+      if (ended) return;
       lastMessageAt = getNowMs();
       let payload = null;
       try {
@@ -355,7 +388,7 @@ export function createWorkerWebSocketLink({
       }
       if (payload?.type === "server.draining") {
         dbg("worker ws draining", payload.reason || "relay-shutdown");
-        try { ws?.close?.(); } catch {}
+        try { socket.close?.(); } catch {}
         return;
       }
       if (payload?.type === "queue.changed") {
@@ -375,17 +408,16 @@ export function createWorkerWebSocketLink({
         dbg("worker ws blocked", payload.reason || "blocked");
       }
     });
-    ws.addEventListener("close", () => {
-      clearReadyRefreshTimer();
-      // The next connection may reach a different (restarted) relay.
-      serverCapabilities = new Set();
-      if (stopped) return;
-      dbg("worker ws disconnected");
-      scheduleReconnect();
+    socket.addEventListener("close", () => {
+      endSocket("close");
     });
-    ws.addEventListener("error", (error) => {
+    socket.addEventListener("error", (error) => {
+      if (ended) return;
       dbg("worker ws error", error?.message || String(error));
-      try { ws?.close?.(); } catch {}
+      // Ended before the close call, so an error or close it raises
+      // (synchronously on some runtimes) finds the socket already handled.
+      endSocket("error");
+      try { socket.close?.(); } catch {}
     });
   }