@oar-sh/oar 0.9.2 → 0.9.4

package/shared/worker-runtime/api-client.mjs

@@ -1,33 +1,123 @@
+import http from "node:http";
+import https from "node:https";
+
+// The error a non-2xx answer throws, the same for both transports:
+// `HTTP <status> <path>: <detail>`, with `status`, `detail` and `body`.
+function httpError(status, routePath, text) {
+  const rawText = String(text || "").trim();
+  let detail = rawText;
+  let payload = null;
+  if (rawText) {
+    try {
+      payload = JSON.parse(rawText);
+      detail = String(payload?.error || payload?.message || rawText).trim();
+    } catch {
+      detail = rawText;
+    }
+  }
+  const error = new Error(`HTTP ${status} ${routePath}${detail ? `: ${detail}` : ""}`);
+  error.status = status;
+  error.detail = detail;
+  // The parsed error body, for callers that pass the relay's answer on as
+  // is (a refused remote_relay call carries its code there).
+  error.body = payload && typeof payload === "object" ? payload : null;
+  return error;
+}
+
+function abortError(signal) {
+  const error = new Error("The operation was aborted");
+  error.name = "AbortError";
+  error.code = "ABORT_ERR";
+  if (signal?.reason !== undefined) error.cause = signal.reason;
+  return error;
+}
+
+/**
+ * One request over node:http(s) with no clock on it at all. Global fetch
+ * (undici) gives up when no response headers arrive within 300 s, and a
+ * long call's route sends nothing until it has its answer: a remote turn of
+ * up to 600 s, or an approval card that may sit for hours. So: its own
+ * connection (`agent: false` — the global agent carries a socket timeout),
+ * no request timeout, no body timeout. Only `signal` ends it early; the
+ * relay notices the closed request and stops waiting on its side too.
+ */
+function requestWithoutTimeout({ url, method, headers, payload, signal }) {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(abortError(signal));
+      return;
+    }
+    const target = new URL(url);
+    const transport = target.protocol === "https:" ? https : http;
+    let settled = false;
+    let req = null;
+    const onAbort = () => {
+      const error = abortError(signal);
+      finish(error);
+      req?.destroy(error);
+    };
+    function finish(error, value) {
+      if (settled) return;
+      settled = true;
+      signal?.removeEventListener?.("abort", onAbort);
+      if (error) reject(error);
+      else resolve(value);
+    }
+    req = transport.request(target, {
+      method,
+      agent: false,
+      headers: {
+        ...headers,
+        ...(payload !== undefined ? { "Content-Length": Buffer.byteLength(payload) } : {}),
+      },
+    }, (res) => {
+      const chunks = [];
+      res.on("data", (chunk) => { chunks.push(chunk); });
+      res.on("end", () => finish(null, { status: res.statusCode, text: Buffer.concat(chunks).toString("utf8") }));
+      res.on("error", (error) => finish(error));
+      res.on("close", () => {
+        if (!res.complete) finish(new Error(`The relay closed the connection mid-answer (${url})`));
+      });
+    });
+    req.on("error", (error) => finish(error));
+    signal?.addEventListener?.("abort", onAbort, { once: true });
+    if (payload !== undefined) req.write(payload);
+    req.end();
+  });
+}
+
 export function createApiClient({ serverUrl, token, getHeaders }) {
-  return async function api(method, routePath, body) {
+  /**
+   * `options.longCall`: a call the relay may legitimately hold open for many
+   * minutes (the remote_relay tool) — it goes over node:http without any
+   * timeout instead of fetch. `options.signal` aborts either kind; an aborted
+   * call rejects with an AbortError.
+   */
+  return async function api(method, routePath, body, { longCall = false, signal = null } = {}) {
     const url = `${serverUrl}${routePath}`;
     const extraHeaders = typeof getHeaders === "function" ? (getHeaders() || {}) : {};
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${token}`,
+      ...extraHeaders,
+    };
+    const payload = method !== "GET" ? JSON.stringify(body || {}) : undefined;
+
+    if (longCall) {
+      const res = await requestWithoutTimeout({ url, method, headers, payload, signal });
+      if (res.status < 200 || res.status > 299) throw httpError(res.status, routePath, res.text);
+      return JSON.parse(res.text);
+    }
+
     const opts = {
       method,
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${token}`,
-        ...extraHeaders,
-      },
-      ...(method !== "GET" ? { body: JSON.stringify(body || {}) } : {}),
+      headers,
+      ...(payload !== undefined ? { body: payload } : {}),
+      ...(signal ? { signal } : {}),
     };
-
     const res = await fetch(url, opts);
     if (!res.ok) {
-      const rawText = String(await res.text().catch(() => "")).trim();
-      let detail = rawText;
-      if (rawText) {
-        try {
-          const payload = JSON.parse(rawText);
-          detail = String(payload?.error || payload?.message || rawText).trim();
-        } catch {
-          detail = rawText;
-        }
-      }
-      const error = new Error(`HTTP ${res.status} ${routePath}${detail ? `: ${detail}` : ""}`);
-      error.status = res.status;
-      error.detail = detail;
-      throw error;
+      throw httpError(res.status, routePath, await res.text().catch(() => ""));
     }
     return res.json();
   };

package/shared/worker-runtime/heartbeat.mjs

@@ -10,7 +10,24 @@ export function createHeartbeatController({
   // queued behind it, and a background continuation at once). Each reported
   // id gets its owner lease refreshed; unreported owned rows are recovered.
   getActiveQueueMessageIds,
+  // Optional: the worker's steering snapshot ({turnActive, canSteer,
+  // holdReason}) for the composer. Workers that do not steer leave it unset
+  // and the heartbeat body is unchanged.
+  getSteeringState,
+  // Optional: rows the worker gave up settling ([{id, attemptId,
+  // terminalError}]); the relay fails them and names them back in
+  // `settleFailedHandled`, which `onSettleFailedHandled` receives.
+  getSettleFailed,
+  onSettleFailedHandled = () => {},
+  // requestPulse's window: one turn transition can flip the steering
+  // snapshot several times in quick succession, and each flip only needs to
+  // reach the relay once.
+  pulseCoalesceMs = 300,
+  setTimeoutImpl = setTimeout,
+  clearTimeoutImpl = clearTimeout,
 }) {
+  let requestedPulseTimer = null;
+
   async function pulseHeartbeat() {
     if (!getSessionReady()) return false;
     try {
@@ -20,17 +37,49 @@ export function createHeartbeatController({
       const activeQueueMessageIds = typeof getActiveQueueMessageIds === "function"
         ? (getActiveQueueMessageIds() || []).map((id) => String(id || "").trim()).filter(Boolean)
         : [];
+      const steering = typeof getSteeringState === "function"
+        ? getSteeringState()
+        : null;
+      const settleFailed = typeof getSettleFailed === "function"
+        ? (getSettleFailed() || []).filter((entry) => entry && String(entry.id || "").trim())
+        : [];
       const body = {
         ...(activeQueueMessageId ? { activeQueueMessageId } : {}),
         ...(activeQueueMessageIds.length ? { activeQueueMessageIds } : {}),
+        ...(steering && typeof steering === "object" ? { steering } : {}),
+        ...(settleFailed.length ? { settleFailed } : {}),
       };
-      await api("POST", "/api/heartbeat", body);
+      const response = await api("POST", "/api/heartbeat", body);
+      // Failed by the relay, or skipped because this worker has no claim on
+      // the row: either way the worker releases it.
+      const handled = [
+        ...(Array.isArray(response?.settleFailedHandled) ? response.settleFailedHandled : []),
+        ...(Array.isArray(response?.settleFailedSkipped) ? response.settleFailedSkipped : []),
+      ];
+      if (handled.length) {
+        try { onSettleFailedHandled(handled); } catch {}
+      }
       return true;
     } catch {
       return false;
     }
   }
 
+  /**
+   * An out-of-cycle pulse for state the relay should see before the next
+   * periodic one (the composer's steering snapshot, a relay that just came
+   * up). Requests inside one window share a single pulse, sent at its end so
+   * it carries the settled state.
+   */
+  function requestPulse() {
+    if (requestedPulseTimer) return;
+    requestedPulseTimer = setTimeoutImpl(() => {
+      requestedPulseTimer = null;
+      void pulseHeartbeat();
+    }, pulseCoalesceMs);
+    requestedPulseTimer?.unref?.();
+  }
+
   function startHeartbeat() {
     if (getHeartbeatTimer()) return;
     void pulseHeartbeat();
@@ -41,6 +90,10 @@ export function createHeartbeatController({
   }
 
   function stopHeartbeat() {
+    if (requestedPulseTimer) {
+      clearTimeoutImpl(requestedPulseTimer);
+      requestedPulseTimer = null;
+    }
     const timer = getHeartbeatTimer();
     if (!timer) return;
     clearInterval(timer);
@@ -49,6 +102,7 @@ export function createHeartbeatController({
 
   return {
     pulseHeartbeat,
+    requestPulse,
     startHeartbeat,
     stopHeartbeat,
   };

package/shared/worker-runtime/runner-link-wiring.mjs

@@ -0,0 +1,44 @@
+// Glue between a steering-capable session runner and the relay socket link —
+// shared by every worker whose runner exposes a delivery gate (the Claude
+// session runner, the Copilot SDK runner). Kept out of the worker entry files
+// (which run main() on import) so the wiring itself is testable.
+
+import { requeueOwedRows } from '../worker-crash-guard.mjs';
+
+/**
+ * The runner reports every flip of its delivery gate; the link turns them
+ * into relay readiness frames. `attach` is late-bound because the runner is
+ * built before the link (the link's probes call into the runner).
+ */
+export function createRunnerLinkBridge() {
+  let link = null;
+  return {
+    attach(nextLink) {
+      link = nextLink || null;
+    },
+    onDeliveryReadinessChange(ready) {
+      if (!link) return;
+      if (ready) void link.notifyReady('steering-resumed');
+      else link.notifyUnready('steering-held');
+    },
+    // Only a relay that advertised the capability in server.hello understands
+    // the penalty-free hand-back; before any hello, assume an older relay.
+    canHandBackHeldDelivery() {
+      return Boolean(link?.serverSupports?.('steering-held'));
+    },
+  };
+}
+
+/**
+ * Worker shutdown (SIGTERM/SIGINT): rows whose prompt the engine already
+ * consumed and whose settle has not landed are failed terminally before the
+ * process goes — left behind, the relay's dead-worker recovery would requeue
+ * them and run them a second time. Bounded like the crash guard: a shutdown
+ * that hangs is worse than none. Other owed rows are left to the relay's
+ * recovery, as before.
+ */
+export async function failSettlingRowsOnShutdown({ api, runner, timeoutMs = 2_000 } = {}) {
+  const settling = (runner?.getActiveQueueMessageIds?.() || []).filter((entry) => entry?.terminalError);
+  await requeueOwedRows({ api, entries: settling, timeoutMs });
+  return settling.length;
+}

package/shared/worker-runtime/worker-websocket-link.mjs

@@ -45,6 +45,25 @@ export function createWorkerWebSocketLink({
   // task). Never blocks delivery; failures are the handler's to log.
   onControl = () => {},
   getSessionReady = () => false,
+  // Mid-turn steering opt-in. A worker whose runner can absorb another
+  // delivered message into the live turn (the Claude session runner; the
+  // Copilot runner's steerIntoActiveTurn is compatible but not yet opted in)
+  // reports so here, and the link then keeps signalling readiness while a
+  // delivery is still in flight and runs the resulting concurrent delivery
+  // instead of coalescing it. Workers that leave this unset keep the strict
+  // single-flight contract.
+  getSteeringReady = null,
+  // Steering-capable workers only: true while the runner is holding deliveries
+  // (a question card or plan approval is open, a compaction or adoption is in
+  // progress, a turn-opening delivery is mid-flight). A held worker never
+  // signals readiness — idle or not — and tells the server so with
+  // worker.unready, because a delivery landing in a hold is handed back.
+  getDeliveryHeld = null,
+  // Called with the hello's reason once per connection, on the relay's first
+  // server.hello: from then on the relay (possibly a restarted one that knows
+  // nothing of this worker) has the socket. Later hellos only ack the
+  // worker.hello every idle readiness refresh re-sends, and are not reported.
+  onServerHello = () => {},
   getSessionId = () => null,
   getPid = () => null,
   minBackoffMs = 1000,
@@ -65,11 +84,20 @@ export function createWorkerWebSocketLink({
   let readyRefreshTimer = null;
   let reconnectAttempt = 0;
   let reconnectDelayMs = 0;
-  let deliveryInFlight = null;
+  // Insertion-ordered: the first entry is the oldest in-flight delivery, which
+  // is what a legacy worker's coalesced delivery waits on.
+  const deliveriesInFlight = new Set();
+  // message id → in-flight delivery. Only a redelivery of the SAME message is
+  // coalesced; a different message is never folded onto another's promise.
+  const deliveriesById = new Map();
   let lastOpenAt = null;
   let lastMessageAt = null;
   let lastHelloSentAt = null;
   let lastReadySentAt = null;
+  let lastUnreadySentAt = null;
+  // What the connected relay advertised in server.hello (see
+  // WORKER_PROTOCOL_CAPABILITIES in the relay's worker websocket service).
+  let serverCapabilities = new Set();
   let lastPingSentAt = null;
   let lastPongAt = null;
   let lastQueueChangedAt = null;
@@ -94,20 +122,61 @@ export function createWorkerWebSocketLink({
     }
   }
 
+  function steeringCapable() {
+    return typeof getSteeringReady === "function";
+  }
+
+  function steeringReady() {
+    if (!steeringCapable()) return false;
+    try {
+      return getSteeringReady() === true;
+    } catch {
+      return false;
+    }
+  }
+
+  function deliveryHeld() {
+    if (typeof getDeliveryHeld !== "function") return false;
+    try {
+      return getDeliveryHeld() === true;
+    } catch {
+      // Fail closed: a held worker that signals ready bounces every delivery.
+      return true;
+    }
+  }
+
   function notifyHello(reason = "worker-hello") {
     if (!getSessionReady()) return false;
+    // The server treats hello as readiness; a held worker still has to bind
+    // its identity, so it says hello without claiming to be ready.
+    const held = deliveryHeld();
     const sent = send({
       type: "worker.hello",
       reason,
       sessionId: normalizeText(getSessionId()),
       pid: normalizePositiveInt(getPid()),
+      ...(held ? { ready: false } : {}),
     });
     if (sent) lastHelloSentAt = getNowMs();
     return sent;
   }
 
+  function notifyUnready(reason = "worker-unready") {
+    if (!getSessionReady()) return false;
+    const sent = send({
+      type: "worker.unready",
+      reason,
+      sessionId: normalizeText(getSessionId()),
+      pid: normalizePositiveInt(getPid()),
+    });
+    if (sent) lastUnreadySentAt = getNowMs();
+    return sent;
+  }
+
   async function notifyReady(reason = "worker-ready") {
-    if (!getSessionReady() || deliveryInFlight) return false;
+    if (!getSessionReady()) return false;
+    if (deliveryHeld()) return false;
+    if (deliveriesInFlight.size && !steeringReady()) return false;
     const sent = send({
       type: "worker.ready",
       reason,
@@ -132,8 +201,21 @@ export function createWorkerWebSocketLink({
 
   async function deliverPending(pending, reason = "queue-deliver") {
     if (!getSessionReady()) return false;
-    if (deliveryInFlight) return deliveryInFlight;
-    deliveryInFlight = Promise.resolve()
+    // A redelivery of a message that is already running (socket reconnect)
+    // coalesces onto it rather than double-running the turn.
+    const messageId = normalizeText(pending?.message?.id);
+    if (messageId && deliveriesById.has(messageId)) return deliveriesById.get(messageId);
+    // A DIFFERENT message while one is in flight: a steering-capable worker's
+    // runner decides — it steers the message into the live turn, or hands it
+    // back to the queue when the turn is holding (the server's readiness can
+    // lag a hold by a round-trip). Coalescing it here used to drop its payload
+    // and strand the row `processing` until heartbeat recovery. Workers
+    // without the steering opt-in keep strict single-flight: their runners
+    // have no hand-back path, and the heartbeat recovers the row.
+    if (deliveriesInFlight.size && !steeringCapable()) {
+      return deliveriesInFlight.values().next().value;
+    }
+    const delivery = Promise.resolve()
       .then(() => onDeliver(pending, reason))
       .catch((error) => {
         dbg("worker ws delivery failed", reason, error?.message || String(error));
@@ -141,9 +223,12 @@ export function createWorkerWebSocketLink({
       })
       .finally(() => {
         lastDeliveryAt = getNowMs();
-        deliveryInFlight = null;
+        deliveriesInFlight.delete(delivery);
+        if (messageId && deliveriesById.get(messageId) === delivery) deliveriesById.delete(messageId);
       });
-    return deliveryInFlight;
+    deliveriesInFlight.add(delivery);
+    if (messageId) deliveriesById.set(messageId, delivery);
+    return delivery;
   }
 
   function clearReconnectTimer() {
@@ -173,7 +258,18 @@ export function createWorkerWebSocketLink({
       return closeStaleSocket(`no-server-message:${reason}`);
     }
     notifyPing(reason);
-    if (deliveryInFlight) return true;
+    if (deliveryHeld()) {
+      // Re-asserted on every refresh: a ready frame that crossed the hold on
+      // the wire would otherwise leave the server's readiness stuck on.
+      notifyUnready(reason);
+      return true;
+    }
+    if (deliveriesInFlight.size) {
+      // notifyReady gates itself: it only goes through mid-delivery when the
+      // worker is steering-ready. Hello stays an idle-only affair.
+      void notifyReady(reason);
+      return true;
+    }
     notifyHello(reason);
     void notifyReady(reason);
     return true;
@@ -215,6 +311,7 @@ export function createWorkerWebSocketLink({
     }
     const url = toWebSocketUrl(serverUrl, token, getSessionId, getPid);
     ws = new WebSocketImpl(url);
+    let serverHelloSeen = false;
     ws.addEventListener("open", () => {
       const openedAt = getNowMs();
       reconnectAttempt = 0;
@@ -240,6 +337,15 @@ export function createWorkerWebSocketLink({
         return;
       }
       if (payload?.type === "server.hello") {
+        serverCapabilities = new Set(Array.isArray(payload.capabilities) ? payload.capabilities.map(String) : []);
+        if (!serverHelloSeen) {
+          serverHelloSeen = true;
+          try {
+            onServerHello(String(payload.reason || "server-hello"));
+          } catch (error) {
+            dbg("worker ws hello handler failed", error?.message || String(error));
+          }
+        }
         void notifyReady(String(payload.reason || "server-hello"));
         return;
       }
@@ -271,6 +377,8 @@ export function createWorkerWebSocketLink({
     });
     ws.addEventListener("close", () => {
       clearReadyRefreshTimer();
+      // The next connection may reach a different (restarted) relay.
+      serverCapabilities = new Set();
       if (stopped) return;
       dbg("worker ws disconnected");
       scheduleReconnect();
@@ -302,13 +410,14 @@ export function createWorkerWebSocketLink({
     const connected = !!ws && ws.readyState === resolveSocketStateValue(ws, "OPEN", 1);
     return {
       connected,
-      delivering: !!deliveryInFlight,
+      delivering: deliveriesInFlight.size > 0,
       reconnectAttempt,
       reconnectDelayMs,
       lastOpenAt,
       lastMessageAt,
       lastHelloSentAt,
       lastReadySentAt,
+      lastUnreadySentAt,
       lastPingSentAt,
       lastPongAt,
       lastQueueChangedAt,
@@ -318,6 +427,8 @@ export function createWorkerWebSocketLink({
 
   return {
     notifyReady,
+    notifyUnready,
+    serverSupports: (capability) => serverCapabilities.has(String(capability || "")),
     start,
     stop,
     status,