@oar-sh/oar 0.9.9 → 0.9.10

package/server/services/explorer-mutations.mjs

@@ -0,0 +1,122 @@
+// Filesystem changes made from the explorer: create a folder, rename an
+// entry, delete an entry, and count what a delete would remove. The callers
+// (the routes) have already resolved the absolute target inside a browsable
+// root; this module only touches the filesystem and turns its errors into
+// `{ error, statusCode }` answers the routes can send as they are.
+//
+// Symbolic links are refused everywhere. The listings skip them, so the user
+// never saw one in the explorer, and a rename or delete through a link would
+// act on something outside the folder they are looking at.
+
+import fsDefault from 'node:fs';
+import pathDefault from 'node:path';
+
+export const DEFAULT_COUNT_CAP = 10000;
+
+function failure(error, fallback, statusCode = 500) {
+  const code = String(error?.code || '');
+  if (code === 'ENOENT' || code === 'ENOTDIR') return { error: 'Not found', statusCode: 404 };
+  if (code === 'EEXIST' || code === 'ENOTEMPTY') return { error: 'That name is already taken', statusCode: 409 };
+  if (code === 'EACCES' || code === 'EPERM' || code === 'EBUSY') return { error: 'Permission denied', statusCode: 403 };
+  return { error: error?.message || fallback, statusCode };
+}
+
+function lstatOrNull(fs, target) {
+  try {
+    return fs.lstatSync(target);
+  } catch (error) {
+    if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return null;
+    throw error;
+  }
+}
+
+/** Create `target`; missing parents are created with it. */
+export function createDirectory(target, { fs = fsDefault } = {}) {
+  try {
+    if (lstatOrNull(fs, target)) return { error: 'That name is already taken', statusCode: 409 };
+    fs.mkdirSync(target, { recursive: true });
+    return { ok: true };
+  } catch (error) {
+    return failure(error, 'Failed to create the folder');
+  }
+}
+
+/**
+ * Rename `target` to `newName` inside its own parent. A name that differs
+ * only in case is allowed: on a case-insensitive filesystem the destination
+ * stat finds the source itself, which is not a collision.
+ */
+export function renameEntry(target, newName, { fs = fsDefault, path = pathDefault } = {}) {
+  const destination = path.join(path.dirname(target), newName);
+  try {
+    const source = lstatOrNull(fs, target);
+    if (!source) return { error: 'Not found', statusCode: 404 };
+    if (source.isSymbolicLink()) return { error: 'Links cannot be changed here', statusCode: 409 };
+    if (destination === target) return { ok: true, path: destination };
+    const existing = lstatOrNull(fs, destination);
+    const sameEntry = existing && existing.ino === source.ino && existing.dev === source.dev && existing.ino !== 0;
+    if (existing && !sameEntry) return { error: 'That name is already taken', statusCode: 409 };
+    fs.renameSync(target, destination);
+    return { ok: true, path: destination };
+  } catch (error) {
+    return failure(error, 'Failed to rename');
+  }
+}
+
+/** Remove `target`; a folder goes with everything inside it. */
+export function removeEntry(target, { fs = fsDefault } = {}) {
+  try {
+    const stat = lstatOrNull(fs, target);
+    if (!stat) return { error: 'Not found', statusCode: 404 };
+    if (stat.isSymbolicLink()) return { error: 'Links cannot be changed here', statusCode: 409 };
+    fs.rmSync(target, { recursive: true, force: false, maxRetries: 2 });
+    return { ok: true };
+  } catch (error) {
+    return failure(error, 'Failed to delete');
+  }
+}
+
+/**
+ * What a delete of `target` would remove: `{ kind, files, dirs, capped }`.
+ * `dirs` excludes the target itself. The walk stops once `files + dirs`
+ * reaches `cap`, so a huge tree answers quickly with `capped: true`. Links
+ * inside count as files and are not followed.
+ */
+export function countEntries(target, { fs = fsDefault, path = pathDefault, cap = DEFAULT_COUNT_CAP } = {}) {
+  let stat = null;
+  try {
+    stat = lstatOrNull(fs, target);
+  } catch (error) {
+    return failure(error, 'Failed to read the entry');
+  }
+  if (!stat) return { error: 'Not found', statusCode: 404 };
+  if (stat.isSymbolicLink()) return { error: 'Links cannot be changed here', statusCode: 409 };
+  if (!stat.isDirectory()) return { ok: true, kind: 'file', files: 0, dirs: 0, capped: false };
+
+  let files = 0;
+  let dirs = 0;
+  let capped = false;
+  const stack = [target];
+  while (stack.length && !capped) {
+    const dir = stack.pop();
+    let entries = [];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue; // an unreadable branch is still removed by rm; it just stays uncounted
+    }
+    for (const entry of entries) {
+      if (entry.isDirectory() && !entry.isSymbolicLink()) {
+        dirs += 1;
+        stack.push(path.join(dir, entry.name));
+      } else {
+        files += 1;
+      }
+      if (files + dirs >= cap) {
+        capped = true;
+        break;
+      }
+    }
+  }
+  return { ok: true, kind: 'dir', files, dirs, capped };
+}

package/server/services/preview-proxy-server.mjs

@@ -399,13 +399,33 @@ export function createPreviewProxyServer({
 
   // WebSocket upgrades (Vite HMR, an app's own socket) ride the same prefix.
   function handleUpgrade(req, socket, head) {
+    // Node removes its own error listener from a socket the moment it emits
+    // 'upgrade', and the upstream socket handed to 'upgrade' comes bare too. A
+    // browser that drops the connection while the dev server is still
+    // answering would otherwise raise an uncaught ECONNRESET and end the
+    // whole relay. Whatever fails on either side takes both down.
+    let upstreamReq = null;
+    let upstreamSocket = null;
+    const teardown = () => {
+      try { socket.destroy(); } catch {}
+      try { upstreamSocket?.destroy(); } catch {}
+      try { upstreamReq?.destroy(); } catch {}
+    };
+    socket.on('error', teardown);
+    // A client that leaves before the dev server answers abandons the
+    // handshake. Once both sides are piped, an orderly close travels through
+    // the pipes, so the pending writes still reach the other side.
+    socket.once('close', teardown);
+    const reply = (text) => {
+      if (socket.destroyed) return;
+      try { socket.write(text); } catch {}
+    };
+
     const resolved = resolveRequest(req);
     // Static previews have nothing to upgrade to; refuse like an unknown token.
     if (resolved.kind !== 'proxy' || resolved.entry?.mode === 'static') {
-      try {
-        socket.write('HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n');
-      } catch {}
-      try { socket.destroy(); } catch {}
+      reply('HTTP/1.1 404 Not Found\r\nConnection: close\r\n\r\n');
+      teardown();
       return;
     }
 
@@ -426,7 +446,7 @@ export function createPreviewProxyServer({
     headers.connection = 'Upgrade';
     headers.upgrade = String(req.headers.upgrade || 'websocket');
 
-    const upstreamReq = httpImpl.request({
+    upstreamReq = httpImpl.request({
       host: entry.targetHost,
       port: entry.targetPort,
       method: req.method,
@@ -436,25 +456,26 @@ export function createPreviewProxyServer({
     });
     guardConnectTimeout(upstreamReq, connectTimeoutMs, () => {});
 
-    upstreamReq.on('upgrade', (upstreamRes, upstreamSocket, upstreamHead) => {
+    upstreamReq.on('upgrade', (upstreamRes, answeredSocket, upstreamHead) => {
+      upstreamSocket = answeredSocket;
+      upstreamSocket.on('error', teardown);
+      // The client left while the dev server was answering.
+      if (socket.destroyed) {
+        teardown();
+        return;
+      }
       const lines = [`HTTP/1.1 ${upstreamRes.statusCode} ${upstreamRes.statusMessage}`];
       for (const [name, value] of Object.entries(upstreamRes.headers)) {
         for (const entryValue of Array.isArray(value) ? value : [value]) {
           lines.push(`${name}: ${entryValue}`);
         }
       }
-      try {
-        socket.write(`${lines.join('\r\n')}\r\n\r\n`);
-      } catch {
-        try { upstreamSocket.destroy(); } catch {}
-        return;
-      }
-      if (upstreamHead?.length) socket.write(upstreamHead);
+      reply(`${lines.join('\r\n')}\r\n\r\n`);
+      if (upstreamHead?.length) reply(upstreamHead);
       if (head?.length) upstreamSocket.write(head);
+      socket.removeListener('close', teardown);
       trackUpgradedSocket(socket);
       trackUpgradedSocket(upstreamSocket);
-      upstreamSocket.on('error', () => { try { socket.destroy(); } catch {} });
-      socket.on('error', () => { try { upstreamSocket.destroy(); } catch {} });
       upstreamSocket.pipe(socket);
       socket.pipe(upstreamSocket);
     });
@@ -462,18 +483,15 @@ export function createPreviewProxyServer({
     // A dev server that answers an upgrade with a plain response (404 on an
     // unknown HMR path) must not leave the client hanging.
     upstreamReq.on('response', (upstreamRes) => {
-      try {
-        socket.write(`HTTP/1.1 ${upstreamRes.statusCode} ${upstreamRes.statusMessage}\r\nConnection: close\r\n\r\n`);
-      } catch {}
+      reply(`HTTP/1.1 ${upstreamRes.statusCode} ${upstreamRes.statusMessage}\r\nConnection: close\r\n\r\n`);
+      upstreamRes.on('error', () => {});
       upstreamRes.resume();
-      try { socket.destroy(); } catch {}
+      teardown();
     });
 
     upstreamReq.on('error', () => {
-      try {
-        socket.write('HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n');
-      } catch {}
-      try { socket.destroy(); } catch {}
+      reply('HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n');
+      teardown();
     });
 
     upstreamReq.end();

package/server/services/push-dispatch-service.mjs

@@ -324,6 +324,22 @@ export function createPushDispatchService({
     }
   }
 
+  // A turn that answers relay events (its message is the relay's event row,
+  // kind 'relay-event') pushes no "reply ready" or "failed" either: the
+  // watcher answers events many times an hour and pushes by itself when one
+  // matters. Question pushes are unchanged.
+  let messageKindStatement = null;
+  function turnAnswersRelayEvents(sourceMessageId) {
+    const id = String(sourceMessageId || '').trim();
+    if (!id) return false;
+    try {
+      messageKindStatement ??= db.prepare(`SELECT kind FROM messages WHERE id = ?`);
+      return String(messageKindStatement.get(id)?.kind || '').trim() === 'relay-event';
+    } catch {
+      return false;
+    }
+  }
+
   async function sendToSubscription(row, event) {
     const preferences = normalizePushPreferences(parseJsonObject(row.preferences_json));
     const payload = renderPushNotification(event, preferences);
@@ -440,6 +456,7 @@ export function createPushDispatchService({
   // the turn.
   function notifyTurnComplete({ conversationId, messageId, sourceMessageId, text } = {}) {
     if (turnStartedByRemoteAgent(sourceMessageId)) return Promise.resolve({ outcome: 'remote-agent' });
+    if (turnAnswersRelayEvents(sourceMessageId)) return Promise.resolve({ outcome: 'relay-event' });
     return dispatch({
       type: 'turnComplete',
       conversationId: conversationId || null,
@@ -450,6 +467,7 @@ export function createPushDispatchService({
 
   function notifyTurnFailed({ conversationId, messageId, sourceMessageId, text } = {}) {
     if (turnStartedByRemoteAgent(sourceMessageId)) return Promise.resolve({ outcome: 'remote-agent' });
+    if (turnAnswersRelayEvents(sourceMessageId)) return Promise.resolve({ outcome: 'relay-event' });
     return dispatch({
       type: 'turnFailed',
       conversationId: conversationId || null,

package/server/services/relay-event-link-service.mjs

@@ -0,0 +1,289 @@
+'use strict';
+
+/**
+ * The watching side of relay event subscriptions on paired relays: while
+ * this relay holds subscriptions on a paired relay (or an end it has not
+ * delivered there yet), it fetches that relay's events every few seconds
+ * over the pairing and hands them to the event service, which delivers them
+ * like the events of this relay's own sessions. Only this relay calls, so a
+ * pairing in one direction is enough.
+ *
+ * A fetch (POST /api/relay-events/remote/pull) acknowledges what the last
+ * one stored, carries the ends and watcher names the other relay must
+ * learn, and comes back with new events and the subscriptions' state; the
+ * event service writes all of it and the cursor in one transaction, so
+ * nothing is lost or delivered twice when either relay restarts.
+ *
+ * A link with no answer for remoteGapMs (failing fetches, or this relay
+ * down) is lost from its last answer on. The first fetch after that, every
+ * page of it, is held and delivered as one replayed row per watcher with
+ * link_lost and link_restored around it. A shorter outage only delays.
+ */
+
+import { REMOTE_RELAY_ERROR_CODES } from '../../shared/remote-relay-contract.mjs';
+import { RELAY_EVENT_LIMITS } from '../../shared/relay-events.mjs';
+
+export const RELAY_EVENT_REMOTE_SUBSCRIBE_PATH = '/api/relay-events/remote/subscribe';
+export const RELAY_EVENT_REMOTE_PULL_PATH = '/api/relay-events/remote/pull';
+
+const LOG_PREFIX = '[relay-events]';
+// Pages of one fetch before the next tick goes on (remotePullBatch each).
+const MAX_PAGES_PER_FETCH = 50;
+
+function toText(value) {
+  return String(value ?? '').trim();
+}
+
+/**
+ * A paired relay that predates cross-relay subscriptions answers their
+ * routes with Express's HTML 404: no JSON, no code. A 404 of these routes
+ * always carries one.
+ */
+export function isMissingRelayEventRoute(error) {
+  return toText(error?.code) === REMOTE_RELAY_ERROR_CODES.notFound && !toText(error?.remoteBody?.code);
+}
+
+export function relayWithoutCrossRelayEventsText(name) {
+  return `Relay "${toText(name) || 'the paired relay'}" runs an OAR version without event subscriptions across relays; it needs an update before its sessions can be watched from here.`;
+}
+
+function describeFailure(relay, error) {
+  if (isMissingRelayEventRoute(error)) return relayWithoutCrossRelayEventsText(relay?.name);
+  return toText(error?.message || error).slice(0, 300) || 'no answer';
+}
+
+function isPullAnswer(response) {
+  return !!response && typeof response === 'object' && Array.isArray(response.events) && Array.isArray(response.subscriptions);
+}
+
+export function createRelayEventLinkService({
+  relayEvents,
+  // `get(id)`: the registry entry (address and token) of a paired relay.
+  registry,
+  // `request(relay, method, path, { body, timeoutMs })`: remote-relay-client.mjs.
+  client,
+  now = () => Date.now(),
+  setTimer = (fn, ms) => {
+    const timer = setTimeout(fn, ms);
+    timer.unref?.();
+    return timer;
+  },
+  clearTimer = (timer) => clearTimeout(timer),
+  logger = console,
+  pollMs = RELAY_EVENT_LIMITS.remotePollMs,
+  failingPollMs = RELAY_EVENT_LIMITS.remotePollFailingMs,
+  gapMs = RELAY_EVENT_LIMITS.remoteGapMs,
+  timeoutMs = 15_000,
+} = {}) {
+  if (!relayEvents) throw new Error('relayEvents is required');
+  // paired relay id → { timer, running, again, replay }
+  const links = new Map();
+  let started = false;
+
+  function warn(line) {
+    try { logger?.warn?.(`${LOG_PREFIX} ${line}`); } catch {}
+  }
+
+  function linkState(relayId) {
+    if (!links.has(relayId)) links.set(relayId, { timer: null, running: null, again: false, replay: null, replayApplied: false });
+    return links.get(relayId);
+  }
+
+  function forget(relayId) {
+    const link = links.get(relayId);
+    if (link?.timer) clearTimer(link.timer);
+    links.delete(relayId);
+  }
+
+  function schedule(relayId, delayMs) {
+    if (!started) return;
+    const link = linkState(relayId);
+    if (link.timer) clearTimer(link.timer);
+    link.timer = setTimer(() => {
+      link.timer = null;
+      void fetchNow(relayId);
+    }, Math.max(0, delayMs));
+  }
+
+  function wanted(relayId) {
+    return relayEvents.remoteRelaysToPoll().includes(relayId);
+  }
+
+  function failingForLong(relayId) {
+    const first = Date.parse(String(relayEvents.getRemoteLink(relayId).first_fail_at || ''));
+    return Number.isFinite(first) && now() - first >= gapMs;
+  }
+
+  /** Whether the link has been without an answer long enough to replay: since when. */
+  function gapSince(relayId) {
+    const health = relayEvents.getRemoteLink(relayId);
+    if (health.lost_at) return health.lost_at;
+    const lastOk = Date.parse(String(health.last_ok_at || ''));
+    return Number.isFinite(lastOk) && now() - lastOk >= gapMs ? health.last_ok_at : null;
+  }
+
+  function startReplay(link, relayId, since) {
+    const perWatcher = new Map();
+    for (const [watcher, subscriptionId] of relayEvents.remoteWatchers(relayId)) {
+      perWatcher.set(watcher, { subscriptionId, kept: 0, dropped: 0, ended: [] });
+    }
+    link.replay = { since, perWatcher };
+    link.replayApplied = false;
+  }
+
+  function addToReplay(link, relayId, applied) {
+    for (const [watcher, info] of applied) {
+      let entry = link.replay.perWatcher.get(watcher);
+      if (!entry) {
+        const subscriptionId = relayEvents.remoteWatchers(relayId).get(watcher);
+        if (!subscriptionId) continue;
+        entry = { subscriptionId, kept: 0, dropped: 0, ended: [] };
+        link.replay.perWatcher.set(watcher, entry);
+      }
+      entry.kept += Number(info.kept) || 0;
+      entry.dropped += Number(info.dropped) || 0;
+      entry.ended.push(...(info.ended || []));
+    }
+  }
+
+  async function fetchPages(relayId, link) {
+    const relay = registry?.get?.(relayId) || null;
+    if (!relay) {
+      // The pairing went without the hook (or before this boot).
+      relayEvents.noteRelayRemoved(relayId);
+      return { ok: false, removed: true, error: 'The pairing with that relay was removed.' };
+    }
+    for (let page = 0; page < MAX_PAGES_PER_FETCH; page += 1) {
+      const body = relayEvents.buildRemotePull(relayId);
+      if (!body.subscriptions.length && !body.ended.length && !link.replay) return { ok: true, idle: true };
+      if (!link.replay) {
+        const since = gapSince(relayId);
+        if (since) startReplay(link, relayId, since);
+      }
+      let response;
+      try {
+        response = await client.request(relay, 'POST', RELAY_EVENT_REMOTE_PULL_PATH, { body, timeoutMs });
+        if (!isPullAnswer(response)) throw new Error(`Relay "${toText(relay.name)}" gave an unexpected answer to the event fetch`);
+      } catch (error) {
+        const text = describeFailure(relay, error);
+        relayEvents.noteRemoteLinkFailure(relayId, text);
+        // Nothing of a replay was written yet: the next fetch starts it again.
+        if (link.replay && !link.replayApplied) link.replay = null;
+        return { ok: false, error: text, unsupported: isMissingRelayEventRoute(error) };
+      }
+      const applied = relayEvents.applyRemotePull(relayId, { relayName: relay.name, response, replay: !!link.replay });
+      if (link.replay) {
+        link.replayApplied = true;
+        addToReplay(link, relayId, applied);
+      }
+      if (response.more !== true) break;
+    }
+    if (link.replay) {
+      const { since, perWatcher } = link.replay;
+      link.replay = null;
+      link.replayApplied = false;
+      relayEvents.noteRemoteLinkRestored(relayId, { relayName: relay.name, since, perWatcher });
+    }
+    return { ok: true };
+  }
+
+  /**
+   * Fetches from a paired relay now (a fetch already running is followed by
+   * one more). Resolves `{ ok, error?, idle?, unsupported? }`; never throws.
+   */
+  function fetchNow(relayId) {
+    const id = toText(relayId);
+    const link = linkState(id);
+    if (link.timer) {
+      clearTimer(link.timer);
+      link.timer = null;
+    }
+    if (link.running) {
+      link.again = true;
+      return link.running;
+    }
+    link.running = (async () => {
+      let result;
+      try {
+        result = await fetchPages(id, link);
+      } catch (error) {
+        warn(`fetch from paired relay ${id.slice(0, 12)} failed: ${toText(error?.message || error).slice(0, 200)}`);
+        result = { ok: false, error: toText(error?.message || error) };
+      } finally {
+        link.running = null;
+      }
+      if (result.removed) {
+        forget(id);
+      } else if (link.again) {
+        link.again = false;
+        schedule(id, 0);
+      } else if (wanted(id)) {
+        schedule(id, result.ok ? pollMs : (failingForLong(id) ? failingPollMs : pollMs));
+      } else if (!link.replay) {
+        forget(id);
+      }
+      return result;
+    })();
+    return link.running;
+  }
+
+  /** A fetch, waited for at most `waitMs` (list_subscriptions wants fresh state). */
+  async function refresh(relayId, { waitMs = 5_000 } = {}) {
+    let timer = null;
+    const timeout = new Promise((resolve) => {
+      timer = setTimer(() => resolve({ ok: false, error: 'timeout' }), waitMs);
+    });
+    try {
+      return await Promise.race([fetchNow(relayId), timeout]);
+    } finally {
+      clearTimer(timer);
+    }
+  }
+
+  /**
+   * The pairing with a relay is being removed (`entry` still has its address
+   * and token): one last fetch tells it every subscription ended, then the
+   * mirrors end here. A relay that does not answer ends them itself after
+   * remoteKeepDays without a fetch.
+   */
+  async function noteRelayRemoved(entry) {
+    const relayId = toText(entry?.id);
+    if (!relayId) return;
+    forget(relayId);
+    const rows = relayEvents.mirrorsOf(relayId);
+    if (rows.length && entry?.url) {
+      const body = {
+        after: Number(relayEvents.getRemoteLink(relayId).cursor) || 0,
+        subscriptions: [],
+        ended: rows.map((row) => row.remote_subscription_id),
+      };
+      try {
+        await client.request(entry, 'POST', RELAY_EVENT_REMOTE_PULL_PATH, { body, timeoutMs: Math.min(timeoutMs, 10_000) });
+      } catch (error) {
+        warn(`the paired relay being removed did not take the ends: ${describeFailure(entry, error)}`);
+      }
+    }
+    relayEvents.noteRelayRemoved(relayId);
+  }
+
+  /** Boot: fetch from every paired relay this relay holds subscriptions on. */
+  function start() {
+    if (started) return;
+    started = true;
+    let delay = 0;
+    for (const relayId of relayEvents.remoteRelaysToPoll()) {
+      schedule(relayId, delay);
+      delay += 250;
+    }
+  }
+
+  function stop() {
+    started = false;
+    for (const link of links.values()) {
+      if (link.timer) clearTimer(link.timer);
+      link.timer = null;
+    }
+  }
+
+  return { start, stop, fetchNow, refresh, noteRelayRemoved, isPolling: (relayId) => links.has(toText(relayId)) };
+}