@oar-sh/oar 0.9.11 → 0.9.12

package/server/services/remote-relay-dispatcher.mjs

@@ -27,6 +27,7 @@ import {
 import { schemaFields } from '../../shared/question-schema.mjs';
 import { ANSWERED_ELSEWHERE_KIND } from '../../shared/steer-settle-markers.mjs';
 import { RELAY_EVENT_DEFAULT_TYPES } from '../../shared/relay-events.mjs';
+import { cleanTaskOutput, parseRunnerSummary, stallMinutes, stripSgr } from '../../shared/task-output-text.mjs';
 import {
   RELAY_EVENT_REMOTE_PULL_PATH,
   RELAY_EVENT_REMOTE_SUBSCRIBE_PATH,
@@ -1281,7 +1282,37 @@ export function createRemoteRelayDispatcher({
    * `lastActivityAt` of a session, from the relay's `sessionState` /
    * `sessionTasks`. A relay that predates them gives none.
    */
-  function sessionActivityFields(row) {
+  /**
+   * The console facts read_session adds to a task (`row` is the conversation
+   * payload): `taskId` for read_task_output, `output` whether there is any to
+   * read, and `health` (minutes since the last output, the quiet threshold,
+   * test-run counts) for a task with an output file.
+   */
+  function taskConsoleFields(row, id) {
+    const taskId = toText(id);
+    if (!taskId || !isPlainObject(row)) return {};
+    const live = (Array.isArray(row.backgroundTasks) ? row.backgroundTasks : []).find((task) => toText(task?.taskId) === taskId);
+    const ended = live ? null : (Array.isArray(row.endedBackgroundTasks) ? row.endedBackgroundTasks : []).find((task) => toText(task?.taskId) === taskId);
+    const task = live || ended;
+    if (!task) return {};
+    const kind = toText(task.outputKind) || (toText(task.outputPath) ? 'file' : 'none');
+    const health = isPlainObject(task.health) ? task.health : null;
+    const quietAfter = Number(row.taskStallMinutes) || null;
+    const minutes = live && health ? stallMinutes(health.lastOutputAt, now()) : null;
+    return {
+      taskId,
+      output: kind !== 'none',
+      ...(health ? {
+        health: {
+          ...(minutes !== null ? { minutesSinceOutput: minutes } : {}),
+          ...(quietAfter && minutes !== null ? { quiet: minutes >= quietAfter, quietAfterMinutes: quietAfter } : {}),
+          ...(isPlainObject(health.counts) ? { counts: { runner: toText(health.runner) || null, ...health.counts } } : {}),
+        },
+      } : {}),
+    };
+  }
+
+  function sessionActivityFields(row, { console: withConsole = false } = {}) {
     const state = toText(row?.sessionState);
     if (!SESSION_STATES.has(state)) return {};
     const tasks = (Array.isArray(row?.sessionTasks) ? row.sessionTasks : [])
@@ -1298,6 +1329,7 @@ export function createRemoteRelayDispatcher({
         ...(toText(task.summary) && toText(task.status) !== 'running' ? { summary: clipLine(task.summary, 200) } : {}),
         ...(task.endedAt ? { endedAt: task.endedAt } : {}),
         ...(toText(task.id) ? { id: toText(task.id) } : {}),
+        ...(withConsole ? taskConsoleFields(row, task.id) : {}),
       }));
     return {
       state,
@@ -1487,7 +1519,7 @@ export function createRemoteRelayDispatcher({
         cwd: toText(conversation?.currentWorkspaceRootPath) || null,
         archived: conversation?.archived === true,
         active: conversation?.activeTurn === true || isBusyState(conversation?.sessionState),
-        ...sessionActivityFields(conversation),
+        ...sessionActivityFields(conversation, { console: true }),
         ...(via ? { via } : {}),
         ...(group ? { group } : {}),
         openUrl: remoteConversationUrl(ctx.relay.url, conversationId) || null,
@@ -1500,6 +1532,101 @@ export function createRemoteRelayDispatcher({
     };
   }
 
+  /**
+   * read_task_output: the end of a background shell or Monitor task's
+   * output (GET …/background-task/:taskId/output on the target relay, which
+   * scrubs it), as plain lines without colours, at most `lines` lines and
+   * 32 KB; minutes since its last output; test-run counts when it is one.
+   */
+  async function readTaskOutput(ctx, args) {
+    let answer;
+    try {
+      answer = await call(ctx, 'GET', sessionPath(args.session, `/background-task/${encodeURIComponent(args.task_id)}/output`), {
+        query: { lines: args.lines },
+      });
+    } catch (error) {
+      const status = Number(error?.status) || null;
+      if (isRemoteRelayError(error) && (status === 404 || toText(error.code) === REMOTE_RELAY_ERROR_CODES.notFound)) {
+        const body = isPlainObject(error.remoteBody) ? error.remoteBody : {};
+        if (toText(body.code) === 'task-not-found') {
+          throw new DispatchFailure(404, REMOTE_RELAY_ERROR_CODES.notFound,
+            `Task ${args.task_id} is not running in session ${args.session} on relay "${ctx.relay.name}" and did not end there within the last hour; read_session lists the session's tasks.`);
+        }
+        if (!/conversation not found/i.test(toText(body.error))) {
+          throw new DispatchFailure(400, REMOTE_RELAY_ERROR_CODES.unsupported,
+            `Relay "${ctx.relay.name}" cannot show background task output (it runs an older OAR version).`);
+        }
+      }
+      throw error;
+    }
+    const model = Array.isArray(answer?.lines) ? answer.lines : [];
+    const task = isPlainObject(answer?.task) ? answer.task : {};
+    const ended = answer?.ended === true;
+    // Plain text, a repeat folded as ×N; cleaned again, whatever the other
+    // side did.
+    const rendered = model.map((line) => {
+      const text = stripSgr(cleanTaskOutput(String(line?.text ?? '')));
+      const count = Number(line?.count) > 1 ? Number(line.count) : 1;
+      return count > 1 ? `${text}  [×${count}]` : text;
+    });
+    let tail = rendered.slice(-args.lines);
+    let cut = tail.length < rendered.length;
+    let bytes = 0;
+    let keepFrom = tail.length;
+    for (let index = tail.length - 1; index >= 0; index -= 1) {
+      bytes += Buffer.byteLength(tail[index], 'utf8') + 1;
+      if (bytes > REMOTE_RELAY_LIMITS.taskOutputBytesMax) break;
+      keepFrom = index;
+    }
+    if (keepFrom > 0) {
+      tail = tail.slice(keepFrom);
+      cut = true;
+    }
+    const health = isPlainObject(answer?.health) ? answer.health : null;
+    const lastOutputAt = Number(health?.lastOutputAt) || Number(answer?.mtime) || null;
+    const minutes = !ended && lastOutputAt ? stallMinutes(lastOutputAt, now()) : null;
+    const quietAfter = Number(answer?.taskStallMinutes) || null;
+    let counts = null;
+    if (isPlainObject(health?.counts)) {
+      counts = { runner: toText(health.runner) || null, ...health.counts, final: health.final === true };
+    } else {
+      const parsed = parseRunnerSummary(model);
+      if (parsed) counts = { runner: parsed.runner, passed: parsed.passed, failed: parsed.failed, skipped: parsed.skipped, final: parsed.final };
+    }
+    const kind = toText(answer?.kind) || 'file';
+    let note = null;
+    if (kind === 'none') note = toText(answer?.message) || 'Output is not available for this provider.';
+    else if (answer?.pending) note = 'The session\'s worker was asked for the output; call again in a few seconds.';
+    else if (answer?.unavailable) note = 'The output file is not there any more (the provider cleaned it up).';
+    else if (kind === 'snapshot') note = 'Only the end of this task\'s output is available (its provider keeps no file).';
+    const status = ended ? (toText(answer?.status) || 'unknown') : 'running';
+    const exitCode = Number.isInteger(answer?.exitCode) ? answer.exitCode : null;
+    const parts = [`${tail.length} line${tail.length === 1 ? '' : 's'}`, ended ? `ended (${status}${exitCode !== null ? `, exit ${exitCode}` : ''})` : 'running'];
+    if (minutes !== null) parts.push(`last output ${minutes} min ago`);
+    if (counts) parts.push(`${counts.passed} passed, ${counts.failed} failed`);
+    return {
+      summary: `read_task_output → ${ctx.relay.name} session ${shortId(args.session)} task ${shortId(args.task_id)}: ${parts.join(', ')}`,
+      task: {
+        id: args.task_id,
+        description: toText(task.description),
+        type: toText(task.taskType) || null,
+        startedAt: task.startedAt || null,
+        status,
+        exitCode,
+        ...(task.endedAt ? { endedAt: task.endedAt } : {}),
+      },
+      stall: minutes === null ? null : {
+        minutes,
+        since: new Date(lastOutputAt).toISOString(),
+        ...(quietAfter ? { quiet: minutes >= quietAfter, quietAfterMinutes: quietAfter } : {}),
+      },
+      counts,
+      tail,
+      truncated: cut || answer?.hasMoreBefore === true,
+      ...(note ? { note } : {}),
+    };
+  }
+
   function outcomeSummary(action, ctx, outcome) {
     const questions = Array.isArray(outcome.pendingQuestions) ? outcome.pendingQuestions.length : 0;
     return `${action} → ${ctx.relay.name} session ${shortId(outcome.session)}: ${outcome.status}`
@@ -2272,6 +2399,7 @@ export function createRemoteRelayDispatcher({
     usage: usageAction,
     list_sessions: listSessions,
     read_session: readSession,
+    read_task_output: readTaskOutput,
     wait: waitAction,
     send,
     create_session: createSession,

package/server/services/session-activity-tracker.mjs

@@ -249,7 +249,9 @@ export function createSessionActivityTracker({
     };
   }
 
-  function finishTask(conversationId, task, { status, summary = null, exitCode = null, reported = true, absent = false } = {}) {
+  // `replaceSummary`: the end as the worker reported it, whose summary (or
+  // none) replaces the live row's, which was a progress line.
+  function finishTask(conversationId, task, { status, summary = null, exitCode = null, reported = true, absent = false, replaceSummary = false } = {}) {
     if (task.graceTimer) {
       clearTimer(task.graceTimer);
       task.graceTimer = null;
@@ -260,6 +262,7 @@ export function createSessionActivityTracker({
     task.status = normalizeTaskStatus(status || 'completed');
     task.statusReported = reported;
     if (summary) task.summary = toText(summary).slice(0, 500);
+    else if (replaceSummary) task.summary = null;
     const code = Number.isInteger(exitCode) ? exitCode : exitCodeFromSummary(task.summary);
     task.exitCode = code;
     task.endedAtMs = now();
@@ -406,6 +409,7 @@ export function createSessionActivityTracker({
       status: normalizeTaskStatus(status || 'completed'),
       summary: toText(summary) || null,
       exitCode: Number.isInteger(exitCode) ? exitCode : null,
+      replaceSummary: true,
     };
     entry.lastActivityAt = now();
     if (!task && key && endedByAbsence.delete(`${id}:${key}`)) {
@@ -416,7 +420,7 @@ export function createSessionActivityTracker({
       if (ended) {
         ended.status = settled.status;
         ended.statusReported = true;
-        if (settled.summary) ended.summary = settled.summary.slice(0, 500);
+        ended.summary = settled.summary ? settled.summary.slice(0, 500) : null;
         ended.exitCode = settled.exitCode ?? exitCodeFromSummary(ended.summary);
       }
       return;

package/server/services/task-output-rooms.mjs

@@ -0,0 +1,259 @@
+'use strict';
+
+// Who receives a background task's output live: the pages whose console
+// shows that task. The console joins with `task_output_watch { conversationId,
+// taskId, offset }` (on open and after every reconnect) and leaves with
+// `task_output_unwatch`; a socket is in at most one task room. While a room
+// has members the relay stats the task's file every second and sends what
+// was appended as `task_output_chunk { conversationId, taskId, offset,
+// endOffset, text, size, mtime, ended }` (at most 64 KB per emit, cleaned like
+// every other read). A page whose next expected offset is not `offset`
+// fetches the tail again; a file that shrank is sent as an empty chunk with
+// `reset: true`. Nobody watching costs nothing: the room's poller stops when
+// it empties.
+
+import { TASK_OUTPUT_LIMITS } from '../../shared/task-output-text.mjs';
+import { inspectOutputFile, readRange, taskOutputSource } from './task-output-service.mjs';
+
+const ROOM_PREFIX = 'task-output:';
+// Chunks one tick may send for a fast writer (4 × 64 KB a second).
+const MAX_CHUNKS_PER_TICK = 4;
+// A watcher whose offset is this far behind the room is caught up by itself;
+// further back, it fetches the tail.
+const CATCH_UP_BYTES = 256 * 1024;
+
+export function taskOutputRoom(conversationId, taskId) {
+  return `${ROOM_PREFIX}${String(conversationId || '')}:${String(taskId || '')}`;
+}
+
+function safeAck(ack, payload) {
+  if (typeof ack !== 'function') return;
+  try { ack(payload); } catch {}
+}
+
+function idOf(value) {
+  const text = String(value ?? '').trim();
+  return text && text.length <= 200 && !/[\s\0]/.test(text) ? text : '';
+}
+
+/**
+ * `resolveTask(conversationId, taskId)` returns `{ task, ended, key }` for a
+ * task in the conversation's live or ended set, or null: the same access
+ * check as the output route. Sockets reaching `registerSocket` are already
+ * authenticated.
+ */
+export function createTaskOutputRooms({
+  io,
+  resolveTask,
+  intervalMs = 1000,
+  chunkBytes = TASK_OUTPUT_LIMITS.chunkBytes,
+  roomSize = (room) => io?.sockets?.adapter?.rooms?.get?.(room)?.size || 0,
+  readChunk = readRange,
+  inspect = inspectOutputFile,
+  setIntervalImpl = setInterval,
+  clearIntervalImpl = clearInterval,
+  logger = console,
+} = {}) {
+  if (typeof resolveTask !== 'function') throw new Error('createTaskOutputRooms requires resolveTask');
+  // room → { conversationId, taskId, path, offset, timer, busy }
+  const watched = new Map();
+
+  function stopRoom(room) {
+    const entry = watched.get(room);
+    if (!entry) return;
+    if (entry.timer) clearIntervalImpl(entry.timer);
+    watched.delete(room);
+  }
+
+  function leave(socket) {
+    const current = socket.data?.taskOutputRoom;
+    if (current) socket.leave(current);
+    if (socket.data) socket.data.taskOutputRoom = null;
+    if (current && roomSize(current) === 0) stopRoom(current);
+  }
+
+  function lookup(conversationId, taskId) {
+    try {
+      return resolveTask(conversationId, taskId);
+    } catch {
+      return undefined;
+    }
+  }
+
+  // Where a room may start in a file of `size` bytes: after the last line
+  // break of its last 64 KB (never inside a character, an escape or a line
+  // whose credentials would be scrubbed in two halves), 0 when there is none
+  // in a shorter file.
+  async function safeOffset(path, size) {
+    if (!(size > 0)) return 0;
+    const start = Math.max(0, size - chunkBytes);
+    try {
+      const probe = await readChunk(path, { start, maxBytes: size - start, holdOpenLine: true, longLineBytes: chunkBytes });
+      return probe?.ok && Number.isInteger(probe.endOffset) ? probe.endOffset : size;
+    } catch {
+      return size;
+    }
+  }
+
+  // `flush`: send the open last line too (the writer is idle, the task ended,
+  // or the bytes were already sent to the room).
+  async function sendRange(target, entry, start, end, ended, { flush = true } = {}) {
+    let offset = start;
+    for (let sent = 0; sent < MAX_CHUNKS_PER_TICK && offset < end; sent += 1) {
+      const chunk = await readChunk(entry.path, {
+        start: offset,
+        maxBytes: Math.min(chunkBytes, end - offset),
+        holdOpenLine: !flush,
+        longLineBytes: chunkBytes,
+      });
+      if (!chunk.ok || chunk.endOffset <= offset) break;
+      target.emit('task_output_chunk', {
+        conversationId: entry.conversationId,
+        taskId: entry.taskId,
+        offset: chunk.offset,
+        endOffset: chunk.endOffset,
+        text: chunk.text,
+        size: chunk.size,
+        mtime: chunk.mtime,
+        ended,
+      });
+      offset = chunk.endOffset;
+    }
+    return offset;
+  }
+
+  async function poll(room) {
+    const entry = watched.get(room);
+    if (!entry || entry.busy) return;
+    if (roomSize(room) === 0) {
+      stopRoom(room);
+      return;
+    }
+    entry.busy = true;
+    try {
+      const found = lookup(entry.conversationId, entry.taskId);
+      // The task left both sets (an hour after its end): nothing more comes.
+      if (found === null) {
+        stopRoom(room);
+        return;
+      }
+      const ended = !!found?.ended;
+      const info = await inspect(entry.path);
+      // Not there (yet, or any more): keep waiting.
+      if (!info.ok) return;
+      const target = io.to(room);
+      // Unchanged since the last tick: the writer is idle, an open line goes out.
+      const idle = info.size === entry.lastSize;
+      entry.lastSize = info.size;
+      if (info.size < entry.offset) {
+        entry.offset = await safeOffset(entry.path, info.size);
+        target.emit('task_output_chunk', {
+          conversationId: entry.conversationId,
+          taskId: entry.taskId,
+          offset: entry.offset,
+          endOffset: entry.offset,
+          text: '',
+          size: info.size,
+          mtime: info.mtimeMs,
+          ended,
+          reset: true,
+        });
+        return;
+      }
+      if (info.size > entry.offset) {
+        entry.offset = await sendRange(target, entry, entry.offset, info.size, ended, { flush: idle || ended });
+      }
+    } catch (error) {
+      logger?.warn?.(`[task-output] live read failed for task ${String(entry.taskId).slice(0, 12)}: ${error?.message || error}`);
+    } finally {
+      const still = watched.get(room);
+      if (still) still.busy = false;
+    }
+  }
+
+  async function watch(socket, payload) {
+    const conversationId = idOf(payload?.conversationId);
+    const taskId = idOf(payload?.taskId);
+    if (!conversationId || !taskId) return { ok: false, code: 'missing-id' };
+    const found = lookup(conversationId, taskId);
+    if (found === undefined) return { ok: false, code: 'unavailable' };
+    if (!found) return { ok: false, code: 'not-found' };
+    const { kind, path } = taskOutputSource(found.task);
+    // A snapshot is fetched again by the console, `none` has nothing.
+    if (!path) return { ok: false, code: kind === 'snapshot' ? 'snapshot' : 'no-output' };
+    let info = await inspect(path);
+    // A file not there yet (a spool is named before its first byte) is
+    // waited for: the room starts at offset 0.
+    if (!info.ok && info.code === 'missing') info = { ok: true, size: 0, mtimeMs: null };
+    if (!info.ok) return { ok: false, code: info.code || 'unreadable' };
+
+    const room = taskOutputRoom(conversationId, taskId);
+    if (socket.data?.taskOutputRoom !== room) {
+      leave(socket);
+      socket.join(room);
+      if (socket.data) socket.data.taskOutputRoom = room;
+    } else if (!socket.rooms?.has?.(room)) {
+      // A recovered socket keeps its data; its rooms may or may not be back.
+      socket.join(room);
+    }
+
+    const requested = Number(payload?.offset);
+    const wanted = Number.isInteger(requested) && requested >= 0 && requested <= info.size ? requested : null;
+    // Where the page's tail ended, so nothing written between its fetch and
+    // this watch is lost; a page far behind (or one that names no offset)
+    // starts at a safe place near the end and fetches the tail up to it.
+    const startOffset = watched.has(room) ? 0
+      : (wanted !== null && info.size - wanted <= CATCH_UP_BYTES ? wanted : await safeOffset(path, info.size));
+    let entry = watched.get(room);
+    if (!entry) {
+      entry = {
+        conversationId,
+        taskId,
+        path,
+        offset: startOffset,
+        lastSize: info.size,
+        timer: null,
+        busy: false,
+      };
+      watched.set(room, entry);
+      entry.timer = setIntervalImpl(() => { poll(room).catch(() => {}); }, intervalMs);
+      entry.timer?.unref?.();
+    } else if (wanted !== null && wanted < entry.offset && entry.offset - wanted <= CATCH_UP_BYTES) {
+      // A second page joins behind the room: it gets the gap on its own.
+      await sendRange(socket, entry, wanted, entry.offset, !!found.ended);
+    }
+    return { ok: true, taskId, offset: entry.offset, size: info.size };
+  }
+
+  function registerSocket(socket) {
+    if (!socket || typeof socket.on !== 'function') return;
+    socket.on('task_output_watch', (payload, ack) => {
+      watch(socket, payload)
+        .then((result) => {
+          // A refused watch leaves no stale room behind: the page shows another task now.
+          if (!result.ok) leave(socket);
+          safeAck(ack, result);
+        })
+        .catch(() => {
+          leave(socket);
+          safeAck(ack, { ok: false, code: 'unavailable' });
+        });
+    });
+    socket.on('task_output_unwatch', (_payload, ack) => {
+      leave(socket);
+      safeAck(ack, { ok: true });
+    });
+    // socket.io takes a closed socket out of its rooms; the poller stops at
+    // its next tick when that emptied the room.
+  }
+
+  return {
+    registerSocket,
+    /** Rooms with a poller, for tests and diagnostics. */
+    watchedRooms: () => [...watched.keys()],
+    poll,
+    stop() {
+      for (const room of [...watched.keys()]) stopRoom(room);
+    },
+  };
+}