agent-relay-server 0.122.0 → 0.123.0

package/runner/src/liveness.ts

@@ -0,0 +1,50 @@
+import type { LivenessSignal } from "agent-relay-sdk";
+
+// The raw runner-tracked signals an adapter folds into a normalized LivenessSignal.
+// This is the INPUT to the contract computation (#746): the runner owns this state
+// (transport frames, the last provider event, the pending timeline transition, the
+// active-work set), passes it to the adapter's `liveness()` verdict, and ships the
+// resulting LivenessSignal on the agent card. Provider-raw interpretation stops here —
+// nothing above the runner ever reads these fields again.
+export interface LivenessInputs {
+  /** Runner→server transport frame/ack timestamps (sdk BusClient.transportState). */
+  transport?: { lastServerFrameAt?: number; lastHeartbeatAckAt?: number };
+  /** Epoch-ms of the most recent provider status/token event the runner observed. */
+  lastProviderEventAt?: number;
+  /** The most recent lifecycle/timeline transition the runner published (status + ts). */
+  timelineEvent?: { status?: string; timestamp?: number };
+  /** Active work records. The adapter classifies its own kinds; `provider-turn` is the
+   *  bare turn, anything else (subagent, background-script, tool, subprocess) is live work. */
+  activeWork: Array<{ kind?: string; startedAt?: number }>;
+  /** Fallback progress floor when no other progress evidence exists yet (e.g. work start). */
+  fallbackProgressAt?: number;
+}
+
+function num(value: unknown): number {
+  return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+/**
+ * The shared liveness fold (#725 / #746). Both reference adapters (Claude #713, Codex #724)
+ * route through this so the verdict is computed in ONE place; each adapter's `liveness()`
+ * decides WHICH raw signals it feeds in (Claude surfaces a `compacting` timeline transition
+ * as progress; Codex surfaces live subprocess work-kinds). The fold itself is provider-
+ * agnostic, which is the point of the contract: a new provider only has to feed its signals.
+ */
+export function computeLivenessSignal(inputs: LivenessInputs): LivenessSignal {
+  const transportLiveAt = Math.max(
+    num(inputs.transport?.lastServerFrameAt),
+    num(inputs.transport?.lastHeartbeatAckAt),
+  );
+  // Progress = provider tokens OR a lifecycle/timeline transition (e.g. native compaction,
+  // #713) — whichever is latest, with the caller-supplied floor as the baseline.
+  const lastProgressAt = Math.max(
+    num(inputs.lastProviderEventAt),
+    num(inputs.timelineEvent?.timestamp),
+    num(inputs.fallbackProgressAt),
+  );
+  // Any work-kind that is NOT the bare provider turn is live tool/subprocess work (#724):
+  // a wedged provider turn with a live `bun test` child must read as progressing.
+  const hasLiveWork = inputs.activeWork.some((work) => Boolean(work.kind && work.kind !== "provider-turn"));
+  return { transportLiveAt, lastProgressAt, hasLiveWork };
+}

package/runner/src/logger.ts

@@ -0,0 +1,99 @@
+import { appendFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { sanitizeFsName } from "agent-relay-sdk/fs-name";
+import { logLevelFromEnv } from "./config";
+
+// Phase 1 observability (#198): one leveled, runtime-togglable logger for the
+// Runner and the provider adapters below it. Replaces the ad-hoc scatter of
+// `console.error`, `logRunnerDiagnostic` (-> runner-<agent>.log) and
+// `sessionLog`/`sessionDebug` (-> session-mirror-<agent>.log) with a single
+// switch and a single greppable, ANSI-free sink.
+//
+// Sink: the per-agent `session-mirror-<agent>.log` — the file the orchestrator
+// already surfaces to the dashboard log-viewer (captureSessionMirror). One place
+// to look when anything in the Runner misbehaves.
+//
+// Level is read once from AGENT_RELAY_LOG_LEVEL (default "info") and can be
+// flipped at runtime via the control port (no restart) — so a phase refactor can
+// be watched at debug without bouncing the agent.
+
+type LogLevel = "debug" | "info" | "warn" | "error" | "fatal";
+
+const ORDER: Record<LogLevel, number> = { debug: 10, info: 20, warn: 30, error: 40, fatal: 50 };
+export const LOG_LEVELS = Object.keys(ORDER) as LogLevel[];
+
+export function parseLogLevel(value: string | undefined | null): LogLevel | undefined {
+  if (!value) return undefined;
+  const v = value.trim().toLowerCase();
+  return (LOG_LEVELS as string[]).includes(v) ? (v as LogLevel) : undefined;
+}
+
+// Shared with the orchestrator (session-mirror reader) + runner outbox via the
+// SDK helper, so all sites resolve the identical filename for a given agent id.
+function safeLogName(value: string): string {
+  return sanitizeFsName(value, { replacement: "_", maxLen: 180 });
+}
+
+interface LoggerConfig {
+  agentId?: string;
+  level?: LogLevel;
+  headless?: boolean;
+  logDir?: string;
+}
+
+export class Logger {
+  private level: LogLevel;
+  private agentId: string;
+  private headless: boolean;
+  private logDir: string;
+
+  constructor(config: LoggerConfig = {}) {
+    this.level = config.level ?? parseLogLevel(logLevelFromEnv()) ?? "info";
+    this.agentId = config.agentId ?? "runner";
+    this.headless = config.headless ?? false;
+    this.logDir = config.logDir ?? join(process.env.HOME || ".", ".agent-relay", "logs");
+  }
+
+  // Bind the logger to a concrete agent once the runner knows its id. Preserves a
+  // level already set via env/runtime unless an explicit level is passed.
+  configure(config: LoggerConfig): void {
+    if (config.agentId !== undefined) this.agentId = config.agentId;
+    if (config.headless !== undefined) this.headless = config.headless;
+    if (config.logDir !== undefined) this.logDir = config.logDir;
+    if (config.level !== undefined) this.level = config.level;
+  }
+
+  setLevel(level: LogLevel): void { this.level = level; }
+  getLevel(): LogLevel { return this.level; }
+  isEnabled(level: LogLevel): boolean { return ORDER[level] >= ORDER[this.level]; }
+
+  debug(component: string, message: string): void { this.log("debug", component, message); }
+  info(component: string, message: string): void { this.log("info", component, message); }
+  warn(component: string, message: string): void { this.log("warn", component, message); }
+  error(component: string, message: string): void { this.log("error", component, message); }
+  fatal(component: string, message: string): void { this.log("fatal", component, message); }
+
+  log(level: LogLevel, component: string, message: string): void {
+    if (!this.isEnabled(level)) return;
+    const line = `[${new Date().toISOString()}] ${level.toUpperCase().padEnd(5)} [${component}] ${oneLine(message)}\n`;
+    try {
+      mkdirSync(this.logDir, { recursive: true });
+      appendFileSync(join(this.logDir, `session-mirror-${safeLogName(this.agentId)}.log`), line);
+    } catch {
+      // Best-effort. If the per-agent file can't be written, surface error/fatal to
+      // stderr so it is not lost entirely (headless: lands in the orchestrator log).
+      if (ORDER[level] >= ORDER.error) { try { console.error(line.trimEnd()); } catch { /* give up */ } }
+    }
+  }
+}
+
+// Newlines would split one record across several log lines and break greppability;
+// collapse them so a multi-line message stays one line.
+function oneLine(message: string): string {
+  return message.replace(/\r?\n/g, " ⏎ ");
+}
+
+// Process-global logger. A runner process serves exactly one agent, so a singleton
+// is the right scope; the runner calls configure() once it knows its id, and
+// adapters import this instance directly (no constructor threading).
+export const logger = new Logger();

package/runner/src/mcp-outbox.ts

@@ -0,0 +1,120 @@
+import type { SendMessageInput } from "agent-relay-sdk";
+import type { RelayHttpClient } from "agent-relay-sdk";
+import { errMessage } from "agent-relay-sdk";
+import { relayMcpEndpoint } from "./relay-mcp";
+import type { OutboxRecord } from "./outbox";
+import { deliverContinuationArchiveRecord } from "./continuation-archive";
+import { logger } from "./logger";
+import { isHttpAuthError, isHttpStatusError } from "./runner-helpers";
+
+interface RunnerOutboxDelivery {
+  record: OutboxRecord;
+  http: RelayHttpClient;
+  relayUrl: string;
+  token?: string;
+  updatePayload(seq: number, payload: unknown): void;
+  sessionLog(message: string): void;
+  recoverRuntimeTokenAfterAuthFailure(source: string): void;
+}
+
+export async function deliverRunnerOutboxEvent(input: RunnerOutboxDelivery): Promise<void> {
+  const { record, http } = input;
+  try {
+    if (record.kind === "session-message") {
+      await http.sendMessage({
+        ...(record.payload as SendMessageInput),
+        occurredAt: record.occurredAt,
+        idempotencyKey: record.idempotencyKey,
+      });
+      return;
+    }
+    if (record.kind === "session-message-batch") {
+      const messages = Array.isArray((record.payload as { messages?: unknown }).messages)
+        ? (record.payload as { messages: SendMessageInput[] }).messages
+        : [];
+      const batch = messages.map((message, index) => ({
+        ...message,
+        occurredAt: typeof message.occurredAt === "number" ? message.occurredAt : record.occurredAt,
+        idempotencyKey: message.idempotencyKey ?? `${record.idempotencyKey}:${index}`,
+      }));
+      try {
+        await http.sendMessages(batch);
+      } catch (error) {
+        if (!isDeterministicValidationError(error)) throw error;
+        await deliverSessionBatchItems(input, batch);
+      }
+      return;
+    }
+    if (record.kind === "insight") {
+      await http.recordInsightObservation({
+        ...(record.payload as Parameters<RelayHttpClient["recordInsightObservation"]>[0]),
+        occurredAt: record.occurredAt,
+      });
+      return;
+    }
+    if (record.kind === "continuation-archive") {
+      await deliverContinuationArchiveRecord({
+        record,
+        http,
+        updatePayload: input.updatePayload,
+        sessionLog: input.sessionLog,
+      });
+      return;
+    }
+    if (record.kind === "mcp-tool-call") {
+      await deliverBufferedMcpCall(input);
+      return;
+    }
+    logger.warn("outbox", `dropping event with unknown kind: ${record.kind}`);
+  } catch (error) {
+    if (isHttpStatusError(error, 409)) return;
+    if (isHttpAuthError(error)) input.recoverRuntimeTokenAfterAuthFailure("outbox");
+    throw error;
+  }
+}
+
+async function deliverSessionBatchItems(input: RunnerOutboxDelivery, messages: SendMessageInput[]): Promise<void> {
+  for (const message of messages) {
+    try {
+      await input.http.sendMessage(message);
+    } catch (error) {
+      if (!isDeterministicValidationError(error)) throw error;
+      logger.warn("outbox", `dropping invalid session batch item: ${errMessage(error)}`);
+    }
+  }
+}
+
+function isDeterministicValidationError(error: unknown): boolean {
+  const status = typeof error === "object" && error !== null ? (error as { status?: unknown }).status : undefined;
+  return status === 400 || status === 413 || status === 422;
+}
+
+export async function deliverBufferedMcpCall(input: {
+  record: OutboxRecord;
+  relayUrl: string;
+  token?: string;
+  recoverRuntimeTokenAfterAuthFailure(source: string): void;
+}): Promise<void> {
+  const payload = input.record.payload as { tool: string; arguments: Record<string, unknown> };
+  const headers: Record<string, string> = { "content-type": "application/json" };
+  if (input.token) headers.authorization = `Bearer ${input.token}`;
+  const response = await fetch(relayMcpEndpoint(input.relayUrl), {
+    method: "POST",
+    headers,
+    body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name: payload.tool, arguments: payload.arguments } }),
+  });
+  if (response.status === 401 || response.status === 403) {
+    input.recoverRuntimeTokenAfterAuthFailure("mcp-outbox");
+    throw new Error(`relay rejected buffered ${payload.tool} with ${response.status}`);
+  }
+  if (response.status >= 500) throw new Error(`relay ${response.status} on buffered ${payload.tool}`);
+  if (!response.ok) {
+    const body = await response.text().catch(() => "");
+    logger.warn("mcp-outbox", `buffered ${payload.tool} permanently rejected (${response.status}); dropping: ${body.slice(0, 200)}`);
+    return;
+  }
+  const json = await response.json().catch(() => null) as { error?: { message?: string } } | null;
+  if (json?.error) {
+    logger.warn("mcp-outbox", `buffered ${payload.tool} returned a tool error; dropping: ${json.error.message ?? "(no detail)"}`);
+  }
+}

package/runner/src/message-body-config/config.ts

@@ -0,0 +1,3 @@
+export function messageBodyMaxCharsFromEnv(): string | undefined {
+  return process.env.AGENT_RELAY_MESSAGE_BODY_MAX_CHARS;
+}

package/runner/src/message-body-config.ts

@@ -0,0 +1 @@
+export { messageBodyMaxCharsFromEnv } from "./message-body-config/config";

package/runner/src/native-self-resume.ts

@@ -0,0 +1,202 @@
+import { errMessage } from "agent-relay-sdk";
+import type { ProviderStatusEvent, SemanticStatus } from "./adapter";
+import { nativeSelfResumeTimeoutMsFromEnv } from "./config";
+
+interface RunnerTimelineEvent {
+  status: string;
+  id?: string;
+  timestamp: number;
+  title?: string;
+  body?: string;
+  icon?: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface NativeSelfResumeHandoff {
+  commandId: string;
+  generation?: number;
+  compactedAt?: number;
+  timer: ReturnType<typeof setTimeout>;
+  resolve: (result: Record<string, unknown>) => void;
+  reject: (error: Error) => void;
+  promise: Promise<Record<string, unknown>>;
+}
+
+export function isNativeSelfResumeCompact(type: string, params: Record<string, unknown>): boolean {
+  if (type !== "agent.compact") return false;
+  const selfResume = params.selfResume;
+  return Boolean(selfResume && typeof selfResume === "object" && !Array.isArray(selfResume) && (selfResume as Record<string, unknown>).mode === "native");
+}
+
+export class NativeSelfResumeTracker {
+  private current?: NativeSelfResumeHandoff;
+
+  get active(): NativeSelfResumeHandoff | undefined {
+    return this.current;
+  }
+
+  begin(commandId: string, params: Record<string, unknown>): NativeSelfResumeHandoff {
+    this.finish(this.current);
+    let resolve!: (result: Record<string, unknown>) => void;
+    let reject!: (error: Error) => void;
+    const promise = new Promise<Record<string, unknown>>((res, rej) => {
+      resolve = res;
+      reject = rej;
+    });
+    const selfResume = params.selfResume && typeof params.selfResume === "object" && !Array.isArray(params.selfResume)
+      ? params.selfResume as Record<string, unknown>
+      : undefined;
+    const generation = typeof selfResume?.generation === "number" ? selfResume.generation : undefined;
+    const timeoutMs = nativeSelfResumeTimeoutMsFromEnv();
+    const handoff: NativeSelfResumeHandoff = {
+      commandId,
+      generation,
+      promise,
+      resolve,
+      reject,
+      timer: setTimeout(() => {
+        if (this.current !== handoff) return;
+        this.current = undefined;
+        reject(new Error(`native self-resume timed out after ${Math.round(timeoutMs / 1000)}s`));
+      }, timeoutMs),
+    };
+    this.current = handoff;
+    return handoff;
+  }
+
+  finish(handoff?: NativeSelfResumeHandoff): void {
+    if (!handoff) return;
+    clearTimeout(handoff.timer);
+    if (this.current === handoff) this.current = undefined;
+  }
+
+  resolve(handoff: NativeSelfResumeHandoff, result: Record<string, unknown>): void {
+    this.finish(handoff);
+    handoff.resolve(result);
+  }
+
+  reject(handoff: NativeSelfResumeHandoff, error: Error): void {
+    this.finish(handoff);
+    handoff.reject(error);
+  }
+
+  noteCompacted(timestamp: number = Date.now()): void {
+    if (!this.current) return;
+    this.current.compactedAt = timestamp;
+  }
+}
+
+interface NativeSelfResumeRecoveryOptions {
+  handoff: NativeSelfResumeHandoff;
+  tracker: NativeSelfResumeTracker;
+  providerSessionId: string;
+  status: SemanticStatus;
+  diagnostics: Record<string, unknown>;
+  now: number;
+  publishTimeline(event: RunnerTimelineEvent): void;
+  setTerminalProviderExit(marker: Record<string, unknown>): void;
+  setProviderStatus(update: ProviderStatusEvent): void;
+  restartProvider(resumeId?: string): Promise<void>;
+  publishStatus(): void;
+  scheduleDrain(): void;
+}
+
+export async function recoverNativeSelfResumeExit(opts: NativeSelfResumeRecoveryOptions): Promise<void> {
+  const resumeId = typeof opts.diagnostics.claudeResumeId === "string" && opts.diagnostics.claudeResumeId.length > 0
+    ? opts.diagnostics.claudeResumeId
+    : undefined;
+  if (opts.handoff.compactedAt && resumeId) {
+    await recoverWithClaudeResume(opts, resumeId);
+    return;
+  }
+  await recoverWithFreshLaunch(opts, resumeId);
+}
+
+async function recoverWithClaudeResume(opts: NativeSelfResumeRecoveryOptions, resumeId: string): Promise<void> {
+  opts.publishTimeline({
+    status: "provider.restart_decision",
+    id: `provider-self-resume-relaunch-${opts.providerSessionId}-${opts.now}`,
+    timestamp: Date.now(),
+    title: "Self-resume relaunching",
+    body: `Claude exited during native self-resume; runner is relaunching with claude --resume ${resumeId}.`,
+    icon: "ti-refresh",
+    metadata: selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      decision: "resume-native-self-resume",
+      reason: "self-resume-exit-after-compact",
+      claudeResumeId: resumeId,
+      compactedAt: opts.handoff.compactedAt,
+    }),
+  });
+  try {
+    await opts.restartProvider(resumeId);
+    opts.publishStatus();
+    opts.scheduleDrain();
+    opts.tracker.resolve(opts.handoff, { recovered: true, via: "claude-resume", claudeResumeId: resumeId });
+  } catch (error) {
+    const message = errMessage(error);
+    opts.setTerminalProviderExit(selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      reason: "self-resume-relaunch-failed",
+      claudeResumeId: resumeId,
+      compactedAt: opts.handoff.compactedAt,
+      lastError: message,
+    }));
+    opts.setProviderStatus({
+      status: "offline",
+      reason: "provider-turn",
+      id: `provider-self-resume-relaunch-failed-${opts.providerSessionId}`,
+      clear: ["provider-turn", "subagent", "background-script"],
+    });
+    opts.tracker.reject(opts.handoff, new Error(`self-resume relaunch failed: ${message}`));
+  }
+}
+
+async function recoverWithFreshLaunch(opts: NativeSelfResumeRecoveryOptions, resumeId?: string): Promise<void> {
+  opts.publishTimeline({
+    status: "provider.restart_decision",
+    id: `provider-self-resume-fresh-launch-${opts.providerSessionId}-${opts.now}`,
+    timestamp: Date.now(),
+    title: "Self-resume fresh-launching",
+    body: "Claude exited during native self-resume before a compacted marker; runner is fresh-launching so Relay can inject the persisted continuation.",
+    icon: "ti-refresh",
+    metadata: selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      decision: "fresh-launch-native-self-resume",
+      reason: "self-resume-exit-before-compact",
+      ...(resumeId ? { claudeResumeId: resumeId } : {}),
+    }),
+  });
+  try {
+    await opts.restartProvider();
+    opts.publishStatus();
+    opts.scheduleDrain();
+    opts.tracker.resolve(opts.handoff, {
+      recovered: true,
+      via: "fresh-launch",
+      ...(resumeId ? { claudeResumeId: resumeId } : {}),
+    });
+  } catch (error) {
+    const message = errMessage(error);
+    opts.setTerminalProviderExit(selfResumeMetadata(opts.handoff, opts.diagnostics, {
+      reason: "self-resume-fresh-launch-failed",
+      ...(resumeId ? { claudeResumeId: resumeId } : {}),
+      lastError: message,
+    }));
+    opts.setProviderStatus({
+      status: "offline",
+      reason: "provider-turn",
+      id: `provider-self-resume-fresh-launch-failed-${opts.providerSessionId}`,
+      clear: ["provider-turn", "subagent", "background-script"],
+    });
+    opts.tracker.reject(opts.handoff, new Error(`self-resume fresh launch failed: ${message}`));
+  }
+}
+
+function selfResumeMetadata(handoff: NativeSelfResumeHandoff, diagnostics: Record<string, unknown>, extra: Record<string, unknown>): Record<string, unknown> {
+  return {
+    eventType: "provider.restart_decision",
+    selfResume: true,
+    selfResumeCommandId: handoff.commandId,
+    selfResumeGeneration: handoff.generation ?? null,
+    ...extra,
+    ...diagnostics,
+  };
+}