@desplega.ai/agent-swarm 1.156.0 → 1.158.0

package/src/otel-impl.ts

@@ -1,8 +1,12 @@
+import { monitorEventLoopDelay } from "node:perf_hooks";
+import { format } from "node:util";
 import {
+  type BatchObservableResult,
   type Counter,
   context,
   type Gauge,
   type Histogram,
+  type Meter,
   metrics,
   propagation,
   ROOT_CONTEXT,
@@ -12,6 +16,7 @@ import {
   type Tracer,
   trace,
 } from "@opentelemetry/api";
+import { logs, SeverityNumber } from "@opentelemetry/api-logs";
 import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import {
@@ -149,7 +154,12 @@ export function resolveServiceName(serviceRole: string): string {
   return serviceRole === "api" ? `${baseServiceName}-api` : baseServiceName;
 }
 
-export async function boot(serviceRole: string): Promise<void> {
+export interface BootOptions {
+  /** Mirror console output into OTel log records (API role only). */
+  exportConsoleLogs?: boolean;
+}
+
+export async function boot(serviceRole: string, options: BootOptions = {}): Promise<void> {
   if (sdk) return;
 
   const configuredResourceAttributes = parseResourceAttributes();
@@ -191,6 +201,14 @@ export async function boot(serviceRole: string): Promise<void> {
 
   sdk.start();
 
+  if (serviceRole === "api") {
+    startEventLoopDelayMetrics(metrics.getMeter(METER_NAME));
+    // NodeSDK already registers a global LoggerProvider exporting over OTLP
+    // to the same endpoint and resource as traces (OTEL_LOGS_EXPORTER
+    // defaults to otlp); nothing feeds it until the bridge is installed.
+    if (options.exportConsoleLogs) installConsoleLogBridge();
+  }
+
   const shutdown = async () => {
     try {
       await sdk?.shutdown();
@@ -204,8 +222,140 @@ export async function boot(serviceRole: string): Promise<void> {
 }
 
 export async function shutdown(): Promise<void> {
+  // SDK first: its final flush still reads the event-loop gauges and drains
+  // bridged log records.
   await sdk?.shutdown();
   sdk = undefined;
+  stopEventLoopDelayMetrics();
+  uninstallConsoleLogBridge();
+}
+
+let stopEventLoopDelay: (() => void) | undefined;
+
+/**
+ * Event-loop delay from `perf_hooks.monitorEventLoopDelay`, exported as the
+ * semconv `nodejs.eventloop.delay.*` gauges (seconds). Each collection reads
+ * the histogram and resets it, so `max` is the worst stall since the last
+ * export. Server spans only start once the loop frees, so a stall never shows
+ * in span durations: a request queued behind a 3s synchronous query still
+ * reports a 14ms span. This metric is what sees the stall.
+ */
+export function startEventLoopDelayMetrics(meter: Meter): void {
+  if (stopEventLoopDelay) return;
+  const histogram = monitorEventLoopDelay({ resolution: 10 });
+  histogram.enable();
+  const gauges = {
+    min: meter.createObservableGauge("nodejs.eventloop.delay.min", {
+      description: "Minimum event loop delay since the last export",
+      unit: "s",
+    }),
+    max: meter.createObservableGauge("nodejs.eventloop.delay.max", {
+      description: "Maximum event loop delay since the last export: the longest stall",
+      unit: "s",
+    }),
+    mean: meter.createObservableGauge("nodejs.eventloop.delay.mean", {
+      description: "Mean event loop delay since the last export",
+      unit: "s",
+    }),
+    p50: meter.createObservableGauge("nodejs.eventloop.delay.p50", {
+      description: "50th percentile event loop delay since the last export",
+      unit: "s",
+    }),
+    p90: meter.createObservableGauge("nodejs.eventloop.delay.p90", {
+      description: "90th percentile event loop delay since the last export",
+      unit: "s",
+    }),
+    p99: meter.createObservableGauge("nodejs.eventloop.delay.p99", {
+      description: "99th percentile event loop delay since the last export",
+      unit: "s",
+    }),
+  };
+  const seconds = (ns: number) => ns / 1e9;
+  const collect = (observer: BatchObservableResult) => {
+    // No sample yet (first interval shorter than the resolution): the
+    // histogram's min is a sentinel then, so report nothing.
+    if (histogram.count === 0) return;
+    observer.observe(gauges.min, seconds(histogram.min));
+    observer.observe(gauges.max, seconds(histogram.max));
+    observer.observe(gauges.mean, seconds(histogram.mean));
+    observer.observe(gauges.p50, seconds(histogram.percentile(50)));
+    observer.observe(gauges.p90, seconds(histogram.percentile(90)));
+    observer.observe(gauges.p99, seconds(histogram.percentile(99)));
+    histogram.reset();
+  };
+  const observables = Object.values(gauges);
+  meter.addBatchObservableCallback(collect, observables);
+  stopEventLoopDelay = () => {
+    meter.removeBatchObservableCallback(collect, observables);
+    histogram.disable();
+    stopEventLoopDelay = undefined;
+  };
+}
+
+export function stopEventLoopDelayMetrics(): void {
+  stopEventLoopDelay?.();
+}
+
+const CONSOLE_LOG_BODY_MAX_CHARS = 16_384;
+const CONSOLE_SEVERITY = {
+  debug: { number: SeverityNumber.DEBUG, text: "DEBUG" },
+  log: { number: SeverityNumber.INFO, text: "INFO" },
+  info: { number: SeverityNumber.INFO, text: "INFO" },
+  warn: { number: SeverityNumber.WARN, text: "WARN" },
+  error: { number: SeverityNumber.ERROR, text: "ERROR" },
+} as const;
+type ConsoleMethod = keyof typeof CONSOLE_SEVERITY;
+
+let restoreConsole: (() => void) | undefined;
+
+/**
+ * Mirror console output into OTel log records (opt-in: OTEL_EXPORT_API_LOGS).
+ * stdout/stderr stay the primary sink; each line is also emitted to the
+ * global LoggerProvider, scrubbed at this egress point and correlated with
+ * the active span.
+ */
+export function installConsoleLogBridge(): void {
+  if (restoreConsole) return;
+  const logger = logs.getLogger(METER_NAME);
+  const originals = new Map<ConsoleMethod, (...args: unknown[]) => void>();
+  let emitting = false;
+  for (const method of Object.keys(CONSOLE_SEVERITY) as ConsoleMethod[]) {
+    const original = console[method].bind(console);
+    originals.set(method, console[method]);
+    console[method] = (...args: unknown[]) => {
+      original(...args);
+      // An exporter or scrubber that logs must not recurse into itself.
+      if (emitting) return;
+      emitting = true;
+      try {
+        // Scrub before truncating so a cut can never split a secret into an
+        // unrecognizable fragment.
+        const text = scrubSecrets(format(...args));
+        const body =
+          text.length > CONSOLE_LOG_BODY_MAX_CHARS
+            ? `${text.slice(0, CONSOLE_LOG_BODY_MAX_CHARS)}…`
+            : text;
+        logger.emit({
+          severityNumber: CONSOLE_SEVERITY[method].number,
+          severityText: CONSOLE_SEVERITY[method].text,
+          body,
+          attributes: { "log.source": "console" },
+        });
+      } catch {
+        // Log export is best-effort; never break the caller's console call.
+      } finally {
+        emitting = false;
+      }
+    };
+  }
+  restoreConsole = () => {
+    for (const [method, original] of originals) console[method] = original;
+    restoreConsole = undefined;
+  };
+}
+
+export function uninstallConsoleLogBridge(): void {
+  restoreConsole?.();
 }
 
 export async function withSpan<T>(

package/src/otel.ts

@@ -94,18 +94,29 @@ export function isOtelEnabled(): boolean {
   return otelConfigured();
 }
 
-export function isPollTracingEnabled(): boolean {
-  const v = (process.env.OTEL_TRACE_POLL ?? "").trim().toLowerCase();
+function envFlagEnabled(name: string): boolean {
+  const v = (process.env[name] ?? "").trim().toLowerCase();
   return v === "1" || v === "true" || v === "yes" || v === "on";
 }
 
+export function isPollTracingEnabled(): boolean {
+  return envFlagEnabled("OTEL_TRACE_POLL");
+}
+
+/** Mirror the API's console output to the OTLP collector as log records. Off by default. */
+export function isApiLogExportEnabled(): boolean {
+  return envFlagEnabled("OTEL_EXPORT_API_LOGS");
+}
+
 export async function initOtel(serviceRole = process.env.AGENT_ROLE || "api"): Promise<void> {
   if (!otelConfigured() || initialized) return;
   initialized = true;
 
   try {
     const impl = await import("./otel-impl");
-    await impl.boot(serviceRole);
+    await impl.boot(serviceRole, {
+      exportConsoleLogs: serviceRole === "api" && isApiLogExportEnabled(),
+    });
     realWithSpan = impl.withSpan;
     realStartSpan = impl.startSpan;
     realWithRemoteContext = impl.withRemoteContext;

package/src/prompts/memories.ts

@@ -18,9 +18,26 @@ export type RelevantMemory = {
   name: string;
   content: string;
   similarity: number;
+  rawSimilarity?: number;
 };
 
-export const SIMILARITY_THRESHOLD = 0.4;
+/**
+ * Minimum pre-boost relevance for a memory to be injected into a task prompt.
+ *
+ * Compared against `rawSimilarity`, the [0,1] match score every retrieval arm
+ * emits before rerank (vec cosine, hybrid fused cosine, graph-derived), never
+ * against the composite `similarity`: the reranker's access, source-quality
+ * and usefulness multipliers can lift a score up to 4.5x, so a composite
+ * threshold admits unrelated rows. 0.55 sits between an unrelated query's best
+ * cosine (~0.33 on the production embedding model) and an exact-name hybrid
+ * hit (~0.65 fused).
+ */
+export const SIMILARITY_THRESHOLD = 0.55;
+
+/** The score the injection threshold applies to. Older servers omit `rawSimilarity`. */
+export function memoryRelevance(memory: { similarity: number; rawSimilarity?: number }): number {
+  return memory.rawSimilarity ?? memory.similarity;
+}
 
 const RATE_TOOL_HINT = `
 
@@ -35,7 +52,7 @@ This trains the swarm to surface better memories next time. Use sparingly:
  * append entirely (matching pre-step-5 behaviour).
  */
 export function renderMemoriesPrompt(memories: RelevantMemory[]): string | null {
-  const useful = memories.filter((m) => m.similarity > SIMILARITY_THRESHOLD);
+  const useful = memories.filter((m) => memoryRelevance(m) > SIMILARITY_THRESHOLD);
   if (useful.length === 0) return null;
 
   const memoryContext = useful

package/src/providers/claude-managed-models.ts

@@ -31,6 +31,7 @@ export const CLAUDE_MANAGED_MODELS = [
   "claude-opus-5",
   "claude-fable-5",
   "claude-mythos-5",
+  "claude-sonnet-5-5",
   "claude-sonnet-5",
   "claude-sonnet-4-6",
   "claude-opus-4-8",
@@ -72,6 +73,7 @@ export interface ClaudeManagedModelPricing extends ClaudeManagedTokenPricing {
  * - claude-mythos-5:  $10 / $50 / $1.00 / $12.50   (limited availability, verified 2026-06-10)
  * - claude-opus-5-5:  $4 / $20 / $0.20 / $5.00     (verified 2026-09-22)
  * - claude-opus-5:    $5 / $25 / $0.50 / $6.25     (verified 2026-07-25)
+ * - claude-sonnet-5-5: $2 / $10 / $0.20 / $2.50    (verified 2026-09-28)
  * - claude-sonnet-5:  $2 / $10 / $0.20 / $2.50     (standard rate, verified 2026-08-29)
  * - claude-sonnet-4-6: $3 / $15 / $0.30 / $3.75    (in / out / cache-read / cache-write)
  * - claude-opus-4-8:   $5 / $25 / $0.50 / $6.25    (verified 2026-05-28)
@@ -116,6 +118,12 @@ export const CLAUDE_MANAGED_MODEL_PRICING: Record<ClaudeManagedModel, ClaudeMana
     cacheReadPerMillion: 1.0,
     cacheWritePerMillion: 12.5,
   },
+  "claude-sonnet-5-5": {
+    inputPerMillion: 2.0,
+    outputPerMillion: 10.0,
+    cacheReadPerMillion: 0.2,
+    cacheWritePerMillion: 2.5,
+  },
   "claude-sonnet-5": {
     inputPerMillion: 2.0,
     outputPerMillion: 10.0,

package/src/providers/codex-agents-md.ts

@@ -21,32 +21,121 @@
  *       instructions.
  *     - Otherwise, write just the block.
  *     - Mark `createdFresh: true` so cleanup removes the file entirely.
+ * - `AGENTS.md` is a symlink (many repos ship `AGENTS.md -> CLAUDE.md`):
+ *     never write through it. Swap in a real file holding the block plus the
+ *     link target's contents, and restore the symlink on cleanup. The target is
+ *     only read, and edits only carried back to it, when its real path is a
+ *     regular file inside the cwd. A crash-recovery record (cwd + link target)
+ *     is kept OUTSIDE the repo, so the next session can restore the link if
+ *     this one crashed before cleanup. Repo content never names a path.
  * - Existing `AGENTS.md` already contains the block: replace the block with
  *   the fresh contents.
  * - Existing `AGENTS.md` without the block: prepend the block.
  *
  * Cleanup mirrors the creation logic — if we created the file fresh, delete
- * it; otherwise re-read the current AGENTS.md and strip just the managed
- * block so anything the agent appended during the session is preserved.
+ * it; if we swapped out a symlink, restore it; otherwise re-read the current
+ * AGENTS.md and strip just the managed block so anything the agent appended
+ * during the session is preserved.
  *
  * The helper is deliberately isolated from the adapter so it can be
  * unit-tested without pulling in the Codex SDK.
  */
 
-import { join } from "node:path";
+import { lstat, mkdir, readlink, realpath, symlink, unlink } from "node:fs/promises";
+import { homedir } from "node:os";
+import { dirname, join, resolve, sep } from "node:path";
 
 const BLOCK_OPEN = "<swarm_system_prompt>";
 const BLOCK_CLOSE = "</swarm_system_prompt>";
 const BLOCK_REGEX = /<swarm_system_prompt>[\s\S]*?<\/swarm_system_prompt>\n?/;
+const DEFAULT_STATE_DIR = join(homedir(), ".cache", "agent-swarm", "codex-agents-md");
 
 export interface CodexAgentsMdHandle {
   cleanup(): Promise<void>;
 }
 
+export interface CodexAgentsMdOptions {
+  /** Where symlink crash-recovery records live. Must be outside any repo. */
+  stateDir?: string;
+}
+
+interface SymlinkRecord {
+  cwd: string;
+  linkTarget: string;
+}
+
 const NOOP_HANDLE: CodexAgentsMdHandle = {
   cleanup: async () => {},
 };
 
+async function readIfExists(path: string): Promise<string | null> {
+  const file = Bun.file(path);
+  return (await file.exists()) ? await file.text() : null;
+}
+
+function recordPathFor(stateDir: string, realCwd: string): string {
+  const key = new Bun.CryptoHasher("sha256").update(realCwd).digest("hex");
+  return join(stateDir, `${key}.json`);
+}
+
+async function readRecord(recordPath: string, realCwd: string): Promise<SymlinkRecord | null> {
+  try {
+    const record = JSON.parse(await Bun.file(recordPath).text()) as Partial<SymlinkRecord>;
+    if (record.cwd === realCwd && typeof record.linkTarget === "string") {
+      return { cwd: record.cwd, linkTarget: record.linkTarget };
+    }
+  } catch {
+    // Missing or unreadable record: nothing to recover.
+  }
+  return null;
+}
+
+/**
+ * Real path of the symlink target when it is a regular file inside the cwd,
+ * otherwise null. Out-of-tree and dangling targets are never read or written.
+ */
+async function inTreeTarget(
+  realCwd: string,
+  agentsMdPath: string,
+  linkTarget: string,
+): Promise<string | null> {
+  const target = await realpath(resolve(dirname(agentsMdPath), linkTarget)).catch(() => null);
+  const prefix = realCwd.endsWith(sep) ? realCwd : `${realCwd}${sep}`;
+  if (target === null || !target.startsWith(prefix)) {
+    return null;
+  }
+  const stat = await lstat(target).catch(() => null);
+  return stat?.isFile() ? target : null;
+}
+
+/**
+ * Put the `AGENTS.md -> linkTarget` symlink back in place of our swapped-in
+ * real file. Edits the agent made to AGENTS.md during the session would have
+ * landed in the link target had the link been in place, so carry them over
+ * (block stripped) when the target is in-tree and its contents differ.
+ */
+async function restoreSymlink(
+  agentsMdPath: string,
+  realCwd: string,
+  linkTarget: string,
+): Promise<void> {
+  const stat = await lstat(agentsMdPath).catch(() => null);
+  const current = stat?.isFile() ? await Bun.file(agentsMdPath).text() : null;
+  await unlink(agentsMdPath).catch(() => {});
+  await symlink(linkTarget, agentsMdPath);
+  if (current === null) {
+    return;
+  }
+  const target = await inTreeTarget(realCwd, agentsMdPath, linkTarget);
+  if (target === null) {
+    return;
+  }
+  const stripped = current.replace(BLOCK_REGEX, "");
+  if (stripped !== (await Bun.file(target).text())) {
+    await Bun.write(target, stripped);
+  }
+}
+
 /**
  * Write (or refresh) a managed `<swarm_system_prompt>` block inside
  * `${cwd}/AGENTS.md`. Returns a handle whose `cleanup()` reverses the edit.
@@ -56,6 +145,7 @@ const NOOP_HANDLE: CodexAgentsMdHandle = {
 export async function writeCodexAgentsMd(
   cwd: string | undefined,
   systemPrompt: string | undefined,
+  options: CodexAgentsMdOptions = {},
 ): Promise<CodexAgentsMdHandle> {
   if (!cwd || !systemPrompt) {
     return NOOP_HANDLE;
@@ -63,33 +153,66 @@ export async function writeCodexAgentsMd(
 
   const agentsMdPath = join(cwd, "AGENTS.md");
   const claudeMdPath = join(cwd, "CLAUDE.md");
+  const stateDir = options.stateDir ?? DEFAULT_STATE_DIR;
+  const realCwd = await realpath(cwd).catch(() => resolve(cwd));
+  const recordPath = recordPathFor(stateDir, realCwd);
   const block = `${BLOCK_OPEN}\n${systemPrompt}\n${BLOCK_CLOSE}`;
 
-  const agentsMdFile = Bun.file(agentsMdPath);
-  const existingAgentsMdExists = await agentsMdFile.exists();
+  let stat = await lstat(agentsMdPath).catch(() => null);
+
+  // A previous session swapped a symlink out and crashed before cleanup. Only
+  // our own out-of-repo record can trigger a restore, and only while AGENTS.md
+  // is still the real file holding our block.
+  const record = await readRecord(recordPath, realCwd);
+  if (record) {
+    if (stat?.isFile() && BLOCK_REGEX.test(await Bun.file(agentsMdPath).text())) {
+      await restoreSymlink(agentsMdPath, realCwd, record.linkTarget);
+      stat = await lstat(agentsMdPath);
+    }
+    await unlink(recordPath).catch(() => {});
+  }
+
+  if (stat?.isSymbolicLink()) {
+    const linkTarget = await readlink(agentsMdPath);
+    const target = await inTreeTarget(realCwd, agentsMdPath, linkTarget);
+    const targetContent = target !== null ? await Bun.file(target).text() : "";
+    // Record before swapping, so a crash at any later point stays recoverable.
+    await mkdir(stateDir, { recursive: true });
+    await Bun.write(recordPath, JSON.stringify({ cwd: realCwd, linkTarget }));
+    // Never write through the link: replace it with a real file for the session.
+    await unlink(agentsMdPath);
+    await Bun.write(agentsMdPath, `${block}\n${targetContent.replace(BLOCK_REGEX, "")}`);
+
+    return {
+      async cleanup(): Promise<void> {
+        try {
+          await restoreSymlink(agentsMdPath, realCwd, linkTarget);
+          await unlink(recordPath).catch(() => {});
+        } catch {
+          // Cleanup is best-effort; the record lets the next session finish
+          // the restore.
+        }
+      },
+    };
+  }
 
   let createdFresh = false;
   let newContent: string;
 
-  if (!existingAgentsMdExists) {
+  if (!stat) {
     // No AGENTS.md yet — prefer CLAUDE.md content as a base if present.
-    const claudeMdFile = Bun.file(claudeMdPath);
-    const claudeMdExists = await claudeMdFile.exists();
-    if (claudeMdExists) {
-      const claudeContent = await claudeMdFile.text();
-      newContent = `${block}\n\n${claudeContent}`;
-    } else {
-      newContent = `${block}\n`;
-    }
+    const claudeContent = await readIfExists(claudeMdPath);
+    newContent = claudeContent !== null ? `${block}\n\n${claudeContent}` : `${block}\n`;
     createdFresh = true;
   } else {
-    const existingContent = await agentsMdFile.text();
+    const existingContent = await Bun.file(agentsMdPath).text();
     if (BLOCK_REGEX.test(existingContent)) {
       // Replace the stale block in place.
       newContent = existingContent.replace(BLOCK_REGEX, `${block}\n`);
     } else {
-      // Prepend the block, keeping existing content intact.
-      newContent = `${block}\n\n${existingContent}`;
+      // Prepend the block with a single newline so cleanup's strip restores
+      // the original bytes exactly.
+      newContent = `${block}\n${existingContent}`;
     }
   }