@evomap/evolver 1.89.17 → 1.89.19

package/src/proxy/mailbox/state.js

@@ -0,0 +1,207 @@
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+const PRIVATE_DIR_MODE = 0o700;
+const PRIVATE_FILE_MODE = 0o600;
+const STATE_LOCK_STALE_MS = 10_000;
+const STATE_LOCK_WAIT_MS = 10;
+
+const MAILBOX_NODE_SECRET_STATE_KEYS = Object.freeze([
+  'node_secret',
+  'node_secret_version',
+  'node_secret_source',
+  'node_secret_env_suppressed',
+]);
+
+const MAILBOX_NODE_SECRET_STATE_KEY_SET = new Set(MAILBOX_NODE_SECRET_STATE_KEYS);
+const MAILBOX_NODE_SECRET_TUPLE_KEYS = Object.freeze([
+  'node_secret',
+  'node_secret_version',
+  'node_secret_source',
+]);
+const MAILBOX_NODE_SECRET_TUPLE_KEY_SET = new Set(MAILBOX_NODE_SECRET_TUPLE_KEYS);
+
+function bestEffortChmod(filePath, mode) {
+  try { fs.chmodSync(filePath, mode); } catch { /* best effort; no-op on Windows */ }
+}
+
+function ensurePrivateDir(dir) {
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true, mode: PRIVATE_DIR_MODE });
+  }
+  bestEffortChmod(dir, PRIVATE_DIR_MODE);
+}
+
+function writePrivateFile(filePath, content) {
+  fs.writeFileSync(filePath, content, { encoding: 'utf8', mode: PRIVATE_FILE_MODE });
+  bestEffortChmod(filePath, PRIVATE_FILE_MODE);
+}
+
+function sleepSync(ms) {
+  try {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+  } catch {
+    const end = Date.now() + ms;
+    while (Date.now() < end) {}
+  }
+}
+
+function acquireStateFileLock(stateFile) {
+  ensurePrivateDir(path.dirname(stateFile));
+  const lockDir = `${stateFile}.lock`;
+  const startedAt = Date.now();
+  while (true) {
+    try {
+      fs.mkdirSync(lockDir, { mode: PRIVATE_DIR_MODE });
+      bestEffortChmod(lockDir, PRIVATE_DIR_MODE);
+      return function releaseStateFileLock() {
+        try { fs.rmdirSync(lockDir); } catch {}
+      };
+    } catch (e) {
+      if (!e || e.code !== 'EEXIST') throw e;
+      try {
+        const ageMs = Date.now() - fs.statSync(lockDir).mtimeMs;
+        if (ageMs > STATE_LOCK_STALE_MS) {
+          fs.rmSync(lockDir, { recursive: true, force: true });
+          continue;
+        }
+      } catch (statErr) {
+        if (statErr && statErr.code === 'ENOENT') continue;
+        throw statErr;
+      }
+      if (Date.now() - startedAt > STATE_LOCK_STALE_MS) {
+        const err = new Error('timed out waiting for mailbox state lock');
+        err.code = 'MAILBOX_STATE_LOCK_TIMEOUT';
+        throw err;
+      }
+      sleepSync(STATE_LOCK_WAIT_MS);
+    }
+  }
+}
+
+function isPlainState(value) {
+  return value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isHubRotatedNodeSecretState(state) {
+  return isPlainState(state)
+    && state.node_secret_source === 'hub_rotate'
+    && typeof state.node_secret === 'string'
+    && Boolean(state.node_secret.trim());
+}
+
+function parseNodeSecretVersion(value) {
+  const n = Number(value);
+  return Number.isInteger(n) && n > 0 ? n : null;
+}
+
+function isFullNodeSecretTupleUpdate(updatedSet) {
+  return Boolean(updatedSet) && MAILBOX_NODE_SECRET_TUPLE_KEYS.every((key) => updatedSet.has(key));
+}
+
+function canApplyPartialNodeSecretTupleWrite(key, disk, next) {
+  if (key !== 'node_secret_version') return false;
+  if (!isHubRotatedNodeSecretState(disk)) return true;
+  if (!isHubRotatedNodeSecretState(next)) return false;
+  if (String(next.node_secret).trim() !== String(disk.node_secret).trim()) return false;
+  const nextVersion = parseNodeSecretVersion(next.node_secret_version);
+  const diskVersion = parseNodeSecretVersion(disk.node_secret_version);
+  return Boolean(nextVersion && (!diskVersion || nextVersion >= diskVersion));
+}
+
+/**
+ * @param {string} stateFile
+ * @returns {Record<string, unknown>|null}
+ */
+function readMailboxStateFile(stateFile) {
+  try {
+    if (!fs.existsSync(stateFile)) return null;
+    const parsed = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
+    return isPlainState(parsed) ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+function replaceStateFile(stateFile, state) {
+  const dir = path.dirname(stateFile);
+  ensurePrivateDir(dir);
+  const tmp = `${stateFile}.${process.pid}.tmp`;
+  writePrivateFile(tmp, JSON.stringify(state || {}, null, 2) + '\n');
+  // Windows: renameSync throws EPERM when the destination file already
+  // exists, unlike POSIX where rename(2) atomically replaces the target.
+  if (process.platform === 'win32') {
+    try { fs.unlinkSync(stateFile); } catch (e) {
+      if (e.code !== 'ENOENT') throw e;
+    }
+  }
+  fs.renameSync(tmp, stateFile);
+  bestEffortChmod(stateFile, PRIVATE_FILE_MODE);
+}
+
+/**
+ * Merge a partial state write with the latest on-disk state.
+ *
+ * Callers pass the keys they intentionally changed. Secret-bearing keys are
+ * only written when they are in `updatedKeys`, so an old long-running
+ * MailboxStore cannot overwrite a fresher Hub-rotated or divergence-cleared
+ * secret tuple that was written by another process between load and persist.
+ *
+ * @param {string} stateFile
+ * @param {Record<string, unknown>} nextState
+ * @param {Iterable<string>} updatedKeys
+ * @returns {Record<string, unknown>}
+ */
+function writeMergedMailboxStateFile(stateFile, nextState, updatedKeys) {
+  const releaseLock = acquireStateFileLock(stateFile);
+  try {
+    const next = isPlainState(nextState) ? nextState : {};
+    const disk = readMailboxStateFile(stateFile);
+    const hasDiskState = isPlainState(disk);
+    const merged = hasDiskState ? { ...disk } : {};
+    const updatedSet = updatedKeys ? new Set(Array.from(updatedKeys)) : null;
+    const keys = updatedSet ? Array.from(updatedSet) : Object.keys(next);
+    const touchesNodeSecretTuple = keys.some((key) => MAILBOX_NODE_SECRET_TUPLE_KEY_SET.has(key));
+    const preserveDiskNodeSecretTuple = touchesNodeSecretTuple
+      && hasDiskState
+      && isHubRotatedNodeSecretState(disk)
+      && !isFullNodeSecretTupleUpdate(updatedSet);
+
+    for (const key of keys) {
+      if (MAILBOX_NODE_SECRET_STATE_KEY_SET.has(key) && hasDiskState && (!updatedSet || !updatedSet.has(key))) {
+        continue;
+      }
+      if (
+        MAILBOX_NODE_SECRET_TUPLE_KEY_SET.has(key) &&
+        preserveDiskNodeSecretTuple &&
+        !canApplyPartialNodeSecretTupleWrite(key, disk, next)
+      ) {
+        continue;
+      }
+      if (!Object.prototype.hasOwnProperty.call(next, key)) {
+        delete merged[key];
+        continue;
+      }
+      merged[key] = next[key];
+    }
+
+    replaceStateFile(stateFile, merged);
+    return merged;
+  } finally {
+    releaseLock();
+  }
+}
+
+module.exports = {
+  PRIVATE_DIR_MODE,
+  PRIVATE_FILE_MODE,
+  MAILBOX_NODE_SECRET_STATE_KEYS,
+  bestEffortChmod,
+  ensurePrivateDir,
+  writePrivateFile,
+  readMailboxStateFile,
+  isHubRotatedNodeSecretState,
+  writeMergedMailboxStateFile,
+};

package/src/proxy/mailbox/store.js

@@ -3,28 +3,20 @@
 const fs = require('fs');
 const path = require('path');
 const crypto = require('crypto');
+const {
+  PRIVATE_FILE_MODE,
+  bestEffortChmod,
+  ensurePrivateDir,
+  writePrivateFile,
+  readMailboxStateFile,
+  writeMergedMailboxStateFile,
+} = require('./state');
 
 const DEFAULT_CHANNEL = 'evomap-hub';
 const SCHEMA_VERSION = 1;
 const PROXY_PROTOCOL_VERSION = '0.1.0';
-const PRIVATE_DIR_MODE = 0o700;
-const PRIVATE_FILE_MODE = 0o600;
-
-function bestEffortChmod(filePath, mode) {
-  try { fs.chmodSync(filePath, mode); } catch { /* best effort; no-op on Windows */ }
-}
-
-function ensurePrivateDir(dir) {
-  if (!fs.existsSync(dir)) {
-    fs.mkdirSync(dir, { recursive: true, mode: PRIVATE_DIR_MODE });
-  }
-  bestEffortChmod(dir, PRIVATE_DIR_MODE);
-}
-
-function writePrivateFile(filePath, content) {
-  fs.writeFileSync(filePath, content, { encoding: 'utf8', mode: PRIVATE_FILE_MODE });
-  bestEffortChmod(filePath, PRIVATE_FILE_MODE);
-}
+const DEFAULT_MAX_JSONL_LINE_BYTES = 16 * 1024 * 1024;
+const JSONL_READ_CHUNK_BYTES = 64 * 1024;
 
 // Merge `fields` into `target` while stripping keys that can mutate the
 // prototype chain. Mailbox rows are persisted as JSONL and rebuilt on
@@ -107,22 +99,117 @@ function safeParse(payload) {
 // Windows NTFS provides the same atomicity guarantee for O_APPEND writes to
 // regular files that POSIX local filesystems do, so the removal above is
 // equally valid on Windows. No platform-specific code is needed here.
+function jsonlLineForWrite(obj, opts = {}) {
+  const line = JSON.stringify(obj) + '\n';
+  const maxLineBytes = Math.max(1, Math.floor(Number(opts.maxLineBytes) || resolveMaxJsonlLineBytes()));
+  const lineBytes = Buffer.byteLength(line, 'utf8');
+  if (lineBytes > maxLineBytes) {
+    const err = new Error(`mailbox JSONL line is ${lineBytes} bytes; maximum is ${maxLineBytes} bytes`);
+    err.code = 'MAILBOX_JSONL_LINE_TOO_LARGE';
+    err.lineBytes = lineBytes;
+    err.maxLineBytes = maxLineBytes;
+    throw err;
+  }
+  return line;
+}
+
 function appendLine(filePath, obj) {
-  fs.appendFileSync(filePath, JSON.stringify(obj) + '\n', { encoding: 'utf8', mode: PRIVATE_FILE_MODE });
+  const line = jsonlLineForWrite(obj);
+  fs.appendFileSync(filePath, line, { encoding: 'utf8', mode: PRIVATE_FILE_MODE });
   bestEffortChmod(filePath, PRIVATE_FILE_MODE);
 }
 
-function readLines(filePath) {
-  if (!fs.existsSync(filePath)) return [];
-  const content = fs.readFileSync(filePath, 'utf8');
-  const lines = content.split('\n');
-  const results = [];
-  for (const line of lines) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    try { results.push(JSON.parse(trimmed)); } catch { /* skip corrupt lines */ }
+function resolveMaxJsonlLineBytes(env = process.env) {
+  const raw = Number(env.EVOMAP_MAILBOX_MAX_LINE_BYTES);
+  if (Number.isFinite(raw) && raw > 0) return Math.floor(raw);
+  return DEFAULT_MAX_JSONL_LINE_BYTES;
+}
+
+function forEachJsonLine(filePath, onRow, opts = {}) {
+  const stats = { parsed: 0, corrupt: 0, overlong: 0 };
+  if (!fs.existsSync(filePath)) return stats;
+
+  const maxLineBytes = Math.max(1, Math.floor(Number(opts.maxLineBytes) || resolveMaxJsonlLineBytes()));
+  const chunk = Buffer.allocUnsafe(JSONL_READ_CHUNK_BYTES);
+  let fd = null;
+  let parts = [];
+  let lineBytes = 0;
+  let dropping = false;
+
+  const resetLine = () => {
+    parts = [];
+    lineBytes = 0;
+    dropping = false;
+  };
+
+  const appendSegment = (segment) => {
+    if (!segment || segment.length === 0 || dropping) return;
+    if (lineBytes + segment.length > maxLineBytes) {
+      parts = [];
+      lineBytes = 0;
+      dropping = true;
+      return;
+    }
+    parts.push(Buffer.from(segment));
+    lineBytes += segment.length;
+  };
+
+  const finishLine = (hasNewline = false) => {
+    const totalLineBytes = lineBytes + (hasNewline ? 1 : 0);
+    if (dropping) {
+      stats.overlong += 1;
+      resetLine();
+      return;
+    }
+    if (totalLineBytes > maxLineBytes) {
+      stats.overlong += 1;
+      resetLine();
+      return;
+    }
+    if (lineBytes === 0) {
+      resetLine();
+      return;
+    }
+    const trimmed = Buffer.concat(parts, lineBytes).toString('utf8').trim();
+    resetLine();
+    if (!trimmed) return;
+    try {
+      onRow(JSON.parse(trimmed));
+      stats.parsed += 1;
+    } catch {
+      stats.corrupt += 1;
+    }
+  };
+
+  try {
+    fd = fs.openSync(filePath, 'r');
+    while (true) {
+      const bytesRead = fs.readSync(fd, chunk, 0, chunk.length, null);
+      if (bytesRead <= 0) break;
+      let start = 0;
+      for (let i = 0; i < bytesRead; i++) {
+        if (chunk[i] !== 0x0a) continue;
+        appendSegment(chunk.subarray(start, i));
+        finishLine(true);
+        start = i + 1;
+      }
+      if (start < bytesRead) appendSegment(chunk.subarray(start, bytesRead));
+    }
+    finishLine();
+  } catch (err) {
+    stats.read_failed = 1;
+    const detail = err && err.message ? err.message : String(err);
+    const wrapped = new Error(`failed to read mailbox JSONL ${filePath}: ${detail}`);
+    wrapped.code = 'MAILBOX_JSONL_READ_FAILED';
+    wrapped.stats = stats;
+    wrapped.cause = err;
+    throw wrapped;
+  } finally {
+    if (fd !== null) {
+      try { fs.closeSync(fd); } catch { /* ignore close errors */ }
+    }
   }
-  return results;
+  return stats;
 }
 
 // --- In-memory index that backs JSONL persistence ---
@@ -150,19 +237,21 @@ class MailboxStore {
   }
 
   _loadState() {
-    try {
-      if (fs.existsSync(this._stateFile)) {
-        this._state = JSON.parse(fs.readFileSync(this._stateFile, 'utf8'));
-      }
-    } catch {
-      this._state = {};
-    }
+    this._state = readMailboxStateFile(this._stateFile) || {};
     const existingVersion = this._state._schema_version || 0;
+    const beforeMigrationState = { ...this._state };
     if (existingVersion < SCHEMA_VERSION) {
       this._runMigrations(existingVersion, SCHEMA_VERSION);
     }
     this._state._schema_version = SCHEMA_VERSION;
-    this._persistState();
+    const updatedKeys = new Set(['_schema_version']);
+    for (const key of Object.keys(this._state)) {
+      if (this._state[key] !== beforeMigrationState[key]) updatedKeys.add(key);
+    }
+    for (const key of Object.keys(beforeMigrationState)) {
+      if (!Object.prototype.hasOwnProperty.call(this._state, key)) updatedKeys.add(key);
+    }
+    this._persistState(updatedKeys);
   }
 
   _runMigrations(fromVersion, toVersion) {
@@ -174,9 +263,7 @@ class MailboxStore {
     }
   }
 
-  _persistState() {
-    const dir = path.dirname(this._stateFile);
-    ensurePrivateDir(dir);
+  _persistState(updatedKeys) {
     // Round-7 (§20.5): per-PID tmp path. Two evolver processes (daemon +
     // ad-hoc CLI / proxy + loop) writing to the same `${stateFile}.tmp`
     // would otherwise interleave: process B's writeFileSync truncates
@@ -186,20 +273,11 @@ class MailboxStore {
     // load-bearing trigger for the "401-loop -> reauth backoff -> dead
     // for 30 min..4 h" symptom this branch targets. Matches the
     // precedent set by _persistNodeSecret in src/gep/a2aProtocol.js.
-    const tmp = `${this._stateFile}.${process.pid}.tmp`;
-    writePrivateFile(tmp, JSON.stringify(this._state, null, 2) + '\n');
-    // Windows: fs.renameSync throws EPERM when the destination file already
-    // exists, unlike POSIX where rename(2) atomically replaces the target.
-    // Remove the destination first so the rename succeeds on all platforms.
-    // The window between unlink and rename is intentionally tiny; a crash in
-    // that window leaves the tmp file behind (recovered on next _persistState).
-    if (process.platform === 'win32') {
-      try { fs.unlinkSync(this._stateFile); } catch (e) {
-        if (e.code !== 'ENOENT') throw e;
-      }
-    }
-    fs.renameSync(tmp, this._stateFile);
-    bestEffortChmod(this._stateFile, PRIVATE_FILE_MODE);
+    //
+    // Use the shared merge helper so stale long-running stores cannot
+    // resurrect an older node_secret tuple after another path rotates or clears
+    // it directly on disk.
+    this._state = writeMergedMailboxStateFile(this._stateFile, this._state, updatedKeys);
   }
 
   _rebuildIndex() {
@@ -208,17 +286,20 @@ class MailboxStore {
     this._inbound = [];
 
     const TERMINAL = new Set(['synced', 'delivered', 'failed', 'rejected']);
-    const rows = readLines(this._messagesFile);
-    for (const rawRow of rows) {
+    const stats = forEachJsonLine(this._messagesFile, (rawRow) => {
       const row = sanitizeRow(rawRow);
+      if (!row || typeof row !== 'object') return;
       if (row._op === 'update') {
         const existing = this._messages.get(row.id);
         if (existing) safeAssign(existing, row.fields);
-        continue;
+        return;
       }
+      if (!row.id) return;
       this._messages.set(row.id, row);
-    }
+    });
+    this._lastRebuildStats = stats;
     for (const [id, msg] of this._messages) {
+      if (!msg || typeof msg !== 'object') continue;
       if (TERMINAL.has(msg.status)) continue;
       if (msg.direction === 'outbound') this._outbound.push(id);
       else if (msg.direction === 'inbound') this._inbound.push(id);
@@ -430,8 +511,9 @@ class MailboxStore {
   }
 
   setCursor(key, value) {
-    this._state[`cursor:${key}`] = value;
-    this._persistState();
+    const stateKey = `cursor:${key}`;
+    this._state[stateKey] = value;
+    this._persistState([stateKey]);
   }
 
   getState(key) {
@@ -441,7 +523,20 @@ class MailboxStore {
 
   setState(key, value) {
     this._state[key] = typeof value === 'string' ? value : JSON.stringify(value);
-    this._persistState();
+    this._persistState([key]);
+  }
+
+  setNodeSecretState({ secret = '', version = '', source = '', envSuppressed = '' } = {}) {
+    this._state.node_secret = secret;
+    this._state.node_secret_version = version ? String(version) : '';
+    this._state.node_secret_source = source;
+    this._state.node_secret_env_suppressed = envSuppressed;
+    this._persistState([
+      'node_secret',
+      'node_secret_version',
+      'node_secret_source',
+      'node_secret_env_suppressed',
+    ]);
   }
 
   // --- Compaction (reduces JSONL file size by rewriting only current state) ---
@@ -457,11 +552,16 @@ class MailboxStore {
     }
     entries.sort((a, b) => a.created_at - b.created_at);
 
+    const lines = entries.map((msg) => jsonlLineForWrite(msg));
+
     const fd = fs.openSync(tmpFile, 'w', PRIVATE_FILE_MODE);
-    for (const msg of entries) {
-      fs.writeSync(fd, JSON.stringify(msg) + '\n');
+    try {
+      for (const line of lines) {
+        fs.writeSync(fd, line);
+      }
+    } finally {
+      fs.closeSync(fd);
     }
-    fs.closeSync(fd);
     bestEffortChmod(tmpFile, PRIVATE_FILE_MODE);
     // Windows: renameSync throws EPERM when the destination already exists.
     // Remove it first so the swap succeeds on all platforms.

package/src/proxy/router/messages_route.js

@@ -144,8 +144,8 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
   return async ({ body, headers }) => {
     const inboundHeaders = headers || {};
     // x-api-key is satisfied by either the inbound header OR a proxy-side
-    // ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN env (token mediation, see
-    // _proxyAnthropic). The proxy server itself has already auth-checked
+    // EVOMAP_ANTHROPIC_API_KEY / ANTHROPIC_API_KEY / ANTHROPIC_AUTH_TOKEN env
+    // (token mediation, see _proxyAnthropic). The proxy server itself has already auth-checked
     // `Authorization: Bearer <proxy_token>` before reaching this handler.
     //
     // Bedrock upstream uses SigV4 with AWS_* env, so neither inbound
@@ -161,7 +161,8 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
     if (upstreamMode !== 'bedrock') {
       const hasInboundKey = !!inboundHeaders['x-api-key'];
       const hasProxyEnvCreds = !!(
-        process.env.ANTHROPIC_API_KEY
+        process.env.EVOMAP_ANTHROPIC_API_KEY
+        || process.env.ANTHROPIC_API_KEY
         || process.env.EVOMAP_ANTHROPIC_AUTH_TOKEN
         || (process.env.EVOMAP_PROXY_AUTO_INJECTED === '1' ? '' : process.env.ANTHROPIC_AUTH_TOKEN)
       );
@@ -289,6 +290,7 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
         inboundHeaders,
         upstreamMode,
       });
+      if (upstream.traceRequestBody !== undefined) trace?.setRequestBody(upstream.traceRequestBody);
     } catch (err) {
       trace?.record({ status: 502, error: err, upstreamMode, model: chosenModel });
       throw err;
@@ -351,6 +353,7 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
       // — but that loses the original 503 body, so cache it here too and
       // restore it on the throw path.
       let drainedFirst = '';
+      let firstAttemptResponseBody = {};
       if (upstream.text) {
         // Bound response-body drain to 10s to prevent hanging on large or
         // slow-streaming error responses. If the drain times out, log it
@@ -388,14 +391,47 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
           if (drainTimer) clearTimeout(drainTimer);
         }
       }
+      if (drainedFirst.length > 0) {
+        try {
+          firstAttemptResponseBody = JSON.parse(drainedFirst);
+        } catch {
+          firstAttemptResponseBody = { error: drainedFirst };
+        }
+      }
+      trace?.recordAttempt({
+        attempt_index: 0,
+        status: upstream.status,
+        requestBody: upstream.traceRequestBody !== undefined ? upstream.traceRequestBody : outboundBody,
+        responseBody: firstAttemptResponseBody,
+        upstreamMode,
+        model: chosenModel,
+        headers: upstream.headers,
+      });
+      let retryBody = body;
       try {
-        const retryBody = rewriteModel(body, originalModel);
+        retryBody = rewriteModel(body, originalModel);
         finalUpstream = await anthropicProxy('/v1/messages', retryBody, {
           inboundHeaders,
           upstreamMode,
         });
+        trace?.setRequestBody(finalUpstream.traceRequestBody !== undefined ? finalUpstream.traceRequestBody : retryBody);
+        trace?.recordAttempt({
+          attempt_index: 1,
+          requestBody: finalUpstream.traceRequestBody !== undefined ? finalUpstream.traceRequestBody : retryBody,
+          upstreamMode,
+          model: originalModel,
+          headers: finalUpstream.headers,
+        });
         finalModel = originalModel;
       } catch (err) {
+        trace?.recordAttempt({
+          attempt_index: 1,
+          status: 502,
+          requestBody: retryBody,
+          error: err,
+          upstreamMode,
+          model: originalModel,
+        });
         // Replay the drained first response so the caller still sees the
         // original 503 + body, not an empty stream.
         finalUpstream = {
@@ -424,6 +460,7 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
         upstreamMode,
         model: finalModel,
         headers: forwardHeaders,
+        attempt_index: finalModel === originalModel && chosenModel !== originalModel ? 1 : undefined,
       });
       return {
         status: finalUpstream.status,
@@ -460,6 +497,21 @@ function buildMessagesHandler({ anthropicProxy, logger, routerEnabled, traceStor
         }));
       }
     }
+    if (
+      enabled
+      && chosenModel
+      && chosenModel !== originalModel
+      && finalModel === originalModel
+    ) {
+      trace?.recordAttempt({
+        attempt_index: 1,
+        status: finalUpstream.status,
+        responseBody: respBody,
+        upstreamMode,
+        model: finalModel,
+        headers: finalUpstream.headers,
+      });
+    }
     trace?.record({
       status: finalUpstream.status,
       responseBody: respBody,