@evident-ai/cli 3.4.1-dev.4a15fbd → 3.4.1-dev.5210d05

package/dist/index.js

@@ -805,6 +805,7 @@ async function reportResourceUsage(agentId, authHeader, usage) {
       headers: { Authorization: authHeader, "Content-Type": "application/json" },
       body: JSON.stringify({
         cpu_percent: usage.cpuPercent,
+        cpu_peak_percent: usage.cpuPeakPercent,
         cpu_count: usage.cpuCount,
         memory_total_bytes: usage.memoryTotalBytes,
         memory_available_bytes: usage.memoryAvailableBytes,
@@ -1215,6 +1216,7 @@ var TelemetryEventTypes = {
 // ../../packages/types/src/tunnel/index.ts
 var MAX_FRAME_BYTES = 256 * 1024;
 var TUNNEL_DRAIN_PING_PATH = "/__evident/drain";
+var TUNNEL_USAGE_REARM_PING_PATH = "/__evident/usage-rearm";
 
 // ../../packages/types/src/runner-files.ts
 var MAX_FILE_PUSH_BYTES = 64 * 1024;
@@ -3150,6 +3152,44 @@ function findLastAssistantReplyFor(messages, userMessageId) {
   }
   return lastOk ?? last;
 }
+function collectSubagentSessions(messages, userMessageId) {
+  if (!messages || messages.length === 0) return [];
+  const byParent = messages.filter(
+    (message) => roleOf(message) === "assistant" && parentIdOf(message) === userMessageId
+  );
+  const assistants = byParent.length > 0 ? byParent : [];
+  if (assistants.length === 0) {
+    const userIndex = messages.findIndex((message) => idOf(message) === userMessageId);
+    if (userIndex === -1) return [];
+    for (let i = userIndex + 1; i < messages.length; i++) {
+      const message = messages[i];
+      if (roleOf(message) === "user") break;
+      if (roleOf(message) === "assistant") assistants.push(message);
+    }
+  }
+  const refs = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const message of assistants) {
+    const parts = Array.isArray(message.parts) ? message.parts : [];
+    for (const part of parts) {
+      if (!part || typeof part !== "object" || part.type !== "tool" || part.tool !== "task")
+        continue;
+      const state = part.state;
+      if (!state || typeof state !== "object") continue;
+      const metadata = state.metadata;
+      if (!metadata || typeof metadata !== "object") continue;
+      const sessionId = metadata.sessionId;
+      if (typeof sessionId !== "string" || sessionId.length === 0 || seen.has(sessionId)) continue;
+      seen.add(sessionId);
+      const start = state.time?.start;
+      refs.push({
+        sessionId,
+        startedAtMs: typeof start === "number" && Number.isFinite(start) ? start : null
+      });
+    }
+  }
+  return refs;
+}
 function messageUsage(messages, userMessageId) {
   if (!messages || messages.length === 0) return null;
   const byParentAll = messages.filter(
@@ -3278,8 +3318,7 @@ function isAbortedTerminalReply(messages, userMessageId) {
   }
   return false;
 }
-function messageFailure(messages, userMessageId) {
-  const reply = findLastAssistantReplyFor(messages, userMessageId);
+function classifyReplyAuthError(reply) {
   const error2 = errorOf(reply);
   if (error2 == null || typeof error2 !== "object") return null;
   const e = error2;
@@ -3304,6 +3343,32 @@ function messageFailure(messages, userMessageId) {
   }
   return null;
 }
+function messageFailure(messages, userMessageId) {
+  return classifyReplyAuthError(findLastAssistantReplyFor(messages, userMessageId));
+}
+function findLatestSubagentAuthOutcome(messages, sinceMs) {
+  if (!messages || messages.length === 0) return null;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const message = messages[i];
+    if (roleOf(message) !== "assistant") continue;
+    const created = createdOf(message);
+    if (sinceMs !== null && typeof created === "number" && created < sinceMs) continue;
+    const failure = classifyReplyAuthError(message);
+    if (failure) {
+      if (!failure.providerId) return null;
+      return { providerId: failure.providerId, outcome: "failed", failure };
+    }
+    const providerId = message.info?.providerID;
+    if (errorOf(message) == null && typeof providerId === "string" && providerId.length > 0) {
+      return { providerId, outcome: "succeeded" };
+    }
+    return null;
+  }
+  return null;
+}
+function findSubagentAuthOutcome(messages, sinceMs) {
+  return findLatestSubagentAuthOutcome(messages, sinceMs);
+}
 function applyZeroProviderFallback(classified, hasConfiguredProvider, replyProviderId, replyModelId = null) {
   if (classified != null) return classified;
   if (hasConfiguredProvider !== false) return null;
@@ -3634,7 +3699,6 @@ var StreamForwarder = class {
   handleFrame(frame) {
     switch (frame.type) {
       case "open":
-        this.callbacks.onOpen?.(frame.sid, frame.method, frame.path);
         void this.handleOpen(frame);
         break;
       case "req_data":
@@ -3670,12 +3734,21 @@ var StreamForwarder = class {
     const { sid, method, path, headers, has_body } = frame;
     const correlationId = headers?.[CORRELATION_ID_HEADER];
     const startedAt = Date.now();
+    if (path !== TUNNEL_DRAIN_PING_PATH && path !== TUNNEL_USAGE_REARM_PING_PATH) {
+      this.callbacks.onOpen?.(sid, method, path);
+    }
     if (path === TUNNEL_DRAIN_PING_PATH) {
       this.callbacks.onDrainPing?.();
       this.send({ type: "head", sid, status: 204, headers: {} });
       this.send({ type: "res_end", sid });
       return;
     }
+    if (path === TUNNEL_USAGE_REARM_PING_PATH) {
+      this.callbacks.onUsageRearmPing?.();
+      this.send({ type: "head", sid, status: 204, headers: {} });
+      this.send({ type: "res_end", sid });
+      return;
+    }
     if (process.env.DEBUG) {
       log("debug", "agent_request", {
         correlation_id: correlationId,
@@ -3820,7 +3893,8 @@ function connectTunnel(options) {
     onResponse,
     onInfo,
     onWarning,
-    onDrainPing
+    onDrainPing,
+    onUsageRearmPing
   } = options;
   const tunnelUrl = getTunnelUrlConfig();
   const url = `${tunnelUrl}/tunnel/${agentId}/connect`;
@@ -3832,7 +3906,8 @@ function connectTunnel(options) {
     });
     const forwarder = new StreamForwarder(ws, port, {
       onHead: () => onResponse?.(),
-      onDrainPing: () => onDrainPing?.()
+      onDrainPing: () => onDrainPing?.(),
+      onUsageRearmPing: () => onUsageRearmPing?.()
     });
     const connectionTimeout = setTimeout(() => {
       ws.close();
@@ -3993,6 +4068,7 @@ var RunnerConnection = class {
           onError: (error2) => events.onError?.(error2),
           onResponse: () => events.onResponse?.(),
           onDrainPing: () => events.onDrainPing?.(),
+          onUsageRearmPing: () => events.onUsageRearmPing?.(),
           onInfo: (message) => events.onInfo?.(message),
           onWarning: (message) => events.onWarning?.(message)
         });
@@ -4437,45 +4513,84 @@ function readDisk(homeDir) {
     };
   }
 }
-function createResourceUsageCollector(homeDir) {
-  let previous = readCpuSample();
-  return async () => {
+var CPU_PEAK_WINDOW_MS = 6e4;
+var CPU_PEAK_WINDOW_SAMPLE_COUNT = 4;
+var CPU_PEAK_SAMPLE_INTERVAL_MS = CPU_PEAK_WINDOW_MS / CPU_PEAK_WINDOW_SAMPLE_COUNT;
+function createCpuPeakSampler() {
+  const sampleHistory = new Array(CPU_PEAK_WINDOW_SAMPLE_COUNT);
+  sampleHistory[0] = readCpuSample();
+  let nextSampleIndex = 1;
+  let sampleCount = 1;
+  let peak = null;
+  const timer = setInterval(() => {
     const current = readCpuSample();
-    const hostCpuPercent = cpuPercentBetween(previous, current);
-    const hostCpuCount = cpus().length;
-    previous = current;
-    const disk = readDisk(homeDir);
-    const opencodeDbBytes = statSessionDbBytes(homeDir);
-    const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
-    const warnings = [];
-    if (disk.warning) warnings.push(disk.warning);
-    if (ecsWarning) warnings.push(ecsWarning);
-    let cpuPercent = hostCpuPercent;
-    let cpuCount = hostCpuCount;
-    let memoryTotalBytes = totalmem();
-    let memoryAvailableBytes = freemem();
-    if (limits !== null) {
-      cpuCount = limits.cpuCount;
-      memoryTotalBytes = limits.memoryTotalBytes;
-      memoryAvailableBytes = clamp(
-        limits.memoryTotalBytes - (totalmem() - freemem()),
-        0,
-        limits.memoryTotalBytes
-      );
-      cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
+    const sampleFromWindowAgo = sampleCount === CPU_PEAK_WINDOW_SAMPLE_COUNT ? sampleHistory[nextSampleIndex] : void 0;
+    if (sampleFromWindowAgo !== void 0) {
+      const percentage = cpuPercentBetween(sampleFromWindowAgo, current);
+      if (percentage !== null) {
+        peak = peak === null ? percentage : Math.max(peak, percentage);
+      }
     }
-    return {
-      usage: {
-        cpuPercent,
-        cpuCount,
-        memoryTotalBytes,
-        memoryAvailableBytes,
-        diskTotalBytes: disk.totalBytes,
-        diskFreeBytes: disk.freeBytes,
-        opencodeDbBytes
-      },
-      warnings
-    };
+    sampleHistory[nextSampleIndex] = current;
+    nextSampleIndex = (nextSampleIndex + 1) % CPU_PEAK_WINDOW_SAMPLE_COUNT;
+    sampleCount = Math.min(sampleCount + 1, CPU_PEAK_WINDOW_SAMPLE_COUNT);
+  }, CPU_PEAK_SAMPLE_INTERVAL_MS);
+  return {
+    takeAndReset: () => {
+      const currentPeak = peak;
+      peak = null;
+      return currentPeak;
+    },
+    stop: () => clearInterval(timer)
+  };
+}
+function createResourceUsageCollector(homeDir) {
+  let previous = readCpuSample();
+  const cpuPeakSampler = createCpuPeakSampler();
+  return {
+    collect: async () => {
+      const current = readCpuSample();
+      const hostCpuPercent = cpuPercentBetween(previous, current);
+      const hostCpuPeakPercent = cpuPeakSampler.takeAndReset();
+      const hostCpuCount = cpus().length;
+      previous = current;
+      const disk = readDisk(homeDir);
+      const opencodeDbBytes = statSessionDbBytes(homeDir);
+      const { limits, warning: ecsWarning } = await readEcsTaskLimits(process.env);
+      const warnings = [];
+      if (disk.warning) warnings.push(disk.warning);
+      if (ecsWarning) warnings.push(ecsWarning);
+      let cpuPercent = hostCpuPercent;
+      let cpuPeakPercent = hostCpuPeakPercent;
+      let cpuCount = hostCpuCount;
+      let memoryTotalBytes = totalmem();
+      let memoryAvailableBytes = freemem();
+      if (limits !== null) {
+        cpuCount = limits.cpuCount;
+        memoryTotalBytes = limits.memoryTotalBytes;
+        memoryAvailableBytes = clamp(
+          limits.memoryTotalBytes - (totalmem() - freemem()),
+          0,
+          limits.memoryTotalBytes
+        );
+        cpuPercent = hostCpuPercent === null ? null : clamp(round2(hostCpuPercent * hostCpuCount / limits.cpuCount), 0, 100);
+        cpuPeakPercent = hostCpuPeakPercent === null ? null : clamp(round2(hostCpuPeakPercent * hostCpuCount / limits.cpuCount), 0, 100);
+      }
+      return {
+        usage: {
+          cpuPercent,
+          cpuPeakPercent,
+          cpuCount,
+          memoryTotalBytes,
+          memoryAvailableBytes,
+          diskTotalBytes: disk.totalBytes,
+          diskFreeBytes: disk.freeBytes,
+          opencodeDbBytes
+        },
+        warnings
+      };
+    },
+    stop: cpuPeakSampler.stop
   };
 }
 
@@ -6062,6 +6177,9 @@ var ChannelDriver = class _ChannelDriver {
         });
         await this.markFailed(conv.id, message.id, sessionId, error2, usage, failure);
       }
+      if (ocId !== null) {
+        await this.reportSubagentAuthFailures(conv.id, ocId, message.id, messages);
+      }
     } catch (err) {
       if (err instanceof ChannelAuthError) throw err;
       this.log({
@@ -7026,6 +7144,12 @@ var ChannelDriver = class _ChannelDriver {
           return;
         }
         inFlight.done = true;
+        await this.reportSubagentAuthFailures(
+          watcher.conv.id,
+          inFlight.opencodeMessageId,
+          inFlight.evidentMessageId,
+          messages
+        );
       }
       this.removeInFlight(watcher, inFlight.evidentMessageId);
       return;
@@ -7268,6 +7392,12 @@ var ChannelDriver = class _ChannelDriver {
         return;
       }
       inFlight.done = true;
+      await this.reportSubagentAuthFailures(
+        watcher.conv.id,
+        inFlight.opencodeMessageId,
+        inFlight.evidentMessageId,
+        messages
+      );
     }
     this.removeInFlight(watcher, inFlight.evidentMessageId);
   }
@@ -7438,6 +7568,7 @@ var ChannelDriver = class _ChannelDriver {
         });
         return;
       }
+      await this.reportSubagentAuthFailures(row.conversation_id, ocId ?? "", row.id, messages);
       this.dontRedispatch.delete(row.id);
       void this.postSignal(row.conversation_id, row.id, "readopt_failed");
       return;
@@ -7599,6 +7730,9 @@ var ChannelDriver = class _ChannelDriver {
       });
       return;
     }
+    if (ocId !== null) {
+      await this.reportSubagentAuthFailures(row.conversation_id, ocId, row.id, messages);
+    }
     this.dontRedispatch.delete(row.id);
     void this.postSignal(row.conversation_id, row.id, "readopt_done");
   }
@@ -8550,6 +8684,111 @@ var ChannelDriver = class _ChannelDriver {
       reply?.info?.modelID ?? null
     );
   }
+  async recordSubagentModelAuthFailure(failure, conversationId, messageId) {
+    const providerId = failure.providerId ?? "(unknown)";
+    try {
+      const res = await this.fetchImpl(
+        `${this.apiUrl}/runners/${this.agentId}/model-auth-failures`,
+        {
+          method: "POST",
+          headers: { Authorization: this.getAuthHeader(), "Content-Type": "application/json" },
+          body: JSON.stringify({
+            provider_id: failure.providerId,
+            model_id: failure.modelId,
+            reason: failure.reason
+          })
+        }
+      );
+      if (!res.ok) {
+        this.log({
+          level: "warn",
+          message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
+          conversation_id: conversationId,
+          message_id: messageId
+        });
+      }
+    } catch (err) {
+      this.log({
+        level: "warn",
+        message: `Sub-agent model-auth report for provider ${providerId} (${failure.reason}) failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conversationId,
+        message_id: messageId
+      });
+    }
+  }
+  async clearSubagentModelAuthFailure(providerId, conversationId, messageId) {
+    try {
+      const res = await this.fetchImpl(
+        `${this.apiUrl}/runners/${this.agentId}/model-auth-failures/${encodeURIComponent(providerId)}`,
+        {
+          method: "DELETE",
+          headers: { Authorization: this.getAuthHeader() }
+        }
+      );
+      if (!res.ok) {
+        this.log({
+          level: "warn",
+          message: `Sub-agent model-auth clear for provider ${providerId} returned HTTP ${res.status} (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)})`,
+          conversation_id: conversationId,
+          message_id: messageId
+        });
+      }
+    } catch (err) {
+      this.log({
+        level: "warn",
+        message: `Sub-agent model-auth clear for provider ${providerId} failed with status network-error (conversation ${conversationId.slice(0, 8)}, message ${messageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
+        conversation_id: conversationId,
+        message_id: messageId
+      });
+    }
+  }
+  async reportSubagentAuthFailures(conversationId, opencodeMessageId, evidentMessageId, messages) {
+    const refs = collectSubagentSessions(messages, opencodeMessageId);
+    if (refs.length === 0) return;
+    const failedProviders = /* @__PURE__ */ new Map();
+    const succeededProviders = /* @__PURE__ */ new Set();
+    for (const ref of refs) {
+      try {
+        const childMessages = await getSessionMessages(this.port, ref.sessionId);
+        if (childMessages === null) {
+          this.log({
+            level: "debug",
+            message: `Could not read sub-agent session ${ref.sessionId} while checking credential failures \u2014 skipping it`,
+            conversation_id: conversationId,
+            message_id: evidentMessageId
+          });
+          continue;
+        }
+        const outcome = findSubagentAuthOutcome(childMessages, ref.startedAtMs);
+        if (!outcome) continue;
+        if (outcome.outcome === "failed") {
+          failedProviders.set(outcome.providerId, outcome.failure);
+        } else {
+          succeededProviders.add(outcome.providerId);
+        }
+      } catch (err) {
+        this.log({
+          level: "warn",
+          message: `Failed to inspect sub-agent session ${ref.sessionId} for credential failures (conversation ${conversationId.slice(0, 8)}, message ${evidentMessageId.slice(0, 8)}): ${err instanceof Error ? err.message : String(err)}`,
+          conversation_id: conversationId,
+          message_id: evidentMessageId
+        });
+      }
+    }
+    for (const [providerId, failure] of failedProviders) {
+      this.log({
+        level: "warn",
+        message: `Sub-agent turn failed on provider ${providerId} (${failure.reason}) \u2014 recording credential evidence`,
+        conversation_id: conversationId,
+        message_id: evidentMessageId
+      });
+      await this.recordSubagentModelAuthFailure(failure, conversationId, evidentMessageId);
+    }
+    for (const providerId of succeededProviders) {
+      if (failedProviders.has(providerId)) continue;
+      await this.clearSubagentModelAuthFailure(providerId, conversationId, evidentMessageId);
+    }
+  }
   /**
    * Best-effort, SINGLE-ATTEMPT runner→server telemetry ping
    * (queued-followup-redrive). `POST .../messages/:id/signal {signal, ...extra}`
@@ -10080,7 +10319,8 @@ function scheduleResourceUsageReporting(state, options) {
     });
     return;
   }
-  const collect = createResourceUsageCollector(homedir5());
+  const { collect, stop } = createResourceUsageCollector(homedir5());
+  state.stopResourceUsageSampling = stop;
   let consecutiveFailures = 0;
   const tick = async () => {
     try {
@@ -10190,6 +10430,8 @@ async function cleanup(state, opts = {}) {
     clearTimeout(state.resourceUsageTimer);
     state.resourceUsageTimer = null;
   }
+  state.stopResourceUsageSampling?.();
+  state.stopResourceUsageSampling = null;
   const credentialSync = state.credentialSync;
   const flushCredentials = credentialSync ? async (phase, publish) => {
     await timeShutdownPhase(state, durations, phase, async () => {
@@ -10357,6 +10599,7 @@ async function run(options) {
     openaiUsageTimer: null,
     openaiUsageRearm: null,
     resourceUsageTimer: null,
+    stopResourceUsageSampling: null,
     credentialSync: null,
     authHeader: ""
   };
@@ -10917,6 +11160,18 @@ async function run(options) {
             if (state.interactive) displayStatus(state);
           });
         },
+        // Both loops are rearmed because `rearm()` is idempotent for the
+        // provider that did not just connect, and is a no-op when reporting is off.
+        onUsageRearmPing: () => {
+          if (!state.running) return;
+          logActivity(state, {
+            type: "info",
+            level: "debug",
+            message: "Usage rearm ping received"
+          });
+          state.claudeUsageRearm?.();
+          state.openaiUsageRearm?.();
+        },
         onInfo: (message) => logActivity(state, { type: "info", message })
       }
     });