@prestyj/agent 5.23.0 → 5.25.0

package/dist/index.js

@@ -346,6 +346,35 @@ function closeIterator(iterator) {
   Promise.resolve(iterator.return()).catch(() => {
   });
 }
+var PREFILL_TIMEOUT_MS_PER_1K_TOKENS = 640;
+var STREAM_FIRST_EVENT_TIMEOUT_MS = 45e3;
+var STREAM_FIRST_EVENT_TIMEOUT_MAX_MS = 18e4;
+var STREAM_FIRST_EVENT_TIMEOUT_SCALE_MIN_TOKENS = 2e4;
+function scaledFirstEventTimeoutMs(promptTokens) {
+  if (promptTokens < STREAM_FIRST_EVENT_TIMEOUT_SCALE_MIN_TOKENS) return null;
+  return Math.min(
+    STREAM_FIRST_EVENT_TIMEOUT_MAX_MS,
+    STREAM_FIRST_EVENT_TIMEOUT_MS + promptTokens / 1e3 * PREFILL_TIMEOUT_MS_PER_1K_TOKENS
+  );
+}
+var CACHE_HEALTH_MIN_PROMPT_TOKENS = 4e4;
+var CACHE_HEALTH_LOW_RATIO = 0.5;
+function assessCacheHealth(usage) {
+  const input = usage.inputTokens ?? 0;
+  const cacheRead = usage.cacheRead ?? 0;
+  const cacheWrite = usage.cacheWrite ?? 0;
+  const promptTokens = input + cacheRead + cacheWrite;
+  if (promptTokens < CACHE_HEALTH_MIN_PROMPT_TOKENS) {
+    return { ratio: null, promptTokens, cacheRead, low: false };
+  }
+  const ratio = cacheRead / promptTokens;
+  return {
+    ratio,
+    promptTokens,
+    cacheRead,
+    low: ratio < CACHE_HEALTH_LOW_RATIO
+  };
+}
 async function* agentLoop(messages, options) {
   const maxTurns = options.maxTurns ?? DEFAULT_MAX_TURNS;
   let effectiveMaxTurns = maxTurns;
@@ -387,6 +416,7 @@ async function* agentLoop(messages, options) {
   let providerCalls = 0;
   let nonStreamingCalls = 0;
   let warnedNonStreaming = false;
+  let warnedPromptCacheMiss = false;
   const MAX_OUTPUT_CEILING_RETRIES = 1;
   let outputCeilingRetries = 0;
   const ceilingKey = outputRouteKey({
@@ -396,7 +426,6 @@ async function* agentLoop(messages, options) {
   });
   const OVERLOAD_BASE_DELAY_MS = 2e3;
   const OVERLOAD_MAX_DELAY_MS = 3e4;
-  const STREAM_FIRST_EVENT_TIMEOUT_MS = 45e3;
   const STREAM_IDLE_TIMEOUT_MS = 9e4;
   const STREAM_HARD_TIMEOUT_MS = 9e4;
   const STREAM_OUTPUT_HARD_TIMEOUT_MS = 3e5;
@@ -405,8 +434,8 @@ async function* agentLoop(messages, options) {
   const NON_STREAMING_HARD_TIMEOUT_MS = 3e5;
   const usesSilentReasoningBudget = options.provider === "sakana" || options.provider === "openai" && options.thinking != null;
   const localBackend = isLocalBackendUrl(options.baseUrl);
-  const firstEventTimeoutMs = localBackend ? Number.POSITIVE_INFINITY : usesSilentReasoningBudget ? STREAM_THINKING_IDLE_TIMEOUT_MS : STREAM_FIRST_EVENT_TIMEOUT_MS;
-  const initialHardTimeoutMs = localBackend || usesSilentReasoningBudget ? STREAM_THINKING_HARD_TIMEOUT_MS : STREAM_HARD_TIMEOUT_MS;
+  const baseFirstEventTimeoutMs = localBackend ? Number.POSITIVE_INFINITY : usesSilentReasoningBudget ? STREAM_THINKING_IDLE_TIMEOUT_MS : STREAM_FIRST_EVENT_TIMEOUT_MS;
+  const baseHardTimeoutMs = localBackend || usesSilentReasoningBudget ? STREAM_THINKING_HARD_TIMEOUT_MS : STREAM_HARD_TIMEOUT_MS;
   const MAX_TOOLCALL_DELTA_CHARS = 1e6;
   const MAX_TOOLCALL_NO_PROGRESS_EVENTS = 2e4;
   let logicalTurnStartedAt = 0;
@@ -418,17 +447,28 @@ async function* agentLoop(messages, options) {
       turn++;
       if (logicalTurnStartedAt === 0) logicalTurnStartedAt = Date.now();
       toolMap = new Map((options.tools ?? []).map((t) => [t.name, t]));
-      if (_diagFn) {
-        let msgChars = 0;
-        for (const m of messages) {
-          if (typeof m.content === "string") msgChars += m.content.length;
-          else if (Array.isArray(m.content)) {
-            for (const p of m.content) {
-              if ("text" in p && typeof p.text === "string") msgChars += p.text.length;
-              if ("content" in p && typeof p.content === "string") msgChars += p.content.length;
-            }
+      let msgChars = 0;
+      for (const m of messages) {
+        if (typeof m.content === "string") msgChars += m.content.length;
+        else if (Array.isArray(m.content)) {
+          for (const p of m.content) {
+            if ("text" in p && typeof p.text === "string") msgChars += p.text.length;
+            if ("content" in p && typeof p.content === "string") msgChars += p.content.length;
           }
         }
+      }
+      let firstEventTimeoutMs;
+      let initialHardTimeoutMs;
+      if (baseFirstEventTimeoutMs === STREAM_FIRST_EVENT_TIMEOUT_MS) {
+        const promptTokens = Math.ceil(msgChars / 3);
+        const scaled = scaledFirstEventTimeoutMs(promptTokens);
+        firstEventTimeoutMs = scaled ?? baseFirstEventTimeoutMs;
+        initialHardTimeoutMs = Math.max(baseHardTimeoutMs, firstEventTimeoutMs + 3e4);
+      } else {
+        firstEventTimeoutMs = baseFirstEventTimeoutMs;
+        initialHardTimeoutMs = baseHardTimeoutMs;
+      }
+      if (_diagFn) {
         diag("turn_start", {
           turn,
           messages: messages.length,
@@ -1051,6 +1091,27 @@ async function* agentLoop(messages, options) {
       if (response.usage.cacheWrite) {
         totalUsage.cacheWrite = (totalUsage.cacheWrite ?? 0) + response.usage.cacheWrite;
       }
+      const cacheHealth = assessCacheHealth(response.usage);
+      if (cacheHealth.ratio !== null) {
+        diag("cache_health", {
+          promptTokens: cacheHealth.promptTokens,
+          cacheRead: cacheHealth.cacheRead,
+          ratio: Math.round(cacheHealth.ratio * 100) / 100,
+          provider: options.provider,
+          model: options.model
+        });
+        if (cacheHealth.low && !warnedPromptCacheMiss) {
+          warnedPromptCacheMiss = true;
+          diag("prompt_cache_miss", {
+            promptTokens: cacheHealth.promptTokens,
+            cacheRead: cacheHealth.cacheRead,
+            ratio: Math.round(cacheHealth.ratio * 100) / 100,
+            provider: options.provider,
+            model: options.model,
+            impact: "large prompts are consistently served mostly uncached \u2014 every turn re-prefills the whole context; if this persists, lower the provider's compaction latency cap (resolveCompactionPolicy)"
+          });
+        }
+      }
       if (!emptyExhausted) {
         messages.push(response.message);
         latestProviderUsage = response.usage;