@agent-native/core 0.153.4 → 0.153.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/templates/analytics/actions/account-deep-dive.ts +1 -0
- package/corpus/templates/analytics/actions/bigquery-query.ts +1 -0
- package/corpus/templates/analytics/actions/bigquery.ts +1 -0
- package/corpus/templates/analytics/actions/content-calendar-schema.ts +1 -0
- package/corpus/templates/analytics/actions/content-calendar.ts +1 -0
- package/corpus/templates/analytics/actions/gcloud.ts +1 -0
- package/corpus/templates/analytics/actions/get-data-program.ts +1 -0
- package/corpus/templates/analytics/actions/get-error-issue.ts +1 -0
- package/corpus/templates/analytics/actions/get-monitor-stats.ts +1 -0
- package/corpus/templates/analytics/actions/get-monitor.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-events.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-summary.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-timeline.ts +1 -0
- package/corpus/templates/analytics/actions/gong-calls.ts +1 -0
- package/corpus/templates/analytics/actions/gong-native-insights.ts +1 -0
- package/corpus/templates/analytics/actions/grafana.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-deals.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-metrics.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-pipelines.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-records.ts +1 -0
- package/corpus/templates/analytics/actions/jira-search.ts +1 -0
- package/corpus/templates/analytics/actions/jira.ts +1 -0
- package/corpus/templates/analytics/actions/list-error-issues.ts +1 -0
- package/corpus/templates/analytics/actions/list-monitors.ts +1 -0
- package/corpus/templates/analytics/actions/list-session-recordings.ts +1 -0
- package/corpus/templates/analytics/actions/match-error-issues.ts +1 -0
- package/corpus/templates/analytics/actions/prometheus.ts +1 -0
- package/corpus/templates/analytics/actions/pylon-issues.ts +1 -0
- package/corpus/templates/analytics/actions/query-agent-native-analytics.ts +1 -0
- package/corpus/templates/analytics/actions/query-dashboard-panel.ts +1 -0
- package/corpus/templates/analytics/actions/query-inbound-forms.ts +1 -0
- package/corpus/templates/analytics/actions/run-monitor-check.ts +1 -0
- package/corpus/templates/analytics/actions/sentry.ts +1 -0
- package/corpus/templates/analytics/actions/seo-blog-pages.ts +1 -0
- package/corpus/templates/analytics/actions/seo-page-keywords.ts +1 -0
- package/corpus/templates/analytics/actions/seo-top-keywords.ts +1 -0
- package/corpus/templates/analytics/actions/slack-messages.ts +1 -0
- package/corpus/templates/analytics/actions/stripe.ts +1 -0
- package/corpus/templates/analytics/actions/test-custom-api-connection.ts +1 -0
- package/corpus/templates/analytics/server/lib/real-data-actions.ts +38 -47
- package/corpus/templates/analytics/server/plugins/agent-chat.ts +40 -34
- package/dist/action.d.ts +13 -0
- package/dist/action.js +3 -0
- package/dist/agent/engine/tool-call-journal-seed.d.ts +27 -10
- package/dist/agent/engine/tool-call-journal-seed.js +46 -32
- package/dist/agent/production-agent.d.ts +33 -0
- package/dist/agent/production-agent.js +172 -57
- package/dist/agent/run-loop-with-resume.d.ts +1 -1
- package/dist/agent/run-loop-with-resume.js +49 -35
- package/dist/client/agent-chat-adapter.js +204 -273
- package/dist/client/sse-event-processor.d.ts +7 -0
- package/dist/client/sse-event-processor.js +21 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/observability/routes.d.ts +5 -5
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/actions/github-repo-files.js +1 -0
- package/dist/provider-api/actions/provider-api.js +1 -0
- package/dist/provider-api/actions/staged-datasets.js +1 -0
- package/dist/provider-api/corpus-jobs.js +1 -0
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +9 -9
- package/dist/server/action-discovery.js +2 -0
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/package.json +3 -3
|
@@ -6,7 +6,7 @@ import { getOrCreateAnalyticsSessionId } from "./analytics-session.js";
|
|
|
6
6
|
import { captureError } from "./analytics.js";
|
|
7
7
|
import { agentNativePath } from "./api-path.js";
|
|
8
8
|
import { formatChatErrorText, normalizeChatError } from "./error-format.js";
|
|
9
|
-
import { AgentAutoContinueSignal, appendMissingFinalResponseWarning, readSSEStream, settleInterruptedToolCalls, } from "./sse-event-processor.js";
|
|
9
|
+
import { AgentAutoContinueSignal, INTERRUPTED_TOOL_RESULT, appendMissingFinalResponseWarning, readSSEStream, settleInterruptedToolCalls, } from "./sse-event-processor.js";
|
|
10
10
|
const TEXT_ATTACHMENT_CONTENT_TYPES = new Set([
|
|
11
11
|
"application/json",
|
|
12
12
|
"application/x-ndjson",
|
|
@@ -24,37 +24,20 @@ const AUTO_CONTINUE_COMPLETION_GUARD = "Before doing more work, inspect the prio
|
|
|
24
24
|
const MAX_RECONNECT_ATTEMPTS = 5;
|
|
25
25
|
const MAX_STARTUP_RECOVERY_ATTEMPTS = 8;
|
|
26
26
|
const MAX_QUEUED_CONFLICT_RETRIES = 120;
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
// How many consecutive continuations that did not advance the turn (see
|
|
28
|
+
// `describeContinuationAdvance`) we tolerate before giving up. This replaced
|
|
29
|
+
// five separate budgets — stale-run, stalled, empty, repeated-narration and
|
|
30
|
+
// repeated-action-preparation — each of which existed because the rung above
|
|
31
|
+
// it had a hole the next one patched. One question answers all five: a round
|
|
32
|
+
// that produced nothing, or produced exactly what the round before it
|
|
33
|
+
// produced, did not advance.
|
|
34
|
+
const MAX_NON_ADVANCING_CONTINUATIONS = 3;
|
|
29
35
|
// Ceiling across the whole turn. Every attempt re-POSTs the full history plus
|
|
30
|
-
// attachments, so a high ceiling only burns tokens and wall time.
|
|
31
|
-
// turns are already stopped far earlier by the
|
|
32
|
-
//
|
|
33
|
-
//
|
|
36
|
+
// attachments, so a high ceiling only burns tokens and wall time. Non-advancing
|
|
37
|
+
// turns are already stopped far earlier by the budget above; this one only ever
|
|
38
|
+
// binds turns that keep making real progress, so it stays just above the
|
|
39
|
+
// longest recovery we deliberately support.
|
|
34
40
|
const MAX_TOTAL_TRANSIENT_CONTINUATIONS = 12;
|
|
35
|
-
// How many consecutive continuations that produce NO progress (no streamed
|
|
36
|
-
// text, no completed/in-flight tool) we tolerate before giving up. A complex
|
|
37
|
-
// first turn can spend the whole soft-timeout window (~40s) "thinking" with no
|
|
38
|
-
// visible output; giving up after a single such window made the agent feel like
|
|
39
|
-
// it "craps out / stops midway" on heavier prompts (Analytics). Retrying a few
|
|
40
|
-
// times lets a transient slow start recover, while the cap still terminates a
|
|
41
|
-
// genuinely stuck turn with a clear message instead of looping forever.
|
|
42
|
-
const MAX_EMPTY_TRANSIENT_CONTINUATIONS = 3;
|
|
43
|
-
// How many consecutive continuations that re-stream the SAME narration without
|
|
44
|
-
// advancing (no in-flight tool, no completed tool) we tolerate before giving
|
|
45
|
-
// up. A model that degenerates into repeating one phrase ("I have the full
|
|
46
|
-
// HTML. Creating the extension now!") emits "new" text every continuation,
|
|
47
|
-
// which keeps resetting the stalled/empty budgets — so without this guard the
|
|
48
|
-
// stuck turn burns the entire MAX_TOTAL_TRANSIENT_CONTINUATIONS budget (each
|
|
49
|
-
// round re-sending any large pasted payload) before bailing. Catching the
|
|
50
|
-
// repeat ends it in a few rounds with a clear, actionable message instead.
|
|
51
|
-
const MAX_REPEATED_TRANSIENT_CONTINUATIONS = 3;
|
|
52
|
-
// How many consecutive continuations that only reach the SAME "preparing
|
|
53
|
-
// action" activity card we tolerate before giving up. This catches runs that
|
|
54
|
-
// keep timing out while assembling a large tool payload: they are not empty,
|
|
55
|
-
// and the narration may vary enough to bypass the text-repeat guard, but the
|
|
56
|
-
// real tool never starts.
|
|
57
|
-
const MAX_REPEATED_ACTION_PREPARATION_CONTINUATIONS = 3;
|
|
58
41
|
const RETRY_BASE_DELAY_MS = 500;
|
|
59
42
|
const RETRY_MAX_DELAY_MS = 8_000;
|
|
60
43
|
const MAX_HISTORY_ATTACHMENT_CHARS = 60_000;
|
|
@@ -636,14 +619,25 @@ function contentToStructuredMessages(content, nextToolCallId, options) {
|
|
|
636
619
|
: (part.args ?? {}),
|
|
637
620
|
});
|
|
638
621
|
if (part.result !== undefined) {
|
|
622
|
+
const body = truncate
|
|
623
|
+
? truncateForHistory(toolResultContent(part.result, part.toolName), MAX_HISTORY_TOOL_RESULT_CHARS, "Tool result")
|
|
624
|
+
: toolResultContent(part.result, part.toolName);
|
|
639
625
|
pendingToolResults.push({
|
|
640
626
|
type: "tool-result",
|
|
641
627
|
toolCallId,
|
|
642
628
|
toolName: part.toolName,
|
|
643
629
|
toolInput: JSON.stringify(part.args ?? {}),
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
630
|
+
// A settled-interrupted tool has a result whose outcome is UNKNOWN.
|
|
631
|
+
// It is neither a success nor a failure, so it carries the marker the
|
|
632
|
+
// server's write-interruption breaker matches on instead of claiming
|
|
633
|
+
// either.
|
|
634
|
+
content: part.outcome === "unknown" &&
|
|
635
|
+
!body.includes(INTERRUPTED_TOOL_RESULT)
|
|
636
|
+
? `${INTERRUPTED_TOOL_RESULT} ${body}`
|
|
637
|
+
: body,
|
|
638
|
+
...(part.isError === true && part.outcome !== "unknown"
|
|
639
|
+
? { isError: true }
|
|
640
|
+
: {}),
|
|
647
641
|
});
|
|
648
642
|
}
|
|
649
643
|
else {
|
|
@@ -652,7 +646,7 @@ function contentToStructuredMessages(content, nextToolCallId, options) {
|
|
|
652
646
|
toolCallId,
|
|
653
647
|
toolName: part.toolName,
|
|
654
648
|
toolInput: JSON.stringify(part.args ?? {}),
|
|
655
|
-
content:
|
|
649
|
+
content: INTERRUPTED_TOOL_RESULT,
|
|
656
650
|
});
|
|
657
651
|
}
|
|
658
652
|
}
|
|
@@ -712,6 +706,14 @@ function assistantUiMessagesToStructuredHistory(messages) {
|
|
|
712
706
|
...(part.result !== undefined
|
|
713
707
|
? { result: toolResultContent(part.result, toolName) }
|
|
714
708
|
: {}),
|
|
709
|
+
// A failed call replayed without `isError` reads to the next turn as
|
|
710
|
+
// an ordinary success whose body happens to contain error prose, so
|
|
711
|
+
// the model cannot tell it already tried this and failed. The
|
|
712
|
+
// server's repeat-error breaker keys on exactly this flag.
|
|
713
|
+
...(part.isError === true ? { isError: true } : {}),
|
|
714
|
+
...(part.outcome === "unknown"
|
|
715
|
+
? { outcome: "unknown" }
|
|
716
|
+
: {}),
|
|
715
717
|
});
|
|
716
718
|
}
|
|
717
719
|
}
|
|
@@ -721,8 +723,29 @@ function assistantUiMessagesToStructuredHistory(messages) {
|
|
|
721
723
|
}
|
|
722
724
|
return structured;
|
|
723
725
|
}
|
|
726
|
+
/**
|
|
727
|
+
* Size this message will actually contribute to the request, AFTER the
|
|
728
|
+
* per-part truncation `assistantUiMessagesToStructuredHistory` applies. Tool
|
|
729
|
+
* calls are most of a tool-heavy turn, so counting only text parts priced a
|
|
730
|
+
* turn of 40 tool calls at ~0: it survived every trim against
|
|
731
|
+
* MAX_HISTORY_TOTAL_CHARS while the user's own prose was evicted around it.
|
|
732
|
+
*/
|
|
724
733
|
function estimateHistoryMessageCost(message) {
|
|
725
|
-
|
|
734
|
+
let cost = messageTextForHistory(message).length;
|
|
735
|
+
for (const part of message.content) {
|
|
736
|
+
if (part.type !== "tool-call")
|
|
737
|
+
continue;
|
|
738
|
+
const tool = part;
|
|
739
|
+
const argsCap = LARGE_INPUT_TOOL_NAMES.has(tool.toolName ?? "")
|
|
740
|
+
? MAX_HISTORY_LARGE_TOOL_ARGS_CHARS
|
|
741
|
+
: MAX_HISTORY_TOOL_ARGS_CHARS;
|
|
742
|
+
const argsText = tool.argsText ?? stableJson(tool.args ?? {});
|
|
743
|
+
cost += Math.min(argsText.length, argsCap);
|
|
744
|
+
if (tool.result !== undefined) {
|
|
745
|
+
cost += Math.min(String(tool.result).length, MAX_HISTORY_TOOL_RESULT_CHARS);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
return Math.max(1, cost);
|
|
726
749
|
}
|
|
727
750
|
function limitPriorMessagesForRequest(messages) {
|
|
728
751
|
const recent = messages.slice(-MAX_HISTORY_MESSAGES);
|
|
@@ -826,6 +849,59 @@ function continuationRepeatSignature(content) {
|
|
|
826
849
|
return "";
|
|
827
850
|
return Array.from(new Set(segments)).sort().join("\u0000");
|
|
828
851
|
}
|
|
852
|
+
/**
|
|
853
|
+
* Everything a continuation could have advanced, as one signature over the
|
|
854
|
+
* work it produced: the distinct tool results it completed, the tool it left
|
|
855
|
+
* in flight, and the action card it left unstarted. Two consecutive
|
|
856
|
+
* continuations with the same signature moved the turn nowhere, whatever the
|
|
857
|
+
* server's reason code called it.
|
|
858
|
+
*
|
|
859
|
+
* DISTINCT, not per-occurrence: a round that degenerates into four hundred
|
|
860
|
+
* copies of one tool call collapses to the same signature as the round before
|
|
861
|
+
* it, which is exactly the shape production shows (up to 28 identical calls in
|
|
862
|
+
* one turn).
|
|
863
|
+
*
|
|
864
|
+
* Narration counts only when the round left NO unfinished work. A model stuck
|
|
865
|
+
* re-wording "I have the file, creating it now!" while the same action never
|
|
866
|
+
* starts emits new prose every round, and prose alone is why that used to look
|
|
867
|
+
* like progress.
|
|
868
|
+
*/
|
|
869
|
+
function describeContinuationAdvance(content, preparingToolName) {
|
|
870
|
+
const work = new Set();
|
|
871
|
+
let stall = null;
|
|
872
|
+
for (const part of content) {
|
|
873
|
+
if (part.type !== "tool-call")
|
|
874
|
+
continue;
|
|
875
|
+
if (part.result !== undefined) {
|
|
876
|
+
if (part.activity !== true) {
|
|
877
|
+
work.add(`done ${part.toolName} ${inFlightToolInputSignature(part)} ${part.result.length} ${part.result.slice(0, 256)}`);
|
|
878
|
+
}
|
|
879
|
+
continue;
|
|
880
|
+
}
|
|
881
|
+
if (part.activity === true) {
|
|
882
|
+
work.add(`preparing ${part.toolName}`);
|
|
883
|
+
stall ??= { kind: "preparing", toolName: part.toolName };
|
|
884
|
+
}
|
|
885
|
+
else {
|
|
886
|
+
work.add(`pending ${part.toolName} ${inFlightToolInputSignature(part)}`);
|
|
887
|
+
stall = { kind: "in-flight", toolName: part.toolName };
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
// A "Preparing X action" card is an activity-trail entry, not a content
|
|
891
|
+
// part, so a round can sit on an unstarted action while its content holds
|
|
892
|
+
// nothing but narration.
|
|
893
|
+
if (preparingToolName && !stall) {
|
|
894
|
+
work.add(`preparing ${preparingToolName}`);
|
|
895
|
+
stall = { kind: "preparing", toolName: preparingToolName };
|
|
896
|
+
}
|
|
897
|
+
if (!stall) {
|
|
898
|
+
const text = continuationRepeatSignature(content);
|
|
899
|
+
if (text)
|
|
900
|
+
work.add(`text ${text}`);
|
|
901
|
+
stall = text ? { kind: "repeat" } : { kind: "empty" };
|
|
902
|
+
}
|
|
903
|
+
return { signature: Array.from(work).sort().join(""), stall };
|
|
904
|
+
}
|
|
829
905
|
/**
|
|
830
906
|
* True when an action was streamed but never returned a result yet — i.e. a
|
|
831
907
|
* `tool_start` with no matching `tool_done`. The server is still executing it,
|
|
@@ -1557,30 +1633,20 @@ export function createAgentChatAdapter(options) {
|
|
|
1557
1633
|
let internalContinuationRequest = latestUserIsRecovery;
|
|
1558
1634
|
let startupRecoveryAttempts = 0;
|
|
1559
1635
|
let queuedConflictRetries = 0;
|
|
1560
|
-
let staleRunContinuationAttempts = 0;
|
|
1561
|
-
let stalledTransientContinuationAttempts = 0;
|
|
1562
|
-
let emptyTransientContinuationAttempts = 0;
|
|
1563
1636
|
let totalTransientContinuationAttempts = 0;
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
let
|
|
1567
|
-
|
|
1568
|
-
//
|
|
1569
|
-
//
|
|
1570
|
-
|
|
1571
|
-
//
|
|
1572
|
-
//
|
|
1573
|
-
|
|
1574
|
-
//
|
|
1575
|
-
//
|
|
1576
|
-
|
|
1577
|
-
let lastInFlightToolSignature;
|
|
1578
|
-
let repeatedInFlightToolCount = 0;
|
|
1579
|
-
let recoveryGaveUpOnInFlightTool = false;
|
|
1580
|
-
const MAX_REPEATED_INFLIGHT_TOOL_STALLS = 3;
|
|
1581
|
-
let lastPreparingToolName;
|
|
1582
|
-
let repeatedActionPreparationCount = 0;
|
|
1583
|
-
let recoveryGaveUpOnActionPreparation = false;
|
|
1637
|
+
// Consecutive continuations whose advance signature matched the previous
|
|
1638
|
+
// round's (or was empty). Reset by any real advance.
|
|
1639
|
+
let nonAdvancingContinuations = 0;
|
|
1640
|
+
let lastAdvanceSignature = null;
|
|
1641
|
+
// Set at the moment we give up, so the user-facing message can name what
|
|
1642
|
+
// the turn was stuck on without a counter per shape of stuck.
|
|
1643
|
+
let recoveryStall = null;
|
|
1644
|
+
// Content already accounted for by a previous advance check. Tracked
|
|
1645
|
+
// separately from `visibleContinuationPrefix`, which deliberately does
|
|
1646
|
+
// NOT advance on a `loop_limit` boundary so that round's output stays in
|
|
1647
|
+
// continuation history; the advance check needs the per-round delta for
|
|
1648
|
+
// every reason code, `loop_limit` included.
|
|
1649
|
+
let advanceCheckPrefix = [];
|
|
1584
1650
|
// Set when the turn is stopped by MAX_TOTAL_TRANSIENT_CONTINUATIONS —
|
|
1585
1651
|
// the whole-turn continuation ceiling, not a dropped connection. This
|
|
1586
1652
|
// can trip on a turn that was making real progress the entire time, so
|
|
@@ -1633,17 +1699,10 @@ export function createAgentChatAdapter(options) {
|
|
|
1633
1699
|
lastRecoverableRunError?.message
|
|
1634
1700
|
? `last_recoverable_error: ${lastRecoverableRunError.message}`
|
|
1635
1701
|
: "",
|
|
1636
|
-
`
|
|
1637
|
-
`
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
`repeated_inflight_tool_stalls: ${repeatedInFlightToolCount}`,
|
|
1641
|
-
lastInFlightToolName
|
|
1642
|
-
? `last_inflight_tool: ${lastInFlightToolName}`
|
|
1643
|
-
: "",
|
|
1644
|
-
`repeated_action_preparation_stalls: ${repeatedActionPreparationCount}`,
|
|
1645
|
-
lastPreparingToolName
|
|
1646
|
-
? `last_preparing_tool: ${lastPreparingToolName}`
|
|
1702
|
+
`non_advancing_continuations: ${nonAdvancingContinuations}`,
|
|
1703
|
+
recoveryStall ? `stalled_on: ${recoveryStall.kind}` : "",
|
|
1704
|
+
recoveryStall?.toolName
|
|
1705
|
+
? `stalled_on_tool: ${recoveryStall.toolName}`
|
|
1647
1706
|
: "",
|
|
1648
1707
|
formatActivityTrail(lastActivityTrail)
|
|
1649
1708
|
? `activity_trail: ${formatActivityTrail(lastActivityTrail)}`
|
|
@@ -1667,15 +1726,15 @@ export function createAgentChatAdapter(options) {
|
|
|
1667
1726
|
.join("\n");
|
|
1668
1727
|
};
|
|
1669
1728
|
const exhaustedRecoveryMessage = (reason) => {
|
|
1670
|
-
if (
|
|
1729
|
+
if (recoveryStall?.kind === "in-flight") {
|
|
1671
1730
|
return "The agent got stuck waiting for the same tool to finish, so I stopped the automatic retries. The tool did not report a completed result.";
|
|
1672
1731
|
}
|
|
1673
|
-
if (
|
|
1732
|
+
if (recoveryStall?.kind === "repeat") {
|
|
1674
1733
|
return "The agent got stuck repeating the same response without finishing, so I stopped the automatic retries. This often happens when it tries to re-type a large pasted file into one action — starting a new chat, or asking for a smaller first step, usually gets it unstuck.";
|
|
1675
1734
|
}
|
|
1676
|
-
if (
|
|
1677
|
-
const tool =
|
|
1678
|
-
? ` the ${humanizeActionName(
|
|
1735
|
+
if (recoveryStall?.kind === "preparing") {
|
|
1736
|
+
const tool = recoveryStall.toolName
|
|
1737
|
+
? ` the ${humanizeActionName(recoveryStall.toolName)} action`
|
|
1679
1738
|
: " the same action";
|
|
1680
1739
|
return `The agent got stuck preparing${tool} input and never started the tool, so I stopped the automatic retries. Try a smaller first step or a more compact version of the request.`;
|
|
1681
1740
|
}
|
|
@@ -1688,6 +1747,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1688
1747
|
reason === "stream_ended")) {
|
|
1689
1748
|
return "The agent request started but did not produce any visible progress before timing out. I stopped the automatic retries so this chat would not stay stuck on Thinking.";
|
|
1690
1749
|
}
|
|
1750
|
+
if (recoveryStall?.kind === "empty") {
|
|
1751
|
+
return "The agent stopped producing new output across several automatic retries, so I stopped it here rather than leaving the chat stuck on Thinking.";
|
|
1752
|
+
}
|
|
1691
1753
|
return "The agent connection kept failing after several automatic recovery attempts.";
|
|
1692
1754
|
};
|
|
1693
1755
|
const dispatchAuthError = (reason) => {
|
|
@@ -1818,14 +1880,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1818
1880
|
attemptedRunIds: [...attemptedRunIds],
|
|
1819
1881
|
activityTrail: [...lastActivityTrail],
|
|
1820
1882
|
startupRecoveryAttempts,
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
repeatedTransientContinuationAttempts,
|
|
1825
|
-
repeatedInFlightToolCount,
|
|
1826
|
-
lastInFlightToolName,
|
|
1827
|
-
repeatedActionPreparationCount,
|
|
1828
|
-
lastPreparingToolName,
|
|
1883
|
+
nonAdvancingContinuations,
|
|
1884
|
+
stalledOn: recoveryStall?.kind,
|
|
1885
|
+
stalledOnTool: recoveryStall?.toolName,
|
|
1829
1886
|
totalTransientContinuationAttempts,
|
|
1830
1887
|
...extra,
|
|
1831
1888
|
},
|
|
@@ -1837,14 +1894,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1837
1894
|
lastSeq,
|
|
1838
1895
|
contentParts: content.length,
|
|
1839
1896
|
startupRecoveryAttempts,
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
repeatedTransientContinuationAttempts,
|
|
1844
|
-
repeatedInFlightToolCount,
|
|
1845
|
-
lastInFlightToolName,
|
|
1846
|
-
repeatedActionPreparationCount,
|
|
1847
|
-
lastPreparingToolName,
|
|
1897
|
+
nonAdvancingContinuations,
|
|
1898
|
+
stalledOn: recoveryStall?.kind,
|
|
1899
|
+
stalledOnTool: recoveryStall?.toolName,
|
|
1848
1900
|
totalTransientContinuationAttempts,
|
|
1849
1901
|
},
|
|
1850
1902
|
},
|
|
@@ -2762,192 +2814,63 @@ export function createAgentChatAdapter(options) {
|
|
|
2762
2814
|
const isTransient = signal.reason !== "loop_limit";
|
|
2763
2815
|
const visibleContent = visibleContentForContinuation();
|
|
2764
2816
|
let currentPartialHistory = contentToContinuationHistory(visibleContent);
|
|
2765
|
-
// Real, content-weight progress: streamed text or a completed tool
|
|
2766
|
-
// result. Used to reset the stalled/empty counters so trivial
|
|
2767
|
-
// whitespace-only output cannot keep the run alive indefinitely.
|
|
2768
2817
|
const madeContentProgress = hasContinuationProgress(visibleContent);
|
|
2769
2818
|
// An action was streamed but has not returned yet (a tool_start with
|
|
2770
|
-
// no tool_done).
|
|
2771
|
-
//
|
|
2772
|
-
//
|
|
2773
|
-
//
|
|
2774
|
-
// loops when the LLM timed out while assembling a large tool input.
|
|
2819
|
+
// no tool_done). Only a run_timeout in that window means the SERVER
|
|
2820
|
+
// is still executing it and a reconnect can still deliver the result;
|
|
2821
|
+
// every other reason means the stream is dead and nothing will
|
|
2822
|
+
// complete the tool.
|
|
2775
2823
|
const hasInFlightTool = hasInFlightToolCall(visibleContent);
|
|
2776
2824
|
const completedTool = lastCompletedTimeoutCandidateTool(content);
|
|
2777
|
-
// Either real output or an actively-running tool counts as progress
|
|
2778
|
-
// for the stalled/empty caps.
|
|
2779
|
-
const madeProgress = madeContentProgress || hasInFlightTool;
|
|
2780
|
-
const emptyRun = !madeProgress && signal.reason !== "loop_limit";
|
|
2781
|
-
const madeDurableToolProgress = visibleContent.some((part) => part.type === "tool-call" &&
|
|
2782
|
-
part.activity !== true &&
|
|
2783
|
-
part.result !== undefined);
|
|
2784
2825
|
const currentPreparingToolName = lastUnresolvedToolActivity(visibleContent) ??
|
|
2785
2826
|
lastPreparingActionTool(signal.activityTrail);
|
|
2786
|
-
//
|
|
2787
|
-
//
|
|
2788
|
-
//
|
|
2789
|
-
//
|
|
2790
|
-
//
|
|
2791
|
-
//
|
|
2792
|
-
//
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
?
|
|
2805
|
-
:
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
|
|
2815
|
-
|
|
2816
|
-
|
|
2817
|
-
|
|
2818
|
-
|
|
2819
|
-
|
|
2820
|
-
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2824
|
-
const isRepeatedActionPreparationCandidate = signal.reason !== "loop_limit" &&
|
|
2825
|
-
currentPreparingToolName !== undefined &&
|
|
2826
|
-
!hasInFlightTool &&
|
|
2827
|
-
!madeDurableToolProgress;
|
|
2828
|
-
if (isRepeatedActionPreparationCandidate) {
|
|
2829
|
-
if (currentPreparingToolName === lastPreparingToolName) {
|
|
2830
|
-
repeatedActionPreparationCount += 1;
|
|
2831
|
-
}
|
|
2832
|
-
else {
|
|
2833
|
-
repeatedActionPreparationCount = 0;
|
|
2834
|
-
lastPreparingToolName = currentPreparingToolName;
|
|
2835
|
-
}
|
|
2836
|
-
}
|
|
2837
|
-
else if (!currentPreparingToolName ||
|
|
2838
|
-
hasInFlightTool ||
|
|
2839
|
-
madeDurableToolProgress) {
|
|
2840
|
-
repeatedActionPreparationCount = 0;
|
|
2841
|
-
lastPreparingToolName = undefined;
|
|
2827
|
+
// THE boundary. One question — did this continuation advance the
|
|
2828
|
+
// turn? — decides every reason code, `loop_limit` included. It used
|
|
2829
|
+
// to skip this block entirely AND reset two of the budgets, so a
|
|
2830
|
+
// model that degenerated into a tool loop re-POSTed the same turnId
|
|
2831
|
+
// until the user gave up (measured in production at 186 minutes with
|
|
2832
|
+
// no answer). The durable per-turn ledger lives server-side inside
|
|
2833
|
+
// one run and never sees a client re-POST, so nothing else bounds it.
|
|
2834
|
+
const advanceDelta = contentAfterContinuationPrefix(content, advanceCheckPrefix);
|
|
2835
|
+
const advance = describeContinuationAdvance(advanceDelta, currentPreparingToolName);
|
|
2836
|
+
const serverStillRunningTool = signal.reason === "run_timeout" && hasInFlightTool;
|
|
2837
|
+
const advanced = serverStillRunningTool ||
|
|
2838
|
+
(advance.signature !== "" &&
|
|
2839
|
+
advance.signature !== lastAdvanceSignature);
|
|
2840
|
+
// Only remember a non-empty signature, so an empty round between two
|
|
2841
|
+
// identical rounds cannot launder the second one into "new work".
|
|
2842
|
+
if (advance.signature)
|
|
2843
|
+
lastAdvanceSignature = advance.signature;
|
|
2844
|
+
nonAdvancingContinuations = advanced
|
|
2845
|
+
? 0
|
|
2846
|
+
: nonAdvancingContinuations + 1;
|
|
2847
|
+
totalTransientContinuationAttempts += 1;
|
|
2848
|
+
advanceCheckPrefix = snapshotContent(content);
|
|
2849
|
+
// If a tool already completed, do not turn a missing closing
|
|
2850
|
+
// sentence into a scary connection failure. Give the model one
|
|
2851
|
+
// continuation opportunity (the completed tool itself counts as an
|
|
2852
|
+
// advance on the first timeout); if the follow-up produces no new
|
|
2853
|
+
// content, stop locally with a clear completed-tool warning.
|
|
2854
|
+
if (signal.reason === "run_timeout" &&
|
|
2855
|
+
completedTool &&
|
|
2856
|
+
!hasInFlightToolCall(content) &&
|
|
2857
|
+
!currentPreparingToolName &&
|
|
2858
|
+
!madeContentProgress &&
|
|
2859
|
+
!hasInFlightTool) {
|
|
2860
|
+
return {
|
|
2861
|
+
ok: false,
|
|
2862
|
+
resetVisibleContent: false,
|
|
2863
|
+
completedToolName: completedTool.toolName,
|
|
2864
|
+
};
|
|
2842
2865
|
}
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
// true and the stalled/empty budgets never trip. Compare this round's
|
|
2847
|
-
// unique-sentence signature to the previous round's; a match with no
|
|
2848
|
-
// tool progress is a non-advancing loop. Tracked by its own counter
|
|
2849
|
-
// so it bails after a few rounds instead of the full transient budget,
|
|
2850
|
-
// without perturbing the stalled/empty/stale accounting.
|
|
2851
|
-
const repeatSignature = continuationRepeatSignature(visibleContent);
|
|
2852
|
-
const isNonAdvancingRepeat = signal.reason !== "loop_limit" &&
|
|
2853
|
-
repeatSignature !== "" &&
|
|
2854
|
-
repeatSignature === lastContinuationRepeatSignature &&
|
|
2855
|
-
!hasInFlightTool &&
|
|
2856
|
-
!madeDurableToolProgress;
|
|
2857
|
-
if (signal.reason === "loop_limit") {
|
|
2858
|
-
stalledTransientContinuationAttempts = 0;
|
|
2859
|
-
emptyTransientContinuationAttempts = 0;
|
|
2866
|
+
if (nonAdvancingContinuations >= MAX_NON_ADVANCING_CONTINUATIONS) {
|
|
2867
|
+
recoveryStall = advance.stall;
|
|
2868
|
+
return { ok: false, resetVisibleContent: false };
|
|
2860
2869
|
}
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
// continuation opportunity (the completed tool itself counts as
|
|
2866
|
-
// progress on the first timeout); if the follow-up produces no new
|
|
2867
|
-
// content, stop locally with a clear completed-tool warning.
|
|
2868
|
-
if (signal.reason === "run_timeout" &&
|
|
2869
|
-
completedTool &&
|
|
2870
|
-
!hasInFlightToolCall(content) &&
|
|
2871
|
-
!currentPreparingToolName &&
|
|
2872
|
-
!madeContentProgress &&
|
|
2873
|
-
!hasInFlightTool) {
|
|
2874
|
-
return {
|
|
2875
|
-
ok: false,
|
|
2876
|
-
resetVisibleContent: false,
|
|
2877
|
-
completedToolName: completedTool.toolName,
|
|
2878
|
-
};
|
|
2879
|
-
}
|
|
2880
|
-
// Bail when the same write tool is stuck in-flight across too many
|
|
2881
|
-
// consecutive continuations. Checked before the text-repeat guard
|
|
2882
|
-
// because hasInFlightTool=true would mask the repeat as progress.
|
|
2883
|
-
if (repeatedInFlightToolCount >= MAX_REPEATED_INFLIGHT_TOOL_STALLS) {
|
|
2884
|
-
recoveryGaveUpOnInFlightTool = true;
|
|
2885
|
-
return { ok: false, resetVisibleContent: false };
|
|
2886
|
-
}
|
|
2887
|
-
if (repeatedActionPreparationCount >
|
|
2888
|
-
MAX_REPEATED_ACTION_PREPARATION_CONTINUATIONS) {
|
|
2889
|
-
recoveryGaveUpOnActionPreparation = true;
|
|
2890
|
-
return { ok: false, resetVisibleContent: false };
|
|
2891
|
-
}
|
|
2892
|
-
// Bail fast on a non-advancing repetition loop, well before the
|
|
2893
|
-
// stalled/empty/total budgets would (each round otherwise re-sends
|
|
2894
|
-
// the whole pasted payload). Tracked separately so it never trips
|
|
2895
|
-
// on legitimately-progressing runs that happen to be slow.
|
|
2896
|
-
if (isNonAdvancingRepeat) {
|
|
2897
|
-
repeatedTransientContinuationAttempts += 1;
|
|
2898
|
-
if (repeatedTransientContinuationAttempts >
|
|
2899
|
-
MAX_REPEATED_TRANSIENT_CONTINUATIONS) {
|
|
2900
|
-
recoveryGaveUpOnRepetition = true;
|
|
2901
|
-
return { ok: false, resetVisibleContent: false };
|
|
2902
|
-
}
|
|
2903
|
-
}
|
|
2904
|
-
else {
|
|
2905
|
-
repeatedTransientContinuationAttempts = 0;
|
|
2906
|
-
}
|
|
2907
|
-
if (repeatSignature) {
|
|
2908
|
-
lastContinuationRepeatSignature = repeatSignature;
|
|
2909
|
-
}
|
|
2910
|
-
// A run_timeout that produced nothing visible and no activity (the
|
|
2911
|
-
// model spent the whole soft-timeout window thinking before its
|
|
2912
|
-
// first output) is NOT an immediate give-up: a transient slow start
|
|
2913
|
-
// routinely recovers on the next continuation. Let it fall through
|
|
2914
|
-
// to the empty-continuation budget below so it retries a bounded
|
|
2915
|
-
// number of times before surfacing the "no visible progress" error.
|
|
2916
|
-
// Reset the empty-continuation counter on real progress — streamed
|
|
2917
|
-
// text/completed tool OR an in-flight tool the server is running —
|
|
2918
|
-
// not merely on a non-zero part count, which whitespace-only or
|
|
2919
|
-
// unresolved-only output would falsely satisfy.
|
|
2920
|
-
if (!madeProgress) {
|
|
2921
|
-
emptyTransientContinuationAttempts += 1;
|
|
2922
|
-
if (emptyTransientContinuationAttempts >
|
|
2923
|
-
MAX_EMPTY_TRANSIENT_CONTINUATIONS) {
|
|
2924
|
-
return { ok: false, resetVisibleContent: false };
|
|
2925
|
-
}
|
|
2926
|
-
}
|
|
2927
|
-
else {
|
|
2928
|
-
emptyTransientContinuationAttempts = 0;
|
|
2929
|
-
}
|
|
2930
|
-
if (signal.reason === "stale_run") {
|
|
2931
|
-
staleRunContinuationAttempts = madeDurableToolProgress
|
|
2932
|
-
? 0
|
|
2933
|
-
: staleRunContinuationAttempts + 1;
|
|
2934
|
-
if (staleRunContinuationAttempts > MAX_STALE_RUN_CONTINUATIONS) {
|
|
2935
|
-
return { ok: false, resetVisibleContent: false };
|
|
2936
|
-
}
|
|
2937
|
-
}
|
|
2938
|
-
stalledTransientContinuationAttempts = madeProgress
|
|
2939
|
-
? 0
|
|
2940
|
-
: stalledTransientContinuationAttempts + 1;
|
|
2941
|
-
if (totalTransientContinuationAttempts >
|
|
2942
|
-
MAX_TOTAL_TRANSIENT_CONTINUATIONS) {
|
|
2943
|
-
recoveryGaveUpOnTransientBudget = true;
|
|
2944
|
-
}
|
|
2945
|
-
if (stalledTransientContinuationAttempts >
|
|
2946
|
-
MAX_STALLED_TRANSIENT_CONTINUATIONS ||
|
|
2947
|
-
totalTransientContinuationAttempts >
|
|
2948
|
-
MAX_TOTAL_TRANSIENT_CONTINUATIONS) {
|
|
2949
|
-
return { ok: false, resetVisibleContent: false };
|
|
2950
|
-
}
|
|
2870
|
+
if (totalTransientContinuationAttempts >
|
|
2871
|
+
MAX_TOTAL_TRANSIENT_CONTINUATIONS) {
|
|
2872
|
+
recoveryGaveUpOnTransientBudget = true;
|
|
2873
|
+
return { ok: false, resetVisibleContent: false };
|
|
2951
2874
|
}
|
|
2952
2875
|
if (isTransient) {
|
|
2953
2876
|
const settledInterruptedTools = settleInterruptedToolCalls(visibleContent, undefined, { includeActivity: true });
|
|
@@ -2997,13 +2920,21 @@ export function createAgentChatAdapter(options) {
|
|
|
2997
2920
|
startupRecoveryAttempts = 0;
|
|
2998
2921
|
clearActiveRun();
|
|
2999
2922
|
if (!isTransient) {
|
|
3000
|
-
return {
|
|
2923
|
+
return {
|
|
2924
|
+
ok: true,
|
|
2925
|
+
resetVisibleContent: false,
|
|
2926
|
+
nonAdvancing: !advanced,
|
|
2927
|
+
};
|
|
3001
2928
|
}
|
|
3002
2929
|
// Keep everything visible during transient recovery. The continuation
|
|
3003
2930
|
// prefix diff tracks what the next request has already seen, so
|
|
3004
2931
|
// preserving text no longer causes duplicate continuation history.
|
|
3005
2932
|
visibleContinuationPrefix = snapshotContent(content);
|
|
3006
|
-
return {
|
|
2933
|
+
return {
|
|
2934
|
+
ok: true,
|
|
2935
|
+
resetVisibleContent: false,
|
|
2936
|
+
nonAdvancing: !advanced,
|
|
2937
|
+
};
|
|
3007
2938
|
};
|
|
3008
2939
|
while (true) {
|
|
3009
2940
|
try {
|
|
@@ -3410,11 +3341,11 @@ export function createAgentChatAdapter(options) {
|
|
|
3410
3341
|
detail: { tabId },
|
|
3411
3342
|
}));
|
|
3412
3343
|
}
|
|
3413
|
-
// A run that
|
|
3344
|
+
// A run that advanced nothing usually never got past
|
|
3414
3345
|
// startup/gateway; re-POSTing the identical payload immediately
|
|
3415
3346
|
// just hits the identical wall. Back that case off instead.
|
|
3416
|
-
if (continuation.
|
|
3417
|
-
await retryDelay(Math.max(0,
|
|
3347
|
+
if (continuation.nonAdvancing) {
|
|
3348
|
+
await retryDelay(Math.max(0, nonAdvancingContinuations - 1), abortSignal);
|
|
3418
3349
|
}
|
|
3419
3350
|
else {
|
|
3420
3351
|
await delay(250, abortSignal);
|
|
@@ -110,6 +110,13 @@ export interface AgentAutoContinueErrorInfo {
|
|
|
110
110
|
recoverable?: boolean;
|
|
111
111
|
upgradeUrl?: string;
|
|
112
112
|
}
|
|
113
|
+
/**
|
|
114
|
+
* Kept verbatim in sync with `INTERRUPTED_TOOL_RESULT_MARKER`
|
|
115
|
+
* (agent/production-agent.ts): the server matches this substring in replayed
|
|
116
|
+
* history to count how many times a write tool was interrupted, which is the
|
|
117
|
+
* only thing that tells "we do not know if it landed" apart from "it failed".
|
|
118
|
+
*/
|
|
119
|
+
export declare const INTERRUPTED_TOOL_RESULT = "Interrupted before this tool returned a result.";
|
|
113
120
|
export declare function settleInterruptedToolCalls(content: ContentPart[], result?: string, options?: {
|
|
114
121
|
includeActivity?: boolean;
|
|
115
122
|
activityResult?: string;
|