@agent-native/core 0.153.4 → 0.153.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/templates/analytics/actions/account-deep-dive.ts +1 -0
- package/corpus/templates/analytics/actions/bigquery-query.ts +1 -0
- package/corpus/templates/analytics/actions/bigquery.ts +1 -0
- package/corpus/templates/analytics/actions/content-calendar-schema.ts +1 -0
- package/corpus/templates/analytics/actions/content-calendar.ts +1 -0
- package/corpus/templates/analytics/actions/gcloud.ts +1 -0
- package/corpus/templates/analytics/actions/get-data-program.ts +4 -0
- package/corpus/templates/analytics/actions/get-error-issue.ts +1 -0
- package/corpus/templates/analytics/actions/get-monitor-stats.ts +1 -0
- package/corpus/templates/analytics/actions/get-monitor.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-events.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-summary.ts +1 -0
- package/corpus/templates/analytics/actions/get-session-replay-timeline.ts +1 -0
- package/corpus/templates/analytics/actions/gong-calls.ts +1 -0
- package/corpus/templates/analytics/actions/gong-native-insights.ts +1 -0
- package/corpus/templates/analytics/actions/grafana.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-deals.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-metrics.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-pipelines.ts +1 -0
- package/corpus/templates/analytics/actions/hubspot-records.ts +1 -0
- package/corpus/templates/analytics/actions/jira-search.ts +1 -0
- package/corpus/templates/analytics/actions/jira.ts +1 -0
- package/corpus/templates/analytics/actions/list-error-issues.ts +1 -0
- package/corpus/templates/analytics/actions/list-monitors.ts +1 -0
- package/corpus/templates/analytics/actions/list-session-recordings.ts +1 -0
- package/corpus/templates/analytics/actions/match-error-issues.ts +1 -0
- package/corpus/templates/analytics/actions/prometheus.ts +1 -0
- package/corpus/templates/analytics/actions/pylon-issues.ts +1 -0
- package/corpus/templates/analytics/actions/query-agent-native-analytics.ts +1 -0
- package/corpus/templates/analytics/actions/query-dashboard-panel.ts +1 -0
- package/corpus/templates/analytics/actions/query-inbound-forms.ts +1 -0
- package/corpus/templates/analytics/actions/run-monitor-check.ts +1 -0
- package/corpus/templates/analytics/actions/sentry.ts +1 -0
- package/corpus/templates/analytics/actions/seo-blog-pages.ts +1 -0
- package/corpus/templates/analytics/actions/seo-page-keywords.ts +1 -0
- package/corpus/templates/analytics/actions/seo-top-keywords.ts +1 -0
- package/corpus/templates/analytics/actions/slack-messages.ts +1 -0
- package/corpus/templates/analytics/actions/stripe.ts +1 -0
- package/corpus/templates/analytics/actions/test-custom-api-connection.ts +1 -0
- package/corpus/templates/analytics/server/lib/real-data-actions.ts +62 -47
- package/corpus/templates/analytics/server/plugins/agent-chat.ts +27 -34
- package/dist/action.d.ts +13 -0
- package/dist/action.js +3 -0
- package/dist/agent/engine/tool-call-journal-seed.d.ts +27 -10
- package/dist/agent/engine/tool-call-journal-seed.js +63 -32
- package/dist/agent/production-agent.d.ts +40 -1
- package/dist/agent/production-agent.js +201 -62
- package/dist/agent/run-loop-with-resume.d.ts +1 -1
- package/dist/agent/run-loop-with-resume.js +49 -35
- package/dist/agent/run-manager.js +17 -11
- package/dist/agent/run-store.d.ts +16 -0
- package/dist/agent/run-store.js +51 -9
- package/dist/agent/thread-data-builder.d.ts +1 -0
- package/dist/agent/thread-data-builder.js +15 -2
- package/dist/client/RouteTransitionIndicator.js +1 -1
- package/dist/client/agent-chat-adapter.js +242 -282
- package/dist/client/sse-event-processor.d.ts +7 -0
- package/dist/client/sse-event-processor.js +28 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/db/client.js +2 -2
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/actions/github-repo-files.js +1 -0
- package/dist/provider-api/actions/provider-api.js +1 -0
- package/dist/provider-api/actions/staged-datasets.js +1 -0
- package/dist/provider-api/corpus-jobs.js +1 -0
- package/dist/server/action-discovery.js +2 -0
- package/dist/server/agent-chat-plugin.js +35 -3
- package/dist/server/realtime-token.d.ts +1 -1
- package/package.json +1 -1
|
@@ -6,7 +6,7 @@ import { getOrCreateAnalyticsSessionId } from "./analytics-session.js";
|
|
|
6
6
|
import { captureError } from "./analytics.js";
|
|
7
7
|
import { agentNativePath } from "./api-path.js";
|
|
8
8
|
import { formatChatErrorText, normalizeChatError } from "./error-format.js";
|
|
9
|
-
import { AgentAutoContinueSignal, appendMissingFinalResponseWarning, readSSEStream, settleInterruptedToolCalls, } from "./sse-event-processor.js";
|
|
9
|
+
import { AgentAutoContinueSignal, INTERRUPTED_TOOL_RESULT, appendMissingFinalResponseWarning, readSSEStream, settleInterruptedToolCalls, } from "./sse-event-processor.js";
|
|
10
10
|
const TEXT_ATTACHMENT_CONTENT_TYPES = new Set([
|
|
11
11
|
"application/json",
|
|
12
12
|
"application/x-ndjson",
|
|
@@ -24,37 +24,29 @@ const AUTO_CONTINUE_COMPLETION_GUARD = "Before doing more work, inspect the prio
|
|
|
24
24
|
const MAX_RECONNECT_ATTEMPTS = 5;
|
|
25
25
|
const MAX_STARTUP_RECOVERY_ATTEMPTS = 8;
|
|
26
26
|
const MAX_QUEUED_CONFLICT_RETRIES = 120;
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
//
|
|
27
|
+
// The consecutive non-advancing continuation (see `describeContinuationAdvance`)
|
|
28
|
+
// that ends the turn: the third one stops it, so two are tolerated. This
|
|
29
|
+
// replaced five separate budgets — stale-run, stalled, empty, repeated-narration
|
|
30
|
+
// and repeated-action-preparation — each of which existed because the rung above
|
|
31
|
+
// it had a hole the next one patched. One question answers all five: a round
|
|
32
|
+
// that produced nothing, or produced exactly what the round before it
|
|
33
|
+
// produced, did not advance.
|
|
34
|
+
const MAX_NON_ADVANCING_CONTINUATIONS = 3;
|
|
35
|
+
// Ceiling across the whole turn for TRANSIENT continuations — rounds that
|
|
36
|
+
// followed a failure (timeout, dropped stream, stale run). Every attempt
|
|
37
|
+
// re-POSTs the full history plus attachments, so a high ceiling only burns
|
|
38
|
+
// tokens and wall time on a connection that keeps breaking.
|
|
34
39
|
const MAX_TOTAL_TRANSIENT_CONTINUATIONS = 12;
|
|
35
|
-
//
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
|
|
43
|
-
//
|
|
44
|
-
|
|
45
|
-
// up. A model that degenerates into repeating one phrase ("I have the full
|
|
46
|
-
// HTML. Creating the extension now!") emits "new" text every continuation,
|
|
47
|
-
// which keeps resetting the stalled/empty budgets — so without this guard the
|
|
48
|
-
// stuck turn burns the entire MAX_TOTAL_TRANSIENT_CONTINUATIONS budget (each
|
|
49
|
-
// round re-sending any large pasted payload) before bailing. Catching the
|
|
50
|
-
// repeat ends it in a few rounds with a clear, actionable message instead.
|
|
51
|
-
const MAX_REPEATED_TRANSIENT_CONTINUATIONS = 3;
|
|
52
|
-
// How many consecutive continuations that only reach the SAME "preparing
|
|
53
|
-
// action" activity card we tolerate before giving up. This catches runs that
|
|
54
|
-
// keep timing out while assembling a large tool payload: they are not empty,
|
|
55
|
-
// and the narration may vary enough to bypass the text-repeat guard, but the
|
|
56
|
-
// real tool never starts.
|
|
57
|
-
const MAX_REPEATED_ACTION_PREPARATION_CONTINUATIONS = 3;
|
|
40
|
+
// Ceiling across the whole turn for WORK-boundary continuations. A `loop_limit`
|
|
41
|
+
// round is not a failure — the server spent a full iteration budget on real
|
|
42
|
+
// tool work and handed the turn back — so counting it against the transient
|
|
43
|
+
// ceiling killed progressing turns at round 13 that had never failed once.
|
|
44
|
+
// Sized against the server's own limits: at its default budget of 400
|
|
45
|
+
// iterations per run this is ~10,000 tool calls, two orders past the deepest
|
|
46
|
+
// legitimate production turn (117 tool calls) that sized that budget. It still
|
|
47
|
+
// has to exist, because the server's per-turn token backstop rides the request
|
|
48
|
+
// body between server-chained chunks and resets on every client re-POST.
|
|
49
|
+
const MAX_LOOP_LIMIT_CONTINUATIONS = 25;
|
|
58
50
|
const RETRY_BASE_DELAY_MS = 500;
|
|
59
51
|
const RETRY_MAX_DELAY_MS = 8_000;
|
|
60
52
|
const MAX_HISTORY_ATTACHMENT_CHARS = 60_000;
|
|
@@ -636,14 +628,25 @@ function contentToStructuredMessages(content, nextToolCallId, options) {
|
|
|
636
628
|
: (part.args ?? {}),
|
|
637
629
|
});
|
|
638
630
|
if (part.result !== undefined) {
|
|
631
|
+
const body = truncate
|
|
632
|
+
? truncateForHistory(toolResultContent(part.result, part.toolName), MAX_HISTORY_TOOL_RESULT_CHARS, "Tool result")
|
|
633
|
+
: toolResultContent(part.result, part.toolName);
|
|
639
634
|
pendingToolResults.push({
|
|
640
635
|
type: "tool-result",
|
|
641
636
|
toolCallId,
|
|
642
637
|
toolName: part.toolName,
|
|
643
638
|
toolInput: JSON.stringify(part.args ?? {}),
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
639
|
+
// A settled-interrupted tool has a result whose outcome is UNKNOWN.
|
|
640
|
+
// It is neither a success nor a failure, so it carries the marker the
|
|
641
|
+
// server's write-interruption breaker matches on instead of claiming
|
|
642
|
+
// either.
|
|
643
|
+
content: part.outcome === "unknown" &&
|
|
644
|
+
!body.includes(INTERRUPTED_TOOL_RESULT)
|
|
645
|
+
? `${INTERRUPTED_TOOL_RESULT} ${body}`
|
|
646
|
+
: body,
|
|
647
|
+
...(part.isError === true && part.outcome !== "unknown"
|
|
648
|
+
? { isError: true }
|
|
649
|
+
: {}),
|
|
647
650
|
});
|
|
648
651
|
}
|
|
649
652
|
else {
|
|
@@ -652,7 +655,7 @@ function contentToStructuredMessages(content, nextToolCallId, options) {
|
|
|
652
655
|
toolCallId,
|
|
653
656
|
toolName: part.toolName,
|
|
654
657
|
toolInput: JSON.stringify(part.args ?? {}),
|
|
655
|
-
content:
|
|
658
|
+
content: INTERRUPTED_TOOL_RESULT,
|
|
656
659
|
});
|
|
657
660
|
}
|
|
658
661
|
}
|
|
@@ -712,6 +715,14 @@ function assistantUiMessagesToStructuredHistory(messages) {
|
|
|
712
715
|
...(part.result !== undefined
|
|
713
716
|
? { result: toolResultContent(part.result, toolName) }
|
|
714
717
|
: {}),
|
|
718
|
+
// A failed call replayed without `isError` reads to the next turn as
|
|
719
|
+
// an ordinary success whose body happens to contain error prose, so
|
|
720
|
+
// the model cannot tell it already tried this and failed. The
|
|
721
|
+
// server's repeat-error breaker keys on exactly this flag.
|
|
722
|
+
...(part.isError === true ? { isError: true } : {}),
|
|
723
|
+
...(part.outcome === "unknown"
|
|
724
|
+
? { outcome: "unknown" }
|
|
725
|
+
: {}),
|
|
715
726
|
});
|
|
716
727
|
}
|
|
717
728
|
}
|
|
@@ -721,8 +732,33 @@ function assistantUiMessagesToStructuredHistory(messages) {
|
|
|
721
732
|
}
|
|
722
733
|
return structured;
|
|
723
734
|
}
|
|
735
|
+
/**
|
|
736
|
+
* Size this message will actually contribute to the request, AFTER the
|
|
737
|
+
* per-part truncation `assistantUiMessagesToStructuredHistory` applies. Tool
|
|
738
|
+
* calls are most of a tool-heavy turn, so counting only text parts priced a
|
|
739
|
+
* turn of 40 tool calls at ~0: it survived every trim against
|
|
740
|
+
* MAX_HISTORY_TOTAL_CHARS while the user's own prose was evicted around it.
|
|
741
|
+
*/
|
|
724
742
|
function estimateHistoryMessageCost(message) {
|
|
725
|
-
|
|
743
|
+
let cost = messageTextForHistory(message).length;
|
|
744
|
+
for (const part of message.content) {
|
|
745
|
+
if (part.type !== "tool-call")
|
|
746
|
+
continue;
|
|
747
|
+
const tool = part;
|
|
748
|
+
const argsCap = LARGE_INPUT_TOOL_NAMES.has(tool.toolName ?? "")
|
|
749
|
+
? MAX_HISTORY_LARGE_TOOL_ARGS_CHARS
|
|
750
|
+
: MAX_HISTORY_TOOL_ARGS_CHARS;
|
|
751
|
+
const argsText = tool.argsText ?? stableJson(tool.args ?? {});
|
|
752
|
+
cost += Math.min(argsText.length, argsCap);
|
|
753
|
+
if (tool.result !== undefined) {
|
|
754
|
+
cost += Math.min(
|
|
755
|
+
// Price the string the request actually carries. `String(result)` is
|
|
756
|
+
// 15 chars ("[object Object]") for every object result, which is most
|
|
757
|
+
// of them.
|
|
758
|
+
toolResultContent(tool.result, tool.toolName).length, MAX_HISTORY_TOOL_RESULT_CHARS);
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
return Math.max(1, cost);
|
|
726
762
|
}
|
|
727
763
|
function limitPriorMessagesForRequest(messages) {
|
|
728
764
|
const recent = messages.slice(-MAX_HISTORY_MESSAGES);
|
|
@@ -826,6 +862,59 @@ function continuationRepeatSignature(content) {
|
|
|
826
862
|
return "";
|
|
827
863
|
return Array.from(new Set(segments)).sort().join("\u0000");
|
|
828
864
|
}
|
|
865
|
+
/**
|
|
866
|
+
* Everything a continuation could have advanced, as one signature over the
|
|
867
|
+
* work it produced: the distinct tool results it completed, the tool it left
|
|
868
|
+
* in flight, and the action card it left unstarted. Two consecutive
|
|
869
|
+
* continuations with the same signature moved the turn nowhere, whatever the
|
|
870
|
+
* server's reason code called it.
|
|
871
|
+
*
|
|
872
|
+
* DISTINCT, not per-occurrence: a round that degenerates into four hundred
|
|
873
|
+
* copies of one tool call collapses to the same signature as the round before
|
|
874
|
+
* it, which is exactly the shape production shows (up to 28 identical calls in
|
|
875
|
+
* one turn).
|
|
876
|
+
*
|
|
877
|
+
* Narration counts only when the round left NO unfinished work. A model stuck
|
|
878
|
+
* re-wording "I have the file, creating it now!" while the same action never
|
|
879
|
+
* starts emits new prose every round, and prose alone is why that used to look
|
|
880
|
+
* like progress.
|
|
881
|
+
*/
|
|
882
|
+
function describeContinuationAdvance(content, preparingToolName) {
|
|
883
|
+
const work = new Set();
|
|
884
|
+
let stall = null;
|
|
885
|
+
for (const part of content) {
|
|
886
|
+
if (part.type !== "tool-call")
|
|
887
|
+
continue;
|
|
888
|
+
if (part.result !== undefined) {
|
|
889
|
+
if (part.activity !== true) {
|
|
890
|
+
work.add(`done ${part.toolName} ${inFlightToolInputSignature(part)} ${part.result.length} ${part.result.slice(0, 256)}`);
|
|
891
|
+
}
|
|
892
|
+
continue;
|
|
893
|
+
}
|
|
894
|
+
if (part.activity === true) {
|
|
895
|
+
work.add(`preparing ${part.toolName}`);
|
|
896
|
+
stall ??= { kind: "preparing", toolName: part.toolName };
|
|
897
|
+
}
|
|
898
|
+
else {
|
|
899
|
+
work.add(`pending ${part.toolName} ${inFlightToolInputSignature(part)}`);
|
|
900
|
+
stall = { kind: "in-flight", toolName: part.toolName };
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
// A "Preparing X action" card is an activity-trail entry, not a content
|
|
904
|
+
// part, so a round can sit on an unstarted action while its content holds
|
|
905
|
+
// nothing but narration.
|
|
906
|
+
if (preparingToolName && !stall) {
|
|
907
|
+
work.add(`preparing ${preparingToolName}`);
|
|
908
|
+
stall = { kind: "preparing", toolName: preparingToolName };
|
|
909
|
+
}
|
|
910
|
+
if (!stall) {
|
|
911
|
+
const text = continuationRepeatSignature(content);
|
|
912
|
+
if (text)
|
|
913
|
+
work.add(`text ${text}`);
|
|
914
|
+
stall = text ? { kind: "repeat" } : { kind: "empty" };
|
|
915
|
+
}
|
|
916
|
+
return { signature: Array.from(work).sort().join(""), stall };
|
|
917
|
+
}
|
|
829
918
|
/**
|
|
830
919
|
* True when an action was streamed but never returned a result yet — i.e. a
|
|
831
920
|
* `tool_start` with no matching `tool_done`. The server is still executing it,
|
|
@@ -1557,36 +1646,30 @@ export function createAgentChatAdapter(options) {
|
|
|
1557
1646
|
let internalContinuationRequest = latestUserIsRecovery;
|
|
1558
1647
|
let startupRecoveryAttempts = 0;
|
|
1559
1648
|
let queuedConflictRetries = 0;
|
|
1560
|
-
let staleRunContinuationAttempts = 0;
|
|
1561
|
-
let stalledTransientContinuationAttempts = 0;
|
|
1562
|
-
let emptyTransientContinuationAttempts = 0;
|
|
1563
1649
|
let totalTransientContinuationAttempts = 0;
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
let
|
|
1567
|
-
//
|
|
1568
|
-
//
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
//
|
|
1572
|
-
//
|
|
1573
|
-
let
|
|
1574
|
-
//
|
|
1575
|
-
//
|
|
1576
|
-
//
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
let
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
//
|
|
1585
|
-
//
|
|
1586
|
-
|
|
1587
|
-
// it needs its own honest message instead of falling through to the
|
|
1588
|
-
// generic "connection kept failing" copy below.
|
|
1589
|
-
let recoveryGaveUpOnTransientBudget = false;
|
|
1650
|
+
// Work-boundary continuations (`loop_limit`), counted apart from the
|
|
1651
|
+
// transient ones — see MAX_LOOP_LIMIT_CONTINUATIONS.
|
|
1652
|
+
let loopLimitContinuations = 0;
|
|
1653
|
+
// Consecutive continuations whose advance signature matched the previous
|
|
1654
|
+
// round's (or was empty). Reset by any real advance.
|
|
1655
|
+
let nonAdvancingContinuations = 0;
|
|
1656
|
+
let lastAdvanceSignature = null;
|
|
1657
|
+
// Set at the moment we give up, so the user-facing message can name what
|
|
1658
|
+
// the turn was stuck on without a counter per shape of stuck.
|
|
1659
|
+
let recoveryStall = null;
|
|
1660
|
+
// Content already accounted for by a previous advance check. Tracked
|
|
1661
|
+
// separately from `visibleContinuationPrefix`, which deliberately does
|
|
1662
|
+
// NOT advance on a `loop_limit` boundary so that round's output stays in
|
|
1663
|
+
// continuation history; the advance check needs the per-round delta for
|
|
1664
|
+
// every reason code, `loop_limit` included.
|
|
1665
|
+
let advanceCheckPrefix = [];
|
|
1666
|
+
// Set when the turn is stopped by a whole-turn continuation ceiling
|
|
1667
|
+
// (MAX_TOTAL_TRANSIENT_CONTINUATIONS or MAX_LOOP_LIMIT_CONTINUATIONS)
|
|
1668
|
+
// rather than by a dropped connection. Either can trip on a turn that was
|
|
1669
|
+
// making real progress the entire time, so it needs its own honest
|
|
1670
|
+
// message instead of falling through to the generic "connection kept
|
|
1671
|
+
// failing" copy below.
|
|
1672
|
+
let recoveryGaveUpOnContinuationBudget = false;
|
|
1590
1673
|
const continuationHistoryFragments = [];
|
|
1591
1674
|
const structuredContinuationFragments = [];
|
|
1592
1675
|
let visibleContinuationPrefix = [];
|
|
@@ -1633,22 +1716,16 @@ export function createAgentChatAdapter(options) {
|
|
|
1633
1716
|
lastRecoverableRunError?.message
|
|
1634
1717
|
? `last_recoverable_error: ${lastRecoverableRunError.message}`
|
|
1635
1718
|
: "",
|
|
1636
|
-
`
|
|
1637
|
-
`
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
`repeated_inflight_tool_stalls: ${repeatedInFlightToolCount}`,
|
|
1641
|
-
lastInFlightToolName
|
|
1642
|
-
? `last_inflight_tool: ${lastInFlightToolName}`
|
|
1643
|
-
: "",
|
|
1644
|
-
`repeated_action_preparation_stalls: ${repeatedActionPreparationCount}`,
|
|
1645
|
-
lastPreparingToolName
|
|
1646
|
-
? `last_preparing_tool: ${lastPreparingToolName}`
|
|
1719
|
+
`non_advancing_continuations: ${nonAdvancingContinuations}`,
|
|
1720
|
+
recoveryStall ? `stalled_on: ${recoveryStall.kind}` : "",
|
|
1721
|
+
recoveryStall?.toolName
|
|
1722
|
+
? `stalled_on_tool: ${recoveryStall.toolName}`
|
|
1647
1723
|
: "",
|
|
1648
1724
|
formatActivityTrail(lastActivityTrail)
|
|
1649
1725
|
? `activity_trail: ${formatActivityTrail(lastActivityTrail)}`
|
|
1650
1726
|
: "",
|
|
1651
1727
|
`total_transient_continuations: ${totalTransientContinuationAttempts}`,
|
|
1728
|
+
`loop_limit_continuations: ${loopLimitContinuations}`,
|
|
1652
1729
|
`background_follow_no_progress_detaches: ${backgroundFollowNoProgressDetaches}`,
|
|
1653
1730
|
`background_follow_consecutive_no_progress_detaches: ${backgroundFollowConsecutiveNoProgressDetaches}`,
|
|
1654
1731
|
backgroundFollowLastDetachReason
|
|
@@ -1667,19 +1744,19 @@ export function createAgentChatAdapter(options) {
|
|
|
1667
1744
|
.join("\n");
|
|
1668
1745
|
};
|
|
1669
1746
|
const exhaustedRecoveryMessage = (reason) => {
|
|
1670
|
-
if (
|
|
1747
|
+
if (recoveryStall?.kind === "in-flight") {
|
|
1671
1748
|
return "The agent got stuck waiting for the same tool to finish, so I stopped the automatic retries. The tool did not report a completed result.";
|
|
1672
1749
|
}
|
|
1673
|
-
if (
|
|
1750
|
+
if (recoveryStall?.kind === "repeat") {
|
|
1674
1751
|
return "The agent got stuck repeating the same response without finishing, so I stopped the automatic retries. This often happens when it tries to re-type a large pasted file into one action — starting a new chat, or asking for a smaller first step, usually gets it unstuck.";
|
|
1675
1752
|
}
|
|
1676
|
-
if (
|
|
1677
|
-
const tool =
|
|
1678
|
-
? ` the ${humanizeActionName(
|
|
1753
|
+
if (recoveryStall?.kind === "preparing") {
|
|
1754
|
+
const tool = recoveryStall.toolName
|
|
1755
|
+
? ` the ${humanizeActionName(recoveryStall.toolName)} action`
|
|
1679
1756
|
: " the same action";
|
|
1680
1757
|
return `The agent got stuck preparing${tool} input and never started the tool, so I stopped the automatic retries. Try a smaller first step or a more compact version of the request.`;
|
|
1681
1758
|
}
|
|
1682
|
-
if (
|
|
1759
|
+
if (recoveryGaveUpOnContinuationBudget) {
|
|
1683
1760
|
return "This turn reached the limit on how many times it can be automatically continued, so I stopped it here. It may have still been making progress — retry as a single, narrower request so it can finish within fewer continuations.";
|
|
1684
1761
|
}
|
|
1685
1762
|
if (content.length === 0 &&
|
|
@@ -1688,6 +1765,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1688
1765
|
reason === "stream_ended")) {
|
|
1689
1766
|
return "The agent request started but did not produce any visible progress before timing out. I stopped the automatic retries so this chat would not stay stuck on Thinking.";
|
|
1690
1767
|
}
|
|
1768
|
+
if (recoveryStall?.kind === "empty") {
|
|
1769
|
+
return "The agent stopped producing new output across several automatic retries, so I stopped it here rather than leaving the chat stuck on Thinking.";
|
|
1770
|
+
}
|
|
1691
1771
|
return "The agent connection kept failing after several automatic recovery attempts.";
|
|
1692
1772
|
};
|
|
1693
1773
|
const dispatchAuthError = (reason) => {
|
|
@@ -1818,14 +1898,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1818
1898
|
attemptedRunIds: [...attemptedRunIds],
|
|
1819
1899
|
activityTrail: [...lastActivityTrail],
|
|
1820
1900
|
startupRecoveryAttempts,
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
repeatedTransientContinuationAttempts,
|
|
1825
|
-
repeatedInFlightToolCount,
|
|
1826
|
-
lastInFlightToolName,
|
|
1827
|
-
repeatedActionPreparationCount,
|
|
1828
|
-
lastPreparingToolName,
|
|
1901
|
+
nonAdvancingContinuations,
|
|
1902
|
+
stalledOn: recoveryStall?.kind,
|
|
1903
|
+
stalledOnTool: recoveryStall?.toolName,
|
|
1829
1904
|
totalTransientContinuationAttempts,
|
|
1830
1905
|
...extra,
|
|
1831
1906
|
},
|
|
@@ -1837,14 +1912,9 @@ export function createAgentChatAdapter(options) {
|
|
|
1837
1912
|
lastSeq,
|
|
1838
1913
|
contentParts: content.length,
|
|
1839
1914
|
startupRecoveryAttempts,
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
repeatedTransientContinuationAttempts,
|
|
1844
|
-
repeatedInFlightToolCount,
|
|
1845
|
-
lastInFlightToolName,
|
|
1846
|
-
repeatedActionPreparationCount,
|
|
1847
|
-
lastPreparingToolName,
|
|
1915
|
+
nonAdvancingContinuations,
|
|
1916
|
+
stalledOn: recoveryStall?.kind,
|
|
1917
|
+
stalledOnTool: recoveryStall?.toolName,
|
|
1848
1918
|
totalTransientContinuationAttempts,
|
|
1849
1919
|
},
|
|
1850
1920
|
},
|
|
@@ -2762,192 +2832,74 @@ export function createAgentChatAdapter(options) {
|
|
|
2762
2832
|
const isTransient = signal.reason !== "loop_limit";
|
|
2763
2833
|
const visibleContent = visibleContentForContinuation();
|
|
2764
2834
|
let currentPartialHistory = contentToContinuationHistory(visibleContent);
|
|
2765
|
-
// Real, content-weight progress: streamed text or a completed tool
|
|
2766
|
-
// result. Used to reset the stalled/empty counters so trivial
|
|
2767
|
-
// whitespace-only output cannot keep the run alive indefinitely.
|
|
2768
2835
|
const madeContentProgress = hasContinuationProgress(visibleContent);
|
|
2769
|
-
//
|
|
2770
|
-
//
|
|
2771
|
-
//
|
|
2772
|
-
//
|
|
2773
|
-
//
|
|
2774
|
-
//
|
|
2775
|
-
|
|
2836
|
+
// THE boundary. One question — did this continuation advance the
|
|
2837
|
+
// turn? — decides every reason code, `loop_limit` included. It used
|
|
2838
|
+
// to skip this block entirely AND reset two of the budgets, so a
|
|
2839
|
+
// model that degenerated into a tool loop re-POSTed the same turnId
|
|
2840
|
+
// until the user gave up (measured in production at 186 minutes with
|
|
2841
|
+
// no answer). The durable per-turn ledger lives server-side inside
|
|
2842
|
+
// one run and never sees a client re-POST, so nothing else bounds it.
|
|
2843
|
+
const advanceDelta = contentAfterContinuationPrefix(content, advanceCheckPrefix);
|
|
2844
|
+
// Everything that asks "what did THIS round do" reads the delta.
|
|
2845
|
+
// `visibleContent` accumulates across a `loop_limit` chain — its
|
|
2846
|
+
// prefix deliberately does not advance there — so one unresolved
|
|
2847
|
+
// "Preparing X" card from an earlier round would otherwise stall
|
|
2848
|
+
// every later round on the same name and kill a turn that was
|
|
2849
|
+
// streaming new prose the whole time.
|
|
2850
|
+
// An action streamed but not returned yet (a tool_start with no
|
|
2851
|
+
// tool_done) means the SERVER is still executing it, so a run_timeout
|
|
2852
|
+
// in that window can still deliver the result on reconnect; every
|
|
2853
|
+
// other reason means the stream is dead and nothing will complete it.
|
|
2854
|
+
const hasInFlightTool = hasInFlightToolCall(advanceDelta);
|
|
2776
2855
|
const completedTool = lastCompletedTimeoutCandidateTool(content);
|
|
2777
|
-
|
|
2778
|
-
// for the stalled/empty caps.
|
|
2779
|
-
const madeProgress = madeContentProgress || hasInFlightTool;
|
|
2780
|
-
const emptyRun = !madeProgress && signal.reason !== "loop_limit";
|
|
2781
|
-
const madeDurableToolProgress = visibleContent.some((part) => part.type === "tool-call" &&
|
|
2782
|
-
part.activity !== true &&
|
|
2783
|
-
part.result !== undefined);
|
|
2784
|
-
const currentPreparingToolName = lastUnresolvedToolActivity(visibleContent) ??
|
|
2856
|
+
const currentPreparingToolName = lastUnresolvedToolActivity(advanceDelta) ??
|
|
2785
2857
|
lastPreparingActionTool(signal.activityTrail);
|
|
2786
|
-
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
//
|
|
2792
|
-
//
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
p.result === undefined &&
|
|
2801
|
-
p.activity !== true);
|
|
2802
|
-
const currentInFlightToolName = currentInFlightToolPart?.toolName;
|
|
2803
|
-
const currentInFlightToolSignature = currentInFlightToolPart
|
|
2804
|
-
? inFlightToolInputSignature(currentInFlightToolPart)
|
|
2805
|
-
: undefined;
|
|
2806
|
-
const isBrokenInFlightTool = signal.reason === "stream_ended" || signal.reason === "no_progress";
|
|
2807
|
-
if (currentInFlightToolName && isBrokenInFlightTool) {
|
|
2808
|
-
if (currentInFlightToolName === lastInFlightToolName &&
|
|
2809
|
-
currentInFlightToolSignature === lastInFlightToolSignature) {
|
|
2810
|
-
repeatedInFlightToolCount += 1;
|
|
2811
|
-
}
|
|
2812
|
-
else {
|
|
2813
|
-
// New tool, or the same tool retried with a CHANGED (e.g.
|
|
2814
|
-
// smaller) payload — give the changed input a fresh stall budget
|
|
2815
|
-
// instead of aborting it as a repeat of the prior payload.
|
|
2816
|
-
repeatedInFlightToolCount = 0;
|
|
2817
|
-
lastInFlightToolName = currentInFlightToolName;
|
|
2818
|
-
lastInFlightToolSignature = currentInFlightToolSignature;
|
|
2819
|
-
}
|
|
2820
|
-
}
|
|
2821
|
-
else if (!currentInFlightToolName) {
|
|
2822
|
-
repeatedInFlightToolCount = 0;
|
|
2858
|
+
const advance = describeContinuationAdvance(advanceDelta, currentPreparingToolName);
|
|
2859
|
+
const serverStillRunningTool = signal.reason === "run_timeout" && hasInFlightTool;
|
|
2860
|
+
const advanced = serverStillRunningTool ||
|
|
2861
|
+
(advance.signature !== "" &&
|
|
2862
|
+
advance.signature !== lastAdvanceSignature);
|
|
2863
|
+
// Only remember a non-empty signature, so an empty round between two
|
|
2864
|
+
// identical rounds cannot launder the second one into "new work".
|
|
2865
|
+
if (advance.signature)
|
|
2866
|
+
lastAdvanceSignature = advance.signature;
|
|
2867
|
+
nonAdvancingContinuations = advanced
|
|
2868
|
+
? 0
|
|
2869
|
+
: nonAdvancingContinuations + 1;
|
|
2870
|
+
if (isTransient) {
|
|
2871
|
+
totalTransientContinuationAttempts += 1;
|
|
2823
2872
|
}
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
!hasInFlightTool &&
|
|
2827
|
-
!madeDurableToolProgress;
|
|
2828
|
-
if (isRepeatedActionPreparationCandidate) {
|
|
2829
|
-
if (currentPreparingToolName === lastPreparingToolName) {
|
|
2830
|
-
repeatedActionPreparationCount += 1;
|
|
2831
|
-
}
|
|
2832
|
-
else {
|
|
2833
|
-
repeatedActionPreparationCount = 0;
|
|
2834
|
-
lastPreparingToolName = currentPreparingToolName;
|
|
2835
|
-
}
|
|
2873
|
+
else {
|
|
2874
|
+
loopLimitContinuations += 1;
|
|
2836
2875
|
}
|
|
2837
|
-
|
|
2838
|
-
|
|
2839
|
-
|
|
2840
|
-
|
|
2841
|
-
|
|
2876
|
+
advanceCheckPrefix = snapshotContent(content);
|
|
2877
|
+
// If a tool already completed, do not turn a missing closing
|
|
2878
|
+
// sentence into a scary connection failure. Give the model one
|
|
2879
|
+
// continuation opportunity (the completed tool itself counts as an
|
|
2880
|
+
// advance on the first timeout); if the follow-up produces no new
|
|
2881
|
+
// content, stop locally with a clear completed-tool warning.
|
|
2882
|
+
if (signal.reason === "run_timeout" &&
|
|
2883
|
+
completedTool &&
|
|
2884
|
+
!hasInFlightToolCall(content) &&
|
|
2885
|
+
!currentPreparingToolName &&
|
|
2886
|
+
!madeContentProgress &&
|
|
2887
|
+
!hasInFlightTool) {
|
|
2888
|
+
return {
|
|
2889
|
+
ok: false,
|
|
2890
|
+
resetVisibleContent: false,
|
|
2891
|
+
completedToolName: completedTool.toolName,
|
|
2892
|
+
};
|
|
2842
2893
|
}
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
|
|
2846
|
-
// true and the stalled/empty budgets never trip. Compare this round's
|
|
2847
|
-
// unique-sentence signature to the previous round's; a match with no
|
|
2848
|
-
// tool progress is a non-advancing loop. Tracked by its own counter
|
|
2849
|
-
// so it bails after a few rounds instead of the full transient budget,
|
|
2850
|
-
// without perturbing the stalled/empty/stale accounting.
|
|
2851
|
-
const repeatSignature = continuationRepeatSignature(visibleContent);
|
|
2852
|
-
const isNonAdvancingRepeat = signal.reason !== "loop_limit" &&
|
|
2853
|
-
repeatSignature !== "" &&
|
|
2854
|
-
repeatSignature === lastContinuationRepeatSignature &&
|
|
2855
|
-
!hasInFlightTool &&
|
|
2856
|
-
!madeDurableToolProgress;
|
|
2857
|
-
if (signal.reason === "loop_limit") {
|
|
2858
|
-
stalledTransientContinuationAttempts = 0;
|
|
2859
|
-
emptyTransientContinuationAttempts = 0;
|
|
2894
|
+
if (nonAdvancingContinuations >= MAX_NON_ADVANCING_CONTINUATIONS) {
|
|
2895
|
+
recoveryStall = advance.stall;
|
|
2896
|
+
return { ok: false, resetVisibleContent: false };
|
|
2860
2897
|
}
|
|
2861
|
-
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
// progress on the first timeout); if the follow-up produces no new
|
|
2867
|
-
// content, stop locally with a clear completed-tool warning.
|
|
2868
|
-
if (signal.reason === "run_timeout" &&
|
|
2869
|
-
completedTool &&
|
|
2870
|
-
!hasInFlightToolCall(content) &&
|
|
2871
|
-
!currentPreparingToolName &&
|
|
2872
|
-
!madeContentProgress &&
|
|
2873
|
-
!hasInFlightTool) {
|
|
2874
|
-
return {
|
|
2875
|
-
ok: false,
|
|
2876
|
-
resetVisibleContent: false,
|
|
2877
|
-
completedToolName: completedTool.toolName,
|
|
2878
|
-
};
|
|
2879
|
-
}
|
|
2880
|
-
// Bail when the same write tool is stuck in-flight across too many
|
|
2881
|
-
// consecutive continuations. Checked before the text-repeat guard
|
|
2882
|
-
// because hasInFlightTool=true would mask the repeat as progress.
|
|
2883
|
-
if (repeatedInFlightToolCount >= MAX_REPEATED_INFLIGHT_TOOL_STALLS) {
|
|
2884
|
-
recoveryGaveUpOnInFlightTool = true;
|
|
2885
|
-
return { ok: false, resetVisibleContent: false };
|
|
2886
|
-
}
|
|
2887
|
-
if (repeatedActionPreparationCount >
|
|
2888
|
-
MAX_REPEATED_ACTION_PREPARATION_CONTINUATIONS) {
|
|
2889
|
-
recoveryGaveUpOnActionPreparation = true;
|
|
2890
|
-
return { ok: false, resetVisibleContent: false };
|
|
2891
|
-
}
|
|
2892
|
-
// Bail fast on a non-advancing repetition loop, well before the
|
|
2893
|
-
// stalled/empty/total budgets would (each round otherwise re-sends
|
|
2894
|
-
// the whole pasted payload). Tracked separately so it never trips
|
|
2895
|
-
// on legitimately-progressing runs that happen to be slow.
|
|
2896
|
-
if (isNonAdvancingRepeat) {
|
|
2897
|
-
repeatedTransientContinuationAttempts += 1;
|
|
2898
|
-
if (repeatedTransientContinuationAttempts >
|
|
2899
|
-
MAX_REPEATED_TRANSIENT_CONTINUATIONS) {
|
|
2900
|
-
recoveryGaveUpOnRepetition = true;
|
|
2901
|
-
return { ok: false, resetVisibleContent: false };
|
|
2902
|
-
}
|
|
2903
|
-
}
|
|
2904
|
-
else {
|
|
2905
|
-
repeatedTransientContinuationAttempts = 0;
|
|
2906
|
-
}
|
|
2907
|
-
if (repeatSignature) {
|
|
2908
|
-
lastContinuationRepeatSignature = repeatSignature;
|
|
2909
|
-
}
|
|
2910
|
-
// A run_timeout that produced nothing visible and no activity (the
|
|
2911
|
-
// model spent the whole soft-timeout window thinking before its
|
|
2912
|
-
// first output) is NOT an immediate give-up: a transient slow start
|
|
2913
|
-
// routinely recovers on the next continuation. Let it fall through
|
|
2914
|
-
// to the empty-continuation budget below so it retries a bounded
|
|
2915
|
-
// number of times before surfacing the "no visible progress" error.
|
|
2916
|
-
// Reset the empty-continuation counter on real progress — streamed
|
|
2917
|
-
// text/completed tool OR an in-flight tool the server is running —
|
|
2918
|
-
// not merely on a non-zero part count, which whitespace-only or
|
|
2919
|
-
// unresolved-only output would falsely satisfy.
|
|
2920
|
-
if (!madeProgress) {
|
|
2921
|
-
emptyTransientContinuationAttempts += 1;
|
|
2922
|
-
if (emptyTransientContinuationAttempts >
|
|
2923
|
-
MAX_EMPTY_TRANSIENT_CONTINUATIONS) {
|
|
2924
|
-
return { ok: false, resetVisibleContent: false };
|
|
2925
|
-
}
|
|
2926
|
-
}
|
|
2927
|
-
else {
|
|
2928
|
-
emptyTransientContinuationAttempts = 0;
|
|
2929
|
-
}
|
|
2930
|
-
if (signal.reason === "stale_run") {
|
|
2931
|
-
staleRunContinuationAttempts = madeDurableToolProgress
|
|
2932
|
-
? 0
|
|
2933
|
-
: staleRunContinuationAttempts + 1;
|
|
2934
|
-
if (staleRunContinuationAttempts > MAX_STALE_RUN_CONTINUATIONS) {
|
|
2935
|
-
return { ok: false, resetVisibleContent: false };
|
|
2936
|
-
}
|
|
2937
|
-
}
|
|
2938
|
-
stalledTransientContinuationAttempts = madeProgress
|
|
2939
|
-
? 0
|
|
2940
|
-
: stalledTransientContinuationAttempts + 1;
|
|
2941
|
-
if (totalTransientContinuationAttempts >
|
|
2942
|
-
MAX_TOTAL_TRANSIENT_CONTINUATIONS) {
|
|
2943
|
-
recoveryGaveUpOnTransientBudget = true;
|
|
2944
|
-
}
|
|
2945
|
-
if (stalledTransientContinuationAttempts >
|
|
2946
|
-
MAX_STALLED_TRANSIENT_CONTINUATIONS ||
|
|
2947
|
-
totalTransientContinuationAttempts >
|
|
2948
|
-
MAX_TOTAL_TRANSIENT_CONTINUATIONS) {
|
|
2949
|
-
return { ok: false, resetVisibleContent: false };
|
|
2950
|
-
}
|
|
2898
|
+
if (totalTransientContinuationAttempts >
|
|
2899
|
+
MAX_TOTAL_TRANSIENT_CONTINUATIONS ||
|
|
2900
|
+
loopLimitContinuations > MAX_LOOP_LIMIT_CONTINUATIONS) {
|
|
2901
|
+
recoveryGaveUpOnContinuationBudget = true;
|
|
2902
|
+
return { ok: false, resetVisibleContent: false };
|
|
2951
2903
|
}
|
|
2952
2904
|
if (isTransient) {
|
|
2953
2905
|
const settledInterruptedTools = settleInterruptedToolCalls(visibleContent, undefined, { includeActivity: true });
|
|
@@ -2997,13 +2949,21 @@ export function createAgentChatAdapter(options) {
|
|
|
2997
2949
|
startupRecoveryAttempts = 0;
|
|
2998
2950
|
clearActiveRun();
|
|
2999
2951
|
if (!isTransient) {
|
|
3000
|
-
return {
|
|
2952
|
+
return {
|
|
2953
|
+
ok: true,
|
|
2954
|
+
resetVisibleContent: false,
|
|
2955
|
+
nonAdvancing: !advanced,
|
|
2956
|
+
};
|
|
3001
2957
|
}
|
|
3002
2958
|
// Keep everything visible during transient recovery. The continuation
|
|
3003
2959
|
// prefix diff tracks what the next request has already seen, so
|
|
3004
2960
|
// preserving text no longer causes duplicate continuation history.
|
|
3005
2961
|
visibleContinuationPrefix = snapshotContent(content);
|
|
3006
|
-
return {
|
|
2962
|
+
return {
|
|
2963
|
+
ok: true,
|
|
2964
|
+
resetVisibleContent: false,
|
|
2965
|
+
nonAdvancing: !advanced,
|
|
2966
|
+
};
|
|
3007
2967
|
};
|
|
3008
2968
|
while (true) {
|
|
3009
2969
|
try {
|
|
@@ -3410,11 +3370,11 @@ export function createAgentChatAdapter(options) {
|
|
|
3410
3370
|
detail: { tabId },
|
|
3411
3371
|
}));
|
|
3412
3372
|
}
|
|
3413
|
-
// A run that
|
|
3373
|
+
// A run that advanced nothing usually never got past
|
|
3414
3374
|
// startup/gateway; re-POSTing the identical payload immediately
|
|
3415
3375
|
// just hits the identical wall. Back that case off instead.
|
|
3416
|
-
if (continuation.
|
|
3417
|
-
await retryDelay(Math.max(0,
|
|
3376
|
+
if (continuation.nonAdvancing) {
|
|
3377
|
+
await retryDelay(Math.max(0, nonAdvancingContinuations - 1), abortSignal);
|
|
3418
3378
|
}
|
|
3419
3379
|
else {
|
|
3420
3380
|
await delay(250, abortSignal);
|