@memberjunction/ai-agents 6.2.0-edge.3 → 6.2.0-edge.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/README.md +6 -0
  2. package/dist/AgentRunner.d.ts +49 -2
  3. package/dist/AgentRunner.d.ts.map +1 -1
  4. package/dist/AgentRunner.js +114 -21
  5. package/dist/AgentRunner.js.map +1 -1
  6. package/dist/ConversationCompactionManager.d.ts +2 -0
  7. package/dist/ConversationCompactionManager.d.ts.map +1 -1
  8. package/dist/ConversationCompactionManager.js +2 -2
  9. package/dist/ConversationCompactionManager.js.map +1 -1
  10. package/dist/ConversationToolManager.d.ts +13 -2
  11. package/dist/ConversationToolManager.d.ts.map +1 -1
  12. package/dist/ConversationToolManager.js +19 -3
  13. package/dist/ConversationToolManager.js.map +1 -1
  14. package/dist/KnowledgeAgent.d.ts.map +1 -1
  15. package/dist/KnowledgeAgent.js +6 -1
  16. package/dist/KnowledgeAgent.js.map +1 -1
  17. package/dist/MJAIAgentRequestEntityServer.d.ts +6 -0
  18. package/dist/MJAIAgentRequestEntityServer.d.ts.map +1 -1
  19. package/dist/MJAIAgentRequestEntityServer.js +26 -1
  20. package/dist/MJAIAgentRequestEntityServer.js.map +1 -1
  21. package/dist/agent-run-watchdog.d.ts +65 -5
  22. package/dist/agent-run-watchdog.d.ts.map +1 -1
  23. package/dist/agent-run-watchdog.js +143 -28
  24. package/dist/agent-run-watchdog.js.map +1 -1
  25. package/dist/agent-types/loop-agent-prompt-params.d.ts +7 -0
  26. package/dist/agent-types/loop-agent-prompt-params.d.ts.map +1 -1
  27. package/dist/agent-types/loop-agent-prompt-params.js +1 -0
  28. package/dist/agent-types/loop-agent-prompt-params.js.map +1 -1
  29. package/dist/artifact-tools/SearchResultSetToolLibrary.js +4 -4
  30. package/dist/artifact-tools/SearchResultSetToolLibrary.js.map +1 -1
  31. package/dist/base-agent.d.ts +275 -18
  32. package/dist/base-agent.d.ts.map +1 -1
  33. package/dist/base-agent.js +761 -216
  34. package/dist/base-agent.js.map +1 -1
  35. package/dist/client-tool-results.d.ts +45 -0
  36. package/dist/client-tool-results.d.ts.map +1 -0
  37. package/dist/client-tool-results.js +94 -0
  38. package/dist/client-tool-results.js.map +1 -0
  39. package/dist/index.d.ts +5 -0
  40. package/dist/index.d.ts.map +1 -1
  41. package/dist/index.js +5 -0
  42. package/dist/index.js.map +1 -1
  43. package/dist/prior-turn-tool-result-cache.d.ts +31 -12
  44. package/dist/prior-turn-tool-result-cache.d.ts.map +1 -1
  45. package/dist/prior-turn-tool-result-cache.js +59 -16
  46. package/dist/prior-turn-tool-result-cache.js.map +1 -1
  47. package/dist/realtime/bridge-realtime-session-factory.d.ts +44 -2
  48. package/dist/realtime/bridge-realtime-session-factory.d.ts.map +1 -1
  49. package/dist/realtime/bridge-realtime-session-factory.js +68 -20
  50. package/dist/realtime/bridge-realtime-session-factory.js.map +1 -1
  51. package/dist/realtime/bridge-realtime-usage-recorder.d.ts +81 -0
  52. package/dist/realtime/bridge-realtime-usage-recorder.d.ts.map +1 -0
  53. package/dist/realtime/bridge-realtime-usage-recorder.js +194 -0
  54. package/dist/realtime/bridge-realtime-usage-recorder.js.map +1 -0
  55. package/dist/realtime/client-only-channel-server.d.ts +83 -0
  56. package/dist/realtime/client-only-channel-server.d.ts.map +1 -0
  57. package/dist/realtime/client-only-channel-server.js +107 -0
  58. package/dist/realtime/client-only-channel-server.js.map +1 -0
  59. package/dist/realtime/identity-verification-channel-server.d.ts +50 -0
  60. package/dist/realtime/identity-verification-channel-server.d.ts.map +1 -0
  61. package/dist/realtime/identity-verification-channel-server.js +66 -0
  62. package/dist/realtime/identity-verification-channel-server.js.map +1 -0
  63. package/dist/realtime/interactive-component-channel-server.d.ts +39 -0
  64. package/dist/realtime/interactive-component-channel-server.d.ts.map +1 -0
  65. package/dist/realtime/interactive-component-channel-server.js +55 -0
  66. package/dist/realtime/interactive-component-channel-server.js.map +1 -0
  67. package/dist/realtime/realtime-avatar-resolution.d.ts +194 -0
  68. package/dist/realtime/realtime-avatar-resolution.d.ts.map +1 -0
  69. package/dist/realtime/realtime-avatar-resolution.js +208 -0
  70. package/dist/realtime/realtime-avatar-resolution.js.map +1 -0
  71. package/dist/realtime/realtime-channel-policy.d.ts +81 -0
  72. package/dist/realtime/realtime-channel-policy.d.ts.map +1 -0
  73. package/dist/realtime/realtime-channel-policy.js +91 -0
  74. package/dist/realtime/realtime-channel-policy.js.map +1 -0
  75. package/dist/realtime/realtime-channel-server-host.d.ts +14 -0
  76. package/dist/realtime/realtime-channel-server-host.d.ts.map +1 -1
  77. package/dist/realtime/realtime-channel-server-host.js +35 -0
  78. package/dist/realtime/realtime-channel-server-host.js.map +1 -1
  79. package/dist/realtime/realtime-client-session-service.d.ts +430 -30
  80. package/dist/realtime/realtime-client-session-service.d.ts.map +1 -1
  81. package/dist/realtime/realtime-client-session-service.js +766 -111
  82. package/dist/realtime/realtime-client-session-service.js.map +1 -1
  83. package/dist/realtime/realtime-coagent-config.d.ts +94 -15
  84. package/dist/realtime/realtime-coagent-config.d.ts.map +1 -1
  85. package/dist/realtime/realtime-coagent-config.js +109 -4
  86. package/dist/realtime/realtime-coagent-config.js.map +1 -1
  87. package/dist/realtime/realtime-recording-store.d.ts +136 -16
  88. package/dist/realtime/realtime-recording-store.d.ts.map +1 -1
  89. package/dist/realtime/realtime-recording-store.js +505 -39
  90. package/dist/realtime/realtime-recording-store.js.map +1 -1
  91. package/dist/realtime/realtime-session-runner.js +4 -4
  92. package/dist/realtime/realtime-session-runner.js.map +1 -1
  93. package/dist/realtime/realtime-vendor-resolution.d.ts +9 -0
  94. package/dist/realtime/realtime-vendor-resolution.d.ts.map +1 -1
  95. package/dist/realtime/realtime-vendor-resolution.js +16 -5
  96. package/dist/realtime/realtime-vendor-resolution.js.map +1 -1
  97. package/dist/realtime/realtime-video-output-gate.d.ts +83 -0
  98. package/dist/realtime/realtime-video-output-gate.d.ts.map +1 -0
  99. package/dist/realtime/realtime-video-output-gate.js +79 -0
  100. package/dist/realtime/realtime-video-output-gate.js.map +1 -0
  101. package/dist/tool-result-format.d.ts +14 -0
  102. package/dist/tool-result-format.d.ts.map +1 -1
  103. package/dist/tool-result-format.js.map +1 -1
  104. package/package.json +19 -19
@@ -10,12 +10,12 @@
10
10
  * @author MemberJunction.com
11
11
  * @since 2.49.0
12
12
  */
13
- import { FileStorageEngineBase, MJEnvironmentEntityExtended } from '@memberjunction/core-entities';
13
+ import { MJEnvironmentEntityExtended, ConversationEngine } from '@memberjunction/core-entities';
14
14
  import { BuildActionToolSet, FilterDeclarableActions, SanitizeToolName } from './native-tools/action-tool-builder.js';
15
15
  import { BuildNativeToolSet, COMPLETE_TASK_TOOL, SUB_AGENT_TOOL_PREFIX } from './native-tools/control-tools.js';
16
16
  import { BuildAssistantToolCallTurn, BuildToolResultTurn, CompactToolResultContent } from './native-tools/tool-result-turns.js';
17
17
  import { LooksLikeLoopEnvelope } from './native-tools/dual-channel.js';
18
- import { ResolvePromptRunUserID } from "@memberjunction/ai-core-plus";
18
+ import { MJAIAgentRunEntityExtended, ResolvePromptRunUserID } from "@memberjunction/ai-core-plus";
19
19
  import { Metadata, RunView, LogStatus, LogStatusEx, LogError, LogErrorEx, IsVerboseLoggingEnabled, DatabaseProviderBase, ToPlainJSON } from '@memberjunction/core';
20
20
  import { AgentRunWatchdog } from './agent-run-watchdog.js';
21
21
  import { AIPromptRunner, GetToolCallingDecision } from '@memberjunction/ai-prompts';
@@ -23,7 +23,7 @@ import { BaseRealtimeModel, GetAIAPIKey, CredentialScopeAllows, MakeAIAPIKeyReso
23
23
  import { BaseAgentType } from './agent-types/base-agent-type.js';
24
24
  import { ProviderRubricEngine } from '@memberjunction/rubrics';
25
25
  import { ExecuteSelfCheck, PickSelfCheckLink } from './self-check.js';
26
- import { CopyScalarsAndArrays, JSONValidator, MJGlobal, NormalizeUUID, SafeExpressionEvaluator, UUIDsEqual, EscapeSQLString, IsPlainObject, CleanAndParseJSON } from '@memberjunction/global';
26
+ import { CopyScalarsAndArrays, JSONValidator, MJGlobal, NormalizeUUID, SafeExpressionEvaluator, UUIDsEqual, EscapeSQLString, IsPlainObject, CleanAndParseJSON, IsValidUUID } from '@memberjunction/global';
27
27
  // token optimization via @memberjunction/context-crush (SmartCrusher/CacheAligner-inspired)
28
28
  import { CrushJSON, DescribeCrush, PartitionStablePrefix } from '@memberjunction/context-crush';
29
29
  // AST-aware code reduction (CodeCompressor-inspired) — opt-in per agent type
@@ -31,7 +31,9 @@ import { CrushCode } from '@memberjunction/context-crush/code';
31
31
  import { RealtimeSessionRunner } from './realtime/realtime-session-runner.js';
32
32
  import { ResolveNarrationInstructionsTemplate } from './realtime/realtime-narration.js';
33
33
  import { BuildRealtimeOverridesJson, BuildVoiceMannerSection, GetNarrationPaceMs, GetProviderVoiceSettings, GetSessionTuningSettings, GetModelCatalogSessionSettings, DeepMergeConfigs, ResolveEffectiveRealtimeConfig } from './realtime/realtime-coagent-config.js';
34
- import { SelectRealtimeVendorForModel } from './realtime/realtime-vendor-resolution.js';
34
+ import { ListRealtimeVendorsForModel, SelectRealtimeVendorForModel } from './realtime/realtime-vendor-resolution.js';
35
+ import { ChooseRealtimeVendorForAvatar, RealtimeModelShowsAvatar } from './realtime/realtime-video-output-gate.js';
36
+ import { RealtimeSessionCanShowAvatar } from './realtime/realtime-avatar-resolution.js';
35
37
  import { RealtimeClientSessionService, WarnOnUnmatchedProviderVoice } from './realtime/realtime-client-session-service.js';
36
38
  import { BuildRealtimeAgentFraming } from './realtime/realtime-tool-broker.js';
37
39
  import { ReadHostTools, ReadTrimmedString } from './realtime/bridge-host-params.js';
@@ -71,6 +73,7 @@ import { PipelineExecutor, PipelineToolRegistry, ActionInvocable, ArtifactToolIn
71
73
  import { GetValueFromPath, SetMappedValue } from '@memberjunction/ai-core-plus';
72
74
  import { AgentDataPreloader } from './AgentDataPreloader.js';
73
75
  import { ClientToolRequestManager } from './ClientToolRequestManager.js';
76
+ import { BuildClientToolResultMessage, ResolveClientToolResultMaxChars, ReplaceMediaBlocksWithStubs, IsMediaBlock, IMAGE_BLOCK_TOKEN_ESTIMATE } from './client-tool-results.js';
74
77
  import { ConversationMessageResolver } from './utils/ConversationMessageResolver.js';
75
78
  import _ from 'lodash';
76
79
  /**
@@ -210,6 +213,13 @@ export class BaseAgent {
210
213
  */
211
214
  this._generalValidationRetryCount = 0;
212
215
  this._agentRun = null;
216
+ /**
217
+ * The abort controller that governs the current `Execute` call (the merged timeout/upstream
218
+ * signal). Held so {@link initializeAgentRun} can register it with the watchdog's stop relay:
219
+ * when someone marks this run's row `Cancelled`, the watchdog aborts it and the loop's
220
+ * existing cancellation checks end the run. Null outside an execution.
221
+ */
222
+ this._runAbortController = null;
213
223
  /**
214
224
  * The task graph this run submitted and is now waiting on, or null.
215
225
  *
@@ -317,6 +327,11 @@ export class BaseAgent {
317
327
  * Armed only when the run has a conversationId (the cross-turn context gate).
318
328
  */
319
329
  this._conversationToolManager = new ConversationToolManager();
330
+ /**
331
+ * The conversation's branch rows for the current run, loaded once when the run is on a
332
+ * branch (`ExecuteAgentParams.ConversationBranchID`). Empty on the trunk.
333
+ */
334
+ this._conversationBranches = [];
320
335
  /**
321
336
  * Manages in-flight durable memory writes for the current agent run.
322
337
  * Only consulted when the agent has AllowMemoryWrite enabled.
@@ -823,6 +838,12 @@ export class BaseAgent {
823
838
  * on the right one, and neither shows up as an error.
824
839
  */
825
840
  static { this.DEFAULT_ABSOLUTE_MAX_ITERATIONS = 5000; }
841
+ /**
842
+ * The prefix of the abort reason the wall-clock guard in {@link Execute} raises when a run
843
+ * outlives `maxExecutionTimeMs`. The guard appends the agent and the budget after it; a reader
844
+ * ({@link CancellationReasonForAbort}) checks the prefix, never the wording after it.
845
+ */
846
+ static { this.AgentTimeoutAbortReason = 'Agent execution exceeded maxExecutionTimeMs'; }
826
847
  /**
827
848
  * The resolved FileStorageAccount ID for this agent run. Set during Execute()
828
849
  * via the hierarchical resolution chain (Runtime → Agent → Category → Type → fallback).
@@ -1481,13 +1502,15 @@ export class BaseAgent {
1481
1502
  }
1482
1503
  const timeoutId = setTimeout(() => {
1483
1504
  if (!timeoutController.signal.aborted) {
1484
- timeoutController.abort(`Agent '${params.agent.Name}' exceeded maxExecutionTimeMs (${agentTimeoutMS}ms)`);
1505
+ timeoutController.abort(`${BaseAgent.AgentTimeoutAbortReason}: agent '${params.agent.Name}' ran past ${agentTimeoutMS}ms`);
1485
1506
  }
1486
1507
  }, agentTimeoutMS);
1487
1508
  // Route the merged signal back through `params` so the existing body of
1488
1509
  // Execute (and downstream sub-agent invocations that propagate
1489
1510
  // `cancellationToken`) observe it.
1490
1511
  params.cancellationToken = timeoutController.signal;
1512
+ // Hold it for the watchdog's stop relay (registered once the run row exists).
1513
+ this._runAbortController = timeoutController;
1491
1514
  try {
1492
1515
  this.logStatus(`🤖 Starting execution of agent '${params.agent.Name}'`, true, params);
1493
1516
  // =====================================================================================
@@ -1546,7 +1569,11 @@ export class BaseAgent {
1546
1569
  // Arm conversation-history retrieval tools — available only when the run has a
1547
1570
  // conversation to page against (the same gate as all cross-turn context features).
1548
1571
  // A history floor holds here too: the tools page only rows from it onward.
1549
- this._conversationToolManager.Initialize(wrappedParams.conversationId || null, params.contextUser, wrappedParams.ConversationHistoryFrom ?? null);
1572
+ // On a branch, the tools page only rows on that branch's path.
1573
+ this._conversationBranches = wrappedParams.conversationId && wrappedParams.ConversationBranchID
1574
+ ? await ConversationEngine.LoadBranchesFresh(wrappedParams.conversationId, params.contextUser, this.ProviderToUse)
1575
+ : [];
1576
+ this._conversationToolManager.Initialize(wrappedParams.conversationId || null, params.contextUser, wrappedParams.ConversationHistoryFrom ?? null, wrappedParams.ConversationBranchID ?? null, this._conversationBranches);
1550
1577
  this._conversationToolManager.SetSummaryHost(this.buildConversationSummaryHost(wrappedParams));
1551
1578
  // Initialize artifact tools with any input artifacts attached to the run.
1552
1579
  // Artifacts arrive as a typed first-class field on ExecuteAgentParams —
@@ -1675,7 +1702,11 @@ export class BaseAgent {
1675
1702
  primaryScopeEntityId = primaryEntity.ID;
1676
1703
  }
1677
1704
  }
1678
- const [config] = await Promise.all([
1705
+ // The user's new message, captured by identity before anything below adds to the list, so a
1706
+ // resumed stopped run can be spliced in ahead of it once the parallel phase settles.
1707
+ const continuationMessage = wrappedParams.conversationMessages[wrappedParams.conversationMessages.length - 1];
1708
+ // Positions matter: [0] is the config and [5] the stopped-run continuation.
1709
+ const [config, , , , , stoppedRunContinuation] = await Promise.all([
1679
1710
  this.loadAgentConfiguration(params.agent),
1680
1711
  this.preloadAgentData(wrappedParams),
1681
1712
  this.InjectContextMemory(typeof inputText === 'string' ? inputText : '', params.agent, userId, companyId, params.contextUser, wrappedParams.conversationMessages, primaryScopeEntityId, primaryScopeRecordId, secondaryScopes, scopeConfig),
@@ -1684,10 +1715,15 @@ export class BaseAgent {
1684
1715
  // conversationId). Runs here so the results are in the messages before
1685
1716
  // the pre-turn compaction check and the first prompt.
1686
1717
  this.injectPriorTurnToolResults(wrappedParams),
1718
+ // If the previous turn was STOPPED by the user, load what it did so this turn continues
1719
+ // the work instead of redoing it. Applied below, after this phase settles.
1720
+ this.loadStoppedRunContinuation(wrappedParams),
1687
1721
  // Decision discovery (plan Task 3.1), off unless the decisionsEnabled and decisionDiscovery prompt
1688
1722
  // params are true: may add a <suggested_agent> system message, within DECISION_DISCOVERY_TIMEOUT_MS.
1689
1723
  this.InjectDecisionDiscovery(params.agent, params.contextUser, wrappedParams.conversationMessages, params.data)
1690
1724
  ]);
1725
+ // Resume (or summarize) a run the user stopped, now that nothing else is writing the list.
1726
+ this.applyStoppedRunContinuation(wrappedParams, stoppedRunContinuation, continuationMessage);
1691
1727
  // Inject scope-resolved prompt parts (role-faithful) for this agent's prompt, alongside
1692
1728
  // memory/RAG. Synchronous — parts are cached on AIEngine. Uses the same run scope.
1693
1729
  this.InjectScopedPromptParts(params.agent, wrappedParams.conversationMessages, primaryScopeEntityId, primaryScopeRecordId, secondaryScopes);
@@ -1775,6 +1811,7 @@ export class BaseAgent {
1775
1811
  // consumers that re-read `params` after the call see what they
1776
1812
  // passed in, not our chained signal.
1777
1813
  params.cancellationToken = upstreamToken;
1814
+ this._runAbortController = null;
1778
1815
  // A cancellation, or an error thrown out of the step loop, leaves any held decision
1779
1816
  // requests unasked. The terminate path has already logged and cleared its own.
1780
1817
  this.skipHeldDecisions(params, this.runEndForHeldDecisions());
@@ -1954,10 +1991,16 @@ export class BaseAgent {
1954
1991
  ? (params.data?.realtimeSelfNames).filter((n) => typeof n === 'string')
1955
1992
  : undefined;
1956
1993
  const hostTools = ReadHostTools(params.data?.realtimeHostTools);
1994
+ // Note: resolveHostTools is an in-process callback function passed directly in params.data.
1995
+ // It will not survive process boundaries or cross-host job queue serialization.
1996
+ const resolveHostTools = typeof params.data?.resolveHostTools === 'function'
1997
+ ? params.data.resolveHostTools
1998
+ : undefined;
1957
1999
  return {
1958
2000
  CoAgent: params.agent,
1959
2001
  TargetAgentID: targetID,
1960
2002
  HostTools: hostTools,
2003
+ ResolveHostTools: resolveHostTools,
1961
2004
  HostFraming: ReadTrimmedString(params.data?.realtimeHostFraming, 'realtimeHostFraming'),
1962
2005
  PriorTranscript: ReadTrimmedString(params.data?.realtimePriorTranscript, 'realtimePriorTranscript'),
1963
2006
  ConversationID: ReadTrimmedString(params.data?.conversationId, 'conversationId'),
@@ -1974,6 +2017,12 @@ export class BaseAgent {
1974
2017
  UserID: params.contextUser?.ID,
1975
2018
  DisableAutoResponse: meetingMode || undefined,
1976
2019
  SelfNames: selfNames,
2020
+ // The bridge's bot can publish the agent's avatar into the room (set by the room coordinator after it asked).
2021
+ AvatarDelivery: params.data?.realtimeAvatarDelivery === 'room' ? 'room' : undefined,
2022
+ // A server-side session: without room delivery it can't show an avatar, so the model choice prefers none.
2023
+ ServerSide: true,
2024
+ // A phone call (set by the telephony hosts): the caller sees no video, so the session asks for no avatar.
2025
+ PhoneCall: params.data?.realtimePhoneCall === true || undefined,
1977
2026
  // App awareness (Move 1/3/4): the app the session runs in (sources the app cascade layer +
1978
2027
  // RelevantAgents → allowed-agent union) and the live app-context snapshot injected at mint.
1979
2028
  // Both ride params.data, the same conduit async agents use for appContext.
@@ -1989,7 +2038,8 @@ export class BaseAgent {
1989
2038
  * without provider SDKs or DB metadata.
1990
2039
  *
1991
2040
  * Production resolution: pick the highest-power active model of AIModelType `Realtime`; then
1992
- * pick its highest-priority active vendor whose `DriverClass` has a resolvable API key; then
2041
+ * pick its highest-priority active vendor whose `DriverClass` has a resolvable API key, unless another keyed
2042
+ * vendor matches whether the run will show an avatar better ({@link ChooseRealtimeVendorForAvatar}); then
1993
2043
  * instantiate the driver via the `ClassFactory`. Returns `null` (never throws) if any step
1994
2044
  * can't be satisfied — the caller turns that into a clean FAILED result. (Per-agent realtime
1995
2045
  * model preference can later be wired through the agent's prompt-model config, the same path
@@ -2013,31 +2063,62 @@ export class BaseAgent {
2013
2063
  // brings its own credential for a vendor we hold no platform key for now reaches that
2014
2064
  // vendor — which is the point of bringing your own key, not a side effect of it.
2015
2065
  const resolveAPIKey = MakeAIAPIKeyResolver(params.apiKeys, undefined, params.CredentialScope);
2066
+ // The vendor follows whether the run will show an avatar, by the co-agent session prep's rule: a vendor that
2067
+ // renders avatars (Vertex AI for Gemini 3.8 Live) only when it will, or when it is the only keyed vendor.
2068
+ const showsAvatar = this.realtimeRunShowsAvatar(params);
2016
2069
  const candidates = this.selectRealtimeModelCandidates(params.agent, overrideModelID);
2017
2070
  for (const model of candidates) {
2018
2071
  const vendor = SelectRealtimeVendorForModel(model.ID, resolveAPIKey);
2019
- if (!vendor) {
2020
- continue;
2021
- }
2022
- const apiKey = resolveAPIKey(vendor.DriverClass);
2023
- if (!apiKey) {
2024
- continue;
2025
- }
2026
- const instance = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeModel, vendor.DriverClass, apiKey);
2027
- if (!instance) {
2072
+ const first = vendor ? this.instantiateRealtimeRunVendor(model.ID, vendor, resolveAPIKey) : null;
2073
+ if (!vendor || !first) {
2028
2074
  continue;
2029
2075
  }
2030
- return {
2031
- model: instance,
2032
- modelID: model.ID,
2033
- vendorID: vendor.VendorID,
2034
- modelVendorID: vendor.ModelVendorID,
2035
- apiName: vendor.APIName,
2036
- driverClass: vendor.DriverClass
2037
- };
2076
+ return ChooseRealtimeVendorForAvatar(vendor, first, ListRealtimeVendorsForModel(model.ID), showsAvatar, {
2077
+ Instantiate: (other) => this.instantiateRealtimeRunVendor(model.ID, other, resolveAPIKey),
2078
+ RendersAvatar: (resolution) => RealtimeModelShowsAvatar({ ModelID: resolution.modelID, APIName: resolution.apiName, Model: resolution.model }, AIEngine.Instance),
2079
+ });
2038
2080
  }
2039
2081
  return null;
2040
2082
  }
2083
+ /**
2084
+ * Whether a session-driven run will show an avatar: the agent's video setting is on and the run could show one
2085
+ * ({@link RealtimeSessionCanShowAvatar}) — a server-side session, so only when its host publishes the avatar into a
2086
+ * room (`params.data.realtimeAvatarDelivery` `'room'`) and it is not a phone call (`params.data.realtimePhoneCall`).
2087
+ *
2088
+ * @param params The execution parameters.
2089
+ */
2090
+ realtimeRunShowsAvatar(params) {
2091
+ if (this.resolveRealtimeEffectiveConfig(params.agent).realtime?.video?.enabled !== true) {
2092
+ return false;
2093
+ }
2094
+ return RealtimeSessionCanShowAvatar({
2095
+ ServerSide: true,
2096
+ AvatarDelivery: params.data?.realtimeAvatarDelivery === 'room' ? 'room' : undefined,
2097
+ PhoneCall: params.data?.realtimePhoneCall === true,
2098
+ });
2099
+ }
2100
+ /**
2101
+ * A vendor's realtime driver on its key, with the model and vendor identifiers a run resolves to; `null` when the
2102
+ * key or the driver is missing.
2103
+ */
2104
+ instantiateRealtimeRunVendor(modelID, vendor, resolveAPIKey) {
2105
+ const apiKey = resolveAPIKey(vendor.DriverClass);
2106
+ if (!apiKey) {
2107
+ return null;
2108
+ }
2109
+ const instance = MJGlobal.Instance.ClassFactory.CreateInstance(BaseRealtimeModel, vendor.DriverClass, apiKey);
2110
+ if (!instance) {
2111
+ return null;
2112
+ }
2113
+ return {
2114
+ model: instance,
2115
+ modelID,
2116
+ vendorID: vendor.VendorID,
2117
+ modelVendorID: vendor.ModelVendorID,
2118
+ apiName: vendor.APIName,
2119
+ driverClass: vendor.DriverClass
2120
+ };
2121
+ }
2041
2122
  /**
2042
2123
  * The active `Realtime`-AIModelType models to try, in resolution order — the candidate list
2043
2124
  * {@link resolveRealtimeModel} walks until one yields a usable vendor + key + driver. Returns ALL
@@ -5600,23 +5681,15 @@ export class BaseAgent {
5600
5681
  }
5601
5682
  /**
5602
5683
  * Converts ChatMessageContent to a string representation.
5603
- * Handles both simple strings and content block arrays.
5684
+ * Handles both simple strings and content block arrays. Text blocks keep their text;
5685
+ * image, audio, video and file blocks become a short stub instead of their base64.
5604
5686
  *
5605
5687
  * @param content - The message content to convert
5606
5688
  * @returns String representation of the content
5607
5689
  * @protected
5608
5690
  */
5609
5691
  contentToString(content) {
5610
- if (typeof content === 'string') {
5611
- return content;
5612
- }
5613
- // Content is an array of blocks - convert to string
5614
- return content.map((block) => {
5615
- if (block.type === 'text') {
5616
- return block.content;
5617
- }
5618
- return `[${block.type}: ${block.content}]`;
5619
- }).join('\n');
5692
+ return ReplaceMediaBlocksWithStubs(content);
5620
5693
  }
5621
5694
  /**
5622
5695
  * Smart trimming of message content based on detected format.
@@ -5888,9 +5961,7 @@ export class BaseAgent {
5888
5961
  break;
5889
5962
  const originalTokens = candidate.tokens;
5890
5963
  const originalMessage = candidate.message;
5891
- const originalContent = typeof originalMessage.content === 'string'
5892
- ? originalMessage.content
5893
- : JSON.stringify(originalMessage.content);
5964
+ const originalContent = ReplaceMediaBlocksWithStubs(originalMessage.content);
5894
5965
  if (originalMessage.role === 'tool') {
5895
5966
  // compact each tool_result block's text; the block structure is what the provider needs.
5896
5967
  const compacted = CompactToolResultContent(originalMessage, (t) => (t.length > 500 ? `${t.slice(0, 500)}… [compacted from ${t.length} chars]` : t));
@@ -5964,9 +6035,7 @@ export class BaseAgent {
5964
6035
  break;
5965
6036
  const originalTokens = candidate.tokens;
5966
6037
  const originalMessage = candidate.message;
5967
- const originalContent = typeof originalMessage.content === 'string'
5968
- ? originalMessage.content
5969
- : JSON.stringify(originalMessage.content);
6038
+ const originalContent = ReplaceMediaBlocksWithStubs(originalMessage.content);
5970
6039
  // Aggressive compaction - keep only first 200 chars
5971
6040
  const compactedContent = await this.compactMessage(originalMessage, {
5972
6041
  compactMode: 'First N Chars',
@@ -6256,20 +6325,7 @@ The context is now within limits. Please retry your request with the recovered c
6256
6325
  if (!body) {
6257
6326
  return;
6258
6327
  }
6259
- const message = {
6260
- role: 'user',
6261
- content: body,
6262
- metadata: {
6263
- turnAdded: 0,
6264
- messageType: BaseAgent.toolResultMessageType,
6265
- expirationTurns: 2,
6266
- expirationMode: 'Compact',
6267
- compactMode: 'First N Chars',
6268
- compactLength: 500,
6269
- compactPromptId: '',
6270
- },
6271
- };
6272
- params.conversationMessages.push(message);
6328
+ params.conversationMessages.push(this.buildCarryForwardMessage(body));
6273
6329
  this.logStatus(`[PriorTurnToolResults] Carried ${steps.length} tool result(s) forward from the previous run`, true, params);
6274
6330
  }
6275
6331
  catch (error) {
@@ -6277,6 +6333,504 @@ The context is now within limits. Please retry your request with the recovered c
6277
6333
  this.logStatus(`[PriorTurnToolResults] Skipped (contained error): ${error instanceof Error ? error.message : error}`, true, params);
6278
6334
  }
6279
6335
  }
6336
+ /**
6337
+ * Carries a STOPPED previous turn's completed results forward into this run's context.
6338
+ *
6339
+ * {@link injectPriorTurnToolResults} covers the normal case: the previous run settled and
6340
+ * its inline tool results ride along one turn. A run the user stopped never settles, so
6341
+ * that path skips it, and the next turn would redo every action the stopped run had already
6342
+ * finished (observed: a stopped Sage run had five weather lookups and fifteen searches done;
6343
+ * "keep going" redid all of them). This path looks at the newest root run for this agent in
6344
+ * the conversation and, when it is one the user stopped, carries it forward per
6345
+ * {@link ExecuteAgentParams.stopContinuationMode}: by default its own message stack is resumed
6346
+ * so the agent picks up where it stopped; in summary mode its completed Actions and Tool results
6347
+ * go in as one transient message that says to continue, not restart. A settled newest run means
6348
+ * the normal path already has the context, so nothing is added here. During execution the two
6349
+ * halves run separately: {@link loadStoppedRunContinuation} in the parallel phase, then
6350
+ * {@link applyStoppedRunContinuation} once it settles.
6351
+ *
6352
+ * Same gates as the sibling: root depth, a conversation, no history floor. Fail-soft.
6353
+ * @protected
6354
+ */
6355
+ async injectStoppedRunResults(params) {
6356
+ const continuationMessage = params.conversationMessages[params.conversationMessages.length - 1];
6357
+ const continuation = await this.loadStoppedRunContinuation(params);
6358
+ this.applyStoppedRunContinuation(params, continuation, continuationMessage);
6359
+ }
6360
+ /**
6361
+ * The I/O half of {@link injectStoppedRunResults}: finds a user-stopped predecessor and loads its
6362
+ * steps (every step for the full-stack resume, only completed results for the summary). Runs inside
6363
+ * the pre-execution parallel phase; nothing here touches the message list.
6364
+ * @protected
6365
+ */
6366
+ async loadStoppedRunContinuation(params) {
6367
+ if (!params.conversationId || this._depth !== 0 || params.ConversationHistoryFrom) {
6368
+ return null;
6369
+ }
6370
+ try {
6371
+ // A caller that chained this run (the run resolver, after its own stopped-predecessor
6372
+ // lookup, or a client continuing a run) already named the predecessor: check that one
6373
+ // row rather than searching the conversation again. Only an unchained run searches.
6374
+ const runId = params.lastRunId
6375
+ ? await this.userStoppedRunIdIfStopped(params.lastRunId, params)
6376
+ : await this.findUserStoppedPredecessorRunId(params);
6377
+ if (!runId) {
6378
+ return null;
6379
+ }
6380
+ const mode = params.stopContinuationMode ?? BaseAgent.DefaultStopContinuationMode;
6381
+ const steps = mode === 'full_stack'
6382
+ ? await this.loadStoppedRunStack(runId, params)
6383
+ : await this.loadStoppedRunResultSteps(runId, params);
6384
+ return { RunID: runId, Mode: mode, Steps: steps };
6385
+ }
6386
+ catch (error) {
6387
+ this.logStatus(`[StoppedRunResults] Skipped (contained error): ${error instanceof Error ? error.message : error}`, true, params);
6388
+ return null;
6389
+ }
6390
+ }
6391
+ /**
6392
+ * The message half of {@link injectStoppedRunResults}, applied once the pre-execution phase has
6393
+ * settled so it never races the memory and RAG injections running beside the load.
6394
+ *
6395
+ * Full stack: the stopped run's own turns are spliced in ahead of the user's new message, in the
6396
+ * shape the loop built while running, so the agent resumes where it stopped. When the stack cannot
6397
+ * be resumed (no recorded prompt, or its anchor is missing) this falls back to the summary.
6398
+ * Summary: the completed results go in as one carried-forward message. Fail-soft.
6399
+ * @protected
6400
+ */
6401
+ applyStoppedRunContinuation(params, continuation, continuationMessage) {
6402
+ if (!continuation) {
6403
+ return;
6404
+ }
6405
+ try {
6406
+ if (continuation.Mode === 'full_stack' && this.tryResumeStoppedRunStack(params, continuation, continuationMessage)) {
6407
+ return;
6408
+ }
6409
+ const steps = continuation.Steps.filter(BaseAgent.isCarriedForwardResultStep);
6410
+ const body = BaseAgent.BuildStoppedRunResultsMessage(steps, BaseAgent.maxStoppedRunResultChars);
6411
+ if (!body) {
6412
+ return;
6413
+ }
6414
+ params.conversationMessages.push(this.buildCarryForwardMessage(body));
6415
+ this.logStatus(`[StoppedRunResults] Carried ${steps.length} completed step result(s) forward from stopped run ${continuation.RunID}`, true, params);
6416
+ }
6417
+ catch (error) {
6418
+ this.logStatus(`[StoppedRunResults] Skipped (contained error): ${error instanceof Error ? error.message : error}`, true, params);
6419
+ }
6420
+ }
6421
+ /**
6422
+ * The full-stack half of {@link applyStoppedRunContinuation}: true when the stopped run's turns were
6423
+ * spliced in. Any failure leaves the messages untouched and returns false, so the caller carries
6424
+ * the results as a summary instead of losing them.
6425
+ * @private
6426
+ */
6427
+ tryResumeStoppedRunStack(params, continuation, continuationMessage) {
6428
+ try {
6429
+ const resumed = this.buildResumedStoppedRunTurns(continuation.Steps, params);
6430
+ if (resumed && BaseAgent.SpliceResumedTurns(params.conversationMessages, resumed, continuationMessage)) {
6431
+ this.logStatus(`[StoppedRunResume] Resumed ${resumed.length} message(s) from stopped run ${continuation.RunID}`, true, params);
6432
+ return true;
6433
+ }
6434
+ this.logStatus(`[StoppedRunResume] Stack of stopped run ${continuation.RunID} could not be resumed; carrying its results as a summary`, true, params);
6435
+ }
6436
+ catch (error) {
6437
+ this.logStatus(`[StoppedRunResume] Resume failed (${error instanceof Error ? error.message : error}); carrying its results as a summary`, true, params);
6438
+ }
6439
+ return false;
6440
+ }
6441
+ /** The mode a run uses when the caller does not set {@link ExecuteAgentParams.stopContinuationMode}. */
6442
+ static { this.DefaultStopContinuationMode = 'full_stack'; }
6443
+ /**
6444
+ * Placed just before the user's new message when a stopped run's stack is resumed, so the model
6445
+ * knows the turns above it were interrupted and what follows is how to continue.
6446
+ */
6447
+ static { this.StoppedRunResumeNotice = '[Note: The user stopped the previous execution at this point. The instruction that follows is the continuation request.]'; }
6448
+ /** Result code and message for an action the stopped run requested but never ran. */
6449
+ static { this.stoppedBeforeRunResultCode = 'NOT_EXECUTED'; }
6450
+ static { this.stoppedBeforeRunMessage = 'Not executed: the user stopped the run before this action finished.'; }
6451
+ /**
6452
+ * The stopped run's turns in the shape the loop builds, or null when they cannot be recovered.
6453
+ *
6454
+ * The last prompt step recorded the run's whole message stack as it went into that prompt; the
6455
+ * first prompt step recorded the stack the run started from (history plus this run's injections,
6456
+ * which the new run adds for itself). The difference is the run's own turns. If the last prompt
6457
+ * completed, its decision and the steps after it are rebuilt with the loop's own builders.
6458
+ * @protected
6459
+ */
6460
+ buildResumedStoppedRunTurns(steps, params) {
6461
+ const prompts = steps.filter(s => s.StepType === 'Prompt');
6462
+ if (prompts.length === 0) {
6463
+ return null;
6464
+ }
6465
+ const lastPrompt = prompts[prompts.length - 1];
6466
+ const loopTurns = BaseAgent.ExtractStoppedRunLoopTurns(BaseAgent.readPromptStepStack(prompts[0]), BaseAgent.readPromptStepStack(lastPrompt));
6467
+ if (!loopTurns) {
6468
+ return null;
6469
+ }
6470
+ const decision = lastPrompt.Status === 'Completed' ? BaseAgent.readPromptStepDecision(lastPrompt) : null;
6471
+ const tail = this.buildStoppedRunTail(decision, steps.filter(s => s.StepNumber > lastPrompt.StepNumber), params);
6472
+ return [...loopTurns, ...tail].map(BaseAgent.restampResumedTurn);
6473
+ }
6474
+ /**
6475
+ * The run's own turns: what follows the run's starting message in its last recorded stack, minus
6476
+ * the per-request runtime-state fragments (the new run appends its own). The starting message is
6477
+ * matched at its original position first, so a repeated message earlier in the history ("keep
6478
+ * going" twice) cannot be mistaken for it. Null when it cannot be found.
6479
+ */
6480
+ static ExtractStoppedRunLoopTurns(firstStack, lastStack) {
6481
+ if (!firstStack?.length || !lastStack?.length) {
6482
+ return null;
6483
+ }
6484
+ const key = (m) => `${m.role}\u0000${JSON.stringify(m.content)}`;
6485
+ const startKey = key(firstStack[firstStack.length - 1]);
6486
+ const expectedAt = firstStack.length - 1;
6487
+ const at = lastStack[expectedAt] && key(lastStack[expectedAt]) === startKey
6488
+ ? expectedAt
6489
+ : lastStack.map(key).lastIndexOf(startKey);
6490
+ if (at < 0) {
6491
+ return null;
6492
+ }
6493
+ return lastStack.slice(at + 1).filter(m => m.metadata?.volatileState !== true);
6494
+ }
6495
+ /**
6496
+ * Inserts resumed turns ahead of the user's new message (and ahead of the stopped run's own reply
6497
+ * to the user, when that sits right before it), followed by {@link StoppedRunResumeNotice}. The new
6498
+ * message is found by identity, captured before the pre-execution phase added anything. Returns
6499
+ * false, changing nothing, when it is not a user message still in the list.
6500
+ */
6501
+ static SpliceResumedTurns(messages, resumed, continuationMessage) {
6502
+ const at = continuationMessage ? messages.indexOf(continuationMessage) : -1;
6503
+ if (at < 0 || continuationMessage?.role !== 'user') {
6504
+ return false;
6505
+ }
6506
+ const notice = { role: 'user', content: BaseAgent.StoppedRunResumeNotice };
6507
+ const previous = messages[at - 1];
6508
+ const stoppedReplyAt = previous?.role === 'assistant' && !previous.toolCalls?.length ? at - 1 : at;
6509
+ messages.splice(at, 0, notice);
6510
+ messages.splice(stoppedReplyAt, 0, ...resumed);
6511
+ return true;
6512
+ }
6513
+ /**
6514
+ * What the stopped run's last completed prompt decided, rebuilt as the loop would have written it:
6515
+ * the model's native call turn when results go back natively, the invocation record and the results
6516
+ * of the actions that finished (and a not-executed result for the ones that did not) for an Actions
6517
+ * turn, otherwise the completed results as a carried-forward message. Any native call left without
6518
+ * a result is then answered, so the history is valid for every provider.
6519
+ * @protected
6520
+ */
6521
+ buildStoppedRunTail(decision, tailSteps, params) {
6522
+ const tail = [];
6523
+ if (!decision) {
6524
+ return tail;
6525
+ }
6526
+ const scratch = { ...params, conversationMessages: tail };
6527
+ const native = decision.nativeTurn?.sendResultsNatively === true;
6528
+ if (native && decision.nativeTurn) {
6529
+ tail.push(BuildAssistantToolCallTurn(decision.nativeTurn));
6530
+ }
6531
+ if (decision.step === 'Actions' && decision.actions?.length) {
6532
+ if (!native) {
6533
+ tail.push({ role: 'user', content: this.buildActionInvocationMessage(decision.actions) });
6534
+ }
6535
+ const summaries = this.summarizeStoppedRunActions(decision.actions, tailSteps);
6536
+ this.appendActionResults(scratch, summaries, this.buildActionResultsMessage(summaries), undefined, decision);
6537
+ }
6538
+ else {
6539
+ const results = BaseAgent.BuildStoppedRunResultsMessage(tailSteps.filter(BaseAgent.isCarriedForwardResultStep), BaseAgent.maxStoppedRunResultChars);
6540
+ if (results) {
6541
+ tail.push(this.buildCarryForwardMessage(results));
6542
+ }
6543
+ }
6544
+ this.reconcileUnansweredToolCalls(scratch);
6545
+ return tail;
6546
+ }
6547
+ /** One result per requested action, in request order: its completed step's result, or not executed. @private */
6548
+ summarizeStoppedRunActions(requested, tailSteps) {
6549
+ const completed = tailSteps.filter(s => s.StepType === 'Actions' && s.Status === 'Completed');
6550
+ const used = new Set();
6551
+ return requested.map(action => {
6552
+ const step = completed.find(s => !used.has(s) && BaseAgent.readStoppedActionName(s) === action.name);
6553
+ const summary = step ? this.summarizeStoppedActionStep(action.name, step) : null;
6554
+ if (!step || !summary) {
6555
+ return {
6556
+ actionName: action.name,
6557
+ success: false,
6558
+ params: [],
6559
+ resultCode: BaseAgent.stoppedBeforeRunResultCode,
6560
+ message: BaseAgent.stoppedBeforeRunMessage,
6561
+ };
6562
+ }
6563
+ used.add(step);
6564
+ return summary;
6565
+ });
6566
+ }
6567
+ /**
6568
+ * A completed action step as the loop summarizes a live action result: output parameters only,
6569
+ * with large media swapped for placeholders. Null when the step's output cannot be read.
6570
+ * @private
6571
+ */
6572
+ summarizeStoppedActionStep(actionName, step) {
6573
+ let output;
6574
+ try {
6575
+ output = step.OutputData ? JSON.parse(step.OutputData) : {};
6576
+ }
6577
+ catch {
6578
+ return null;
6579
+ }
6580
+ const result = output.actionResult;
6581
+ if (!result) {
6582
+ return null;
6583
+ }
6584
+ const success = result.success === true;
6585
+ const outputParams = (result.parameters ?? []).filter(p => p.Type === 'Both' || p.Type === 'Output');
6586
+ const actionEntity = ActionEngineServer.Instance.Actions?.find(a => a.Name === actionName);
6587
+ return {
6588
+ actionName,
6589
+ success,
6590
+ params: this.interceptLargeBinaryContent(outputParams, actionEntity),
6591
+ resultCode: result.resultCode || (success ? 'SUCCESS' : 'ERROR'),
6592
+ message: result.message || (success ? 'Action completed' : 'Unknown error'),
6593
+ };
6594
+ }
6595
+ /** The message stack a prompt step recorded as its input, or null. @private */
6596
+ static readPromptStepStack(step) {
6597
+ try {
6598
+ const input = step.InputData ? JSON.parse(step.InputData) : {};
6599
+ return Array.isArray(input.conversationMessages) ? input.conversationMessages : null;
6600
+ }
6601
+ catch {
6602
+ return null;
6603
+ }
6604
+ }
6605
+ /** The decision a completed prompt step recorded, or null. @private */
6606
+ static readPromptStepDecision(step) {
6607
+ try {
6608
+ const output = step.OutputData ? JSON.parse(step.OutputData) : {};
6609
+ return output.nextStep ?? null;
6610
+ }
6611
+ catch {
6612
+ return null;
6613
+ }
6614
+ }
6615
+ /** The action an Actions step ran, from its recorded input, or null. @private */
6616
+ static readStoppedActionName(step) {
6617
+ try {
6618
+ const input = step.InputData ? JSON.parse(step.InputData) : {};
6619
+ return input.actionName ?? null;
6620
+ }
6621
+ catch {
6622
+ return null;
6623
+ }
6624
+ }
6625
+ /**
6626
+ * A resumed turn re-dated to this run's first turn. The stopped run's turn numbers mean nothing
6627
+ * to this run's counter, and expiry compares the two, so a resumed result ages from now.
6628
+ * @private
6629
+ */
6630
+ static restampResumedTurn(message) {
6631
+ return message.metadata?.turnAdded !== undefined
6632
+ ? { ...message, metadata: { ...message.metadata, turnAdded: 0 } }
6633
+ : message;
6634
+ }
6635
+ /** A step whose result the summary carries: a completed, successful Actions or Tool step. @private */
6636
+ static isCarriedForwardResultStep(step) {
6637
+ const types = BaseAgent.stoppedRunCarryForwardPredicate.stepTypes;
6638
+ return types.includes(step.StepType) &&
6639
+ step.Status === BaseAgent.stoppedRunCarryForwardPredicate.stepStatus &&
6640
+ step.Success === true;
6641
+ }
6642
+ /**
6643
+ * A carried-forward results message: a user-role message that expires after two turns and
6644
+ * compacts to its head, so results reused once never compound across the conversation. Shared
6645
+ * by the settled-run and stopped-run carry-forwards so the two cannot age differently.
6646
+ * @private
6647
+ */
6648
+ buildCarryForwardMessage(content) {
6649
+ return {
6650
+ role: 'user',
6651
+ content,
6652
+ metadata: {
6653
+ turnAdded: 0,
6654
+ messageType: BaseAgent.toolResultMessageType,
6655
+ expirationTurns: 2,
6656
+ expirationMode: 'Compact',
6657
+ compactMode: 'First N Chars',
6658
+ compactLength: 500,
6659
+ compactPromptId: '',
6660
+ },
6661
+ };
6662
+ }
6663
+ /**
6664
+ * The ID of this agent's newest root run on this turn's branch path IF the user stopped it,
6665
+ * else null. One query over the runs that could be the predecessor (settled, or stopped by
6666
+ * the user): the newest decides. A stopped run older than a settled one is history the
6667
+ * settled run already built on, so it is not carried again. The path scopes it the same way
6668
+ * as {@link loadPriorTurnToolResultSteps}: a run stopped on a sibling branch is never resumed.
6669
+ * @private
6670
+ */
6671
+ async findUserStoppedPredecessorRunId(params) {
6672
+ const rv = RunView.FromMetadataProvider(this.ProviderToUse);
6673
+ const newest = await rv.RunView({
6674
+ EntityName: 'MJ: AI Agent Runs',
6675
+ ExtraFilter: MJAIAgentRunEntityExtended.BuildStoppedPredecessorFilter(params.conversationId, params.agent.ID, params.ConversationBranchID ?? null, this._conversationBranches),
6676
+ OrderBy: '__mj_CreatedAt DESC',
6677
+ MaxRows: 1,
6678
+ Fields: ['ID', 'Status', 'CancellationReason'],
6679
+ ResultType: 'simple',
6680
+ // The row this looks for was written by the Stop button moments ago; read the database, not a cache.
6681
+ BypassCache: true,
6682
+ }, params.contextUser);
6683
+ const run = newest.Success ? newest.Results?.[0] : undefined;
6684
+ return run && MJAIAgentRunEntityExtended.IsUserStopped(run.Status, run.CancellationReason) ? run.ID : null;
6685
+ }
6686
+ /**
6687
+ * `runId` if that run is one the user stopped, else null. The chained-run counterpart of
6688
+ * {@link findUserStoppedPredecessorRunId}: one primary-key read of the named run instead of a
6689
+ * search of the conversation, so a run the resolver already resolved is not looked up twice,
6690
+ * and a run chained to a settled predecessor (an ordinary continuation) resumes nothing.
6691
+ * @private
6692
+ */
6693
+ async userStoppedRunIdIfStopped(runId, params) {
6694
+ if (!IsValidUUID(runId)) {
6695
+ return null;
6696
+ }
6697
+ const rv = RunView.FromMetadataProvider(this.ProviderToUse);
6698
+ const result = await rv.RunView({
6699
+ EntityName: 'MJ: AI Agent Runs',
6700
+ ExtraFilter: `ID='${runId}'`,
6701
+ MaxRows: 1,
6702
+ Fields: ['ID', 'Status', 'CancellationReason'],
6703
+ ResultType: 'simple',
6704
+ // The row this checks was written by the Stop button moments ago; read the database, not a cache.
6705
+ BypassCache: true,
6706
+ }, params.contextUser);
6707
+ const run = result.Success ? result.Results?.[0] : undefined;
6708
+ return run && MJAIAgentRunEntityExtended.IsUserStopped(run.Status, run.CancellationReason) ? run.ID : null;
6709
+ }
6710
+ /**
6711
+ * Every step of a stopped run, in step order: the full-stack resume reads the prompt steps'
6712
+ * recorded stacks and which requested actions completed. Read from the database, since the run
6713
+ * was stopped moments ago.
6714
+ * @private
6715
+ */
6716
+ async loadStoppedRunStack(runId, params) {
6717
+ const rv = RunView.FromMetadataProvider(this.ProviderToUse);
6718
+ const steps = await rv.RunView({
6719
+ EntityName: 'MJ: AI Agent Run Steps',
6720
+ ExtraFilter: `AgentRunID='${runId}'`,
6721
+ OrderBy: 'StepNumber ASC',
6722
+ Fields: ['StepNumber', 'StepType', 'StepName', 'Status', 'Success', 'InputData', 'OutputData'],
6723
+ ResultType: 'simple',
6724
+ BypassCache: true,
6725
+ }, params.contextUser);
6726
+ return steps.Success ? (steps.Results || []) : [];
6727
+ }
6728
+ /** The completed, successful Actions and Tool steps of a stopped run, in step order. @private */
6729
+ async loadStoppedRunResultSteps(runId, params) {
6730
+ const rv = RunView.FromMetadataProvider(this.ProviderToUse);
6731
+ const steps = await rv.RunView({
6732
+ EntityName: 'MJ: AI Agent Run Steps',
6733
+ ExtraFilter: `AgentRunID='${runId}' AND StepType IN (${BaseAgent.stoppedRunCarryForwardPredicate.stepTypes.map(t => `'${t}'`).join(', ')}) ` +
6734
+ `AND Status='${BaseAgent.stoppedRunCarryForwardPredicate.stepStatus}' AND Success=1`,
6735
+ OrderBy: 'StepNumber ASC',
6736
+ Fields: ['StepNumber', 'StepType', 'StepName', 'Status', 'Success', 'InputData', 'OutputData'],
6737
+ ResultType: 'simple',
6738
+ BypassCache: true,
6739
+ }, params.contextUser);
6740
+ return steps.Success ? (steps.Results || []) : [];
6741
+ }
6742
+ /**
6743
+ * Budget for the stopped-run carry-forward message. Smaller than a standalone tool result on
6744
+ * purpose: this rides at the front of a continuation turn's context, and a stopped run can
6745
+ * hold dozens of search results. ~7k tokens keeps the useful values and drops the long tails.
6746
+ */
6747
+ static { this.maxStoppedRunResultChars = 28_000; }
6748
+ /**
6749
+ * Renders a stopped run's completed step results into the carried-forward message body.
6750
+ * Pure and static for testability. An Actions step renders as the action call (name and
6751
+ * params) with its output parameters, or its message when it produced none; a Tool step
6752
+ * renders under the same contract as {@link BuildPriorTurnToolResultsMessage}. Failed or
6753
+ * unparseable steps are skipped. Each section is capped at an even share of `maxChars`
6754
+ * (never below a readable floor) and the whole body at `maxChars`, with a note for anything
6755
+ * dropped. Returns null when nothing usable remains.
6756
+ */
6757
+ static BuildStoppedRunResultsMessage(steps, maxChars) {
6758
+ const sections = [];
6759
+ const perSectionMax = Math.max(BaseAgent.minStoppedRunSectionChars, Math.floor(maxChars / Math.max(steps.length, 1)));
6760
+ let usedChars = 0;
6761
+ let dropped = 0;
6762
+ for (const step of steps) {
6763
+ const section = BaseAgent.renderStoppedRunStep(step);
6764
+ if (!section) {
6765
+ continue;
6766
+ }
6767
+ const capped = section.length > perSectionMax
6768
+ ? `${section.slice(0, perSectionMax)}\n[truncated]`
6769
+ : section;
6770
+ if (usedChars + capped.length > maxChars && sections.length > 0) {
6771
+ dropped++;
6772
+ continue;
6773
+ }
6774
+ usedChars += capped.length;
6775
+ sections.push(capped);
6776
+ }
6777
+ if (sections.length === 0) {
6778
+ return null;
6779
+ }
6780
+ const header = 'Your previous turn was stopped by the user before it finished. The actions and tools listed below ' +
6781
+ 'were ALREADY EXECUTED in that turn and their results are still valid. Treat each entry as a result you already ' +
6782
+ 'have: do not run the action or call a tool again for it, and do not invoke the action names below as tools. ' +
6783
+ 'Continue from where the stopped turn left off rather than starting over.';
6784
+ const droppedNote = dropped > 0 ? `\n\n[${dropped} additional result(s) omitted for size; re-call those if needed]` : '';
6785
+ return `${header}\n${sections.join('\n\n')}${droppedNote}`;
6786
+ }
6787
+ /** One stopped-run step as a result section, or null when it carries nothing reusable. @private */
6788
+ static renderStoppedRunStep(step) {
6789
+ if (step.StepType === 'Tool') {
6790
+ return BaseAgent.BuildPriorTurnToolResultsMessage([{ OutputData: step.OutputData }], Number.MAX_SAFE_INTEGER)
6791
+ ?.split('\n').slice(1).join('\n') || null; // the sibling renderer's body without its header line
6792
+ }
6793
+ if (step.StepType !== 'Actions' || !step.InputData || !step.OutputData) {
6794
+ return null;
6795
+ }
6796
+ let input;
6797
+ let output;
6798
+ try {
6799
+ input = JSON.parse(step.InputData);
6800
+ output = JSON.parse(step.OutputData);
6801
+ }
6802
+ catch {
6803
+ return null;
6804
+ }
6805
+ const result = output.actionResult;
6806
+ if (!input.actionName || result?.success !== true) {
6807
+ return null;
6808
+ }
6809
+ const outputs = {};
6810
+ for (const param of result.parameters ?? []) {
6811
+ if (param.Type === BaseAgent.actionOutputParamType && param.Value !== undefined && param.Value !== null) {
6812
+ outputs[param.Name] = param.Value;
6813
+ }
6814
+ }
6815
+ const body = Object.keys(outputs).length > 0 ? RenderToolResultData(outputs) : (result.message ?? '');
6816
+ if (!body) {
6817
+ return null;
6818
+ }
6819
+ // Deliberately NOT the inline-tool heading (`### tool({...})`): an agent shown that shape for an
6820
+ // action tried to call the action as a conversation tool. Say what it is in words.
6821
+ return `### Already executed: action "${input.actionName}" with params ${JSON.stringify(input.actionParams ?? {})}\n` +
6822
+ `Result:\n\`\`\`json\n${body}\n\`\`\``;
6823
+ }
6824
+ /**
6825
+ * The `MJ: AI Agent Runs` predicate for this agent's previous settled root run on one
6826
+ * branch path: the run's reply row must be on that path, so a run made on a sibling
6827
+ * branch is never carried forward, and not replaced by a rerun of its turn, so a replaced
6828
+ * answer's tool results are never carried forward.
6829
+ */
6830
+ static BuildPriorTurnRunFilter(conversationId, agentId, branchId, branches, statusList) {
6831
+ return `ConversationID='${conversationId}' AND Status IN (${statusList}) AND ParentRunID IS NULL AND AgentID='${agentId}'`
6832
+ + ` AND ${MJAIAgentRunEntityExtended.BuildReplyOnBranchPathFilter(conversationId, branchId, branches)}`;
6833
+ }
6280
6834
  /**
6281
6835
  * Loads this agent's previous settled root run's Tool steps for this conversation
6282
6836
  * (settled = {@link settledRunStatuses}: Completed or AwaitingFeedback).
@@ -6292,7 +6846,8 @@ The context is now within limits. Please retry your request with the recovered c
6292
6846
  * @private
6293
6847
  */
6294
6848
  async loadPriorTurnToolResultSteps(params) {
6295
- const cached = PriorTurnToolResultCache.Instance.Get(params.conversationId, params.agent.ID);
6849
+ const branchId = params.ConversationBranchID ?? null;
6850
+ const cached = PriorTurnToolResultCache.Instance.Get(params.conversationId, params.agent.ID, branchId);
6296
6851
  if (cached) {
6297
6852
  this.logStatus(`[PriorTurnToolResults] Prior-run tool results served from cache (${cached.length} step(s), no DB lookup)`, true, params);
6298
6853
  return cached;
@@ -6301,10 +6856,11 @@ The context is now within limits. Please retry your request with the recovered c
6301
6856
  const statusList = predicate.runStatuses.map(s => `'${s}'`).join(', ');
6302
6857
  const rv = RunView.FromMetadataProvider(this.ProviderToUse);
6303
6858
  // AgentID scopes provenance: in a multi-agent conversation, agent B must never
6304
- // inherit agent A's results labeled "your previous turn".
6859
+ // inherit agent A's results labeled "your previous turn". The branch path scopes it
6860
+ // the same way: a run on a sibling branch is never carried forward.
6305
6861
  const priorRun = await rv.RunView({
6306
6862
  EntityName: 'MJ: AI Agent Runs',
6307
- ExtraFilter: `ConversationID='${params.conversationId}' AND Status IN (${statusList}) AND ParentRunID IS NULL AND AgentID='${params.agent.ID}'`,
6863
+ ExtraFilter: BaseAgent.BuildPriorTurnRunFilter(params.conversationId, params.agent.ID, branchId, this._conversationBranches, statusList),
6308
6864
  OrderBy: '__mj_CreatedAt DESC',
6309
6865
  MaxRows: 1,
6310
6866
  Fields: ['ID'],
@@ -6330,9 +6886,11 @@ The context is now within limits. Please retry your request with the recovered c
6330
6886
  * only when the run row settled as {@link settledRunStatuses} (Completed OR
6331
6887
  * AwaitingFeedback — the normal chat-turn ending) — a failed run leaves the previous
6332
6888
  * settled run's entry standing, just as the RunView filter would. Scoped to this
6333
- * run's agent (cache key = conversation + agent) so parallel agents in one
6334
- * conversation never cross-pollinate. An empty projection is cached too (the
6335
- * negative-cache case that spares tool-free conversations the queries every turn).
6889
+ * run's agent and branch (cache key = conversation + agent + branch) so parallel
6890
+ * agents in one conversation, and sibling branches, never cross-pollinate. An empty
6891
+ * projection is cached too (the negative-cache case that spares tool-free
6892
+ * conversations the queries every turn). The run's reply row is stored with the entry,
6893
+ * so the entry is no longer served once a rerun of its turn replaces that row.
6336
6894
  * Same-node edge semantics (failed step INSERTs, concurrent completions) are
6337
6895
  * documented on the cache class. Called from {@link finalizeAgentRun}.
6338
6896
  * @private
@@ -6347,7 +6905,7 @@ The context is now within limits. Please retry your request with the recovered c
6347
6905
  const records = (this._agentRun.Steps || [])
6348
6906
  .filter(s => s.StepType === predicate.stepType && s.Status === predicate.stepStatus)
6349
6907
  .map(s => ({ OutputData: s.OutputData || null }));
6350
- PriorTurnToolResultCache.Instance.Set(conversationId, this._agentRun.AgentID, records);
6908
+ PriorTurnToolResultCache.Instance.Set(conversationId, this._agentRun.AgentID, this._executeParams?.ConversationBranchID ?? null, records, this._agentRun.ConversationDetailID ?? null);
6351
6909
  }
6352
6910
  /**
6353
6911
  * Tool families whose step results are eligible for prior-turn carry-forward —
@@ -6373,7 +6931,7 @@ The context is now within limits. Please retry your request with the recovered c
6373
6931
  * ({@link carryForwardPredicate} → DB filter + cache-population gate) and the
6374
6932
  * post-turn compaction gate ({@link startPostTurnCompaction}).
6375
6933
  */
6376
- static { this.settledRunStatuses = ['Completed', 'AwaitingFeedback']; }
6934
+ static { this.settledRunStatuses = MJAIAgentRunEntityExtended.SettledStatuses; }
6377
6935
  /**
6378
6936
  * The carry-forward row predicate — the SINGLE source shared by the two places that
6379
6937
  * must select the same rows or the cache diverges from the DB path: the RunView
@@ -6391,6 +6949,19 @@ The context is now within limits. Please retry your request with the recovered c
6391
6949
  stepStatus: 'Completed',
6392
6950
  runStatuses: BaseAgent.settledRunStatuses,
6393
6951
  }; }
6952
+ /**
6953
+ * The row predicate for the stopped-run carry-forward's step query: the completed, successful
6954
+ * Actions and Tool steps of the run the user stopped. Declared beside
6955
+ * {@link carryForwardPredicate} for the same reason it is: one place to change what counts.
6956
+ */
6957
+ static { this.stoppedRunCarryForwardPredicate = {
6958
+ stepTypes: ['Actions', 'Tool'],
6959
+ stepStatus: 'Completed',
6960
+ }; }
6961
+ /** An action parameter's direction, as the action executor stamps it. */
6962
+ static { this.actionOutputParamType = 'Output'; }
6963
+ /** A stopped-run result section is never cut below this many characters, whatever the budget. */
6964
+ static { this.minStoppedRunSectionChars = 1_500; }
6394
6965
  /**
6395
6966
  * Display name of the seeded system prompt behind summarizeRange's recursive
6396
6967
  * sub-call (see metadata/prompts/.summarize-range-prompt.json). Resolved with a
@@ -7788,13 +8359,16 @@ The context is now within limits. Please retry your request with the recovered c
7788
8359
  */
7789
8360
  buildAgentBaseCatalog(agent, engine) {
7790
8361
  // Resolve sub-agents: direct ParentID children + active relationships, de-duped, ordered.
8362
+ // Only Active agents are kept, so a Disabled relationship target never reaches the catalog.
7791
8363
  const activeSubAgents = engine.Agents.filter(a => UUIDsEqual(a.ParentID, agent.ID) && a.Status === 'Active')
7792
8364
  .sort((a, b) => a.ExecutionOrder - b.ExecutionOrder);
7793
8365
  const activeAgentRelationships = engine.AgentRelationships.filter(ar => UUIDsEqual(ar.AgentID, agent.ID) && ar.Status === 'Active');
7794
8366
  const uniqueActiveSubAgentIDs = new Set();
7795
8367
  activeSubAgents.forEach(a => uniqueActiveSubAgentIDs.add(a.ID));
7796
8368
  activeAgentRelationships.forEach(ar => uniqueActiveSubAgentIDs.add(ar.SubAgentID));
7797
- const uniqueActiveSubAgents = Array.from(uniqueActiveSubAgentIDs).map(id => engine.Agents.find(a => UUIDsEqual(a.ID, id)));
8369
+ const uniqueActiveSubAgents = Array.from(uniqueActiveSubAgentIDs)
8370
+ .map(id => engine.Agents.find(a => UUIDsEqual(a.ID, id)))
8371
+ .filter((a) => !!a && a.Status === 'Active');
7798
8372
  // Resolve actions from the agent's active AIAgentAction junctions.
7799
8373
  const agentActions = engine.AgentActions.filter(aa => UUIDsEqual(aa.AgentID, agent.ID) && aa.Status === 'Active');
7800
8374
  const baseActionsRaw = ActionEngineServer.Instance.Actions.filter(a => agentActions.some(aa => UUIDsEqual(aa.ActionID, a.ID)));
@@ -8968,7 +9542,7 @@ The context is now within limits. Please retry your request with the recovered c
8968
9542
  lines.push('records. When you choose client tools, set nextStep.type to "ClientTools".');
8969
9543
  lines.push('');
8970
9544
  lines.push('NOTE: Do NOT use client tools for asking the user questions or collecting input.');
8971
- lines.push('Use the "Chat" step for that. Client tools are for programmatic UI interaction only.');
9545
+ lines.push('Use the "Chat" step for that. Client tools read and change the user\'s UI; some return an image.');
8972
9546
  lines.push('');
8973
9547
  for (const tool of tools) {
8974
9548
  const categoryTag = tool.Category ? ` [${tool.Category}]` : '';
@@ -9061,6 +9635,44 @@ The context is now within limits. Please retry your request with the recovered c
9061
9635
  const desc = param.Description ? ` — ${param.Description}` : '';
9062
9636
  return `\`${param.Name}\`${requiredMarker}${suffix}${desc}${defaultStr}`;
9063
9637
  }
9638
+ /**
9639
+ * The `user`-role record of what the model asked to run on an Actions turn ("[You invoked ...]").
9640
+ * Shared by the live Actions step and the stopped-run resume, so a resumed turn reads exactly
9641
+ * like one the loop wrote. See the Actions step for why it is a user-role annotation.
9642
+ * @protected
9643
+ */
9644
+ buildActionInvocationMessage(actions) {
9645
+ if (actions.length === 1) {
9646
+ const aa = actions[0];
9647
+ if (!aa.params || Object.keys(aa.params).length === 0) {
9648
+ return `[You invoked the **${aa.name}** action.]`;
9649
+ }
9650
+ const paramsList = Object.entries(aa.params)
9651
+ .map(([key, value]) => `• **${key}**: ${this.formatParamValueForMessage(value)}`)
9652
+ .join('\n');
9653
+ return `[You invoked the **${aa.name}** action with parameters:\n${paramsList}\n]`;
9654
+ }
9655
+ return `[You invoked **${actions.length} actions** in parallel:\n\n` + actions.map((aa, index) => {
9656
+ const actionText = `${index + 1}. **${aa.name}**`;
9657
+ if (!aa.params || Object.keys(aa.params).length === 0) {
9658
+ return actionText;
9659
+ }
9660
+ const paramsList = Object.entries(aa.params)
9661
+ .map(([key, value]) => ` • **${key}**: ${this.formatParamValueForMessage(value)}`)
9662
+ .join('\n');
9663
+ return `${actionText}\n${paramsList}`;
9664
+ }).join('\n\n') + '\n]';
9665
+ }
9666
+ /**
9667
+ * The action-results message body: a header naming any failures, then each result in the
9668
+ * compact markdown shape. Shared by the live Actions step and the stopped-run resume.
9669
+ * @protected
9670
+ */
9671
+ buildActionResultsMessage(actionSummaries) {
9672
+ const failed = actionSummaries.filter(a => !a.success).length;
9673
+ const header = failed > 0 ? `${failed} of ${actionSummaries.length} action(s) failed:` : 'Action results:';
9674
+ return `${header}\n${this.formatActionResultsAsMarkdown(actionSummaries)}`;
9675
+ }
9064
9676
  /**
9065
9677
  * Formats an array of action result summaries as compact markdown instead of
9066
9678
  * pretty-printed JSON. This typically saves 60-70 % of the tokens that the
@@ -9430,6 +10042,16 @@ The context is now within limits. Please retry your request with the recovered c
9430
10042
  // For now, returning a placeholder
9431
10043
  return 'General';
9432
10044
  }
10045
+ /**
10046
+ * Throws unless `lastRunId` is absent or a UUID. The value can come from a client, and the run
10047
+ * initialization places it in `ExtraFilter` clauses (the payload lookup, {@link validateRunChain}
10048
+ * and {@link resolvePlanModeGate}).
10049
+ */
10050
+ assertLastRunIdIsUUID(lastRunId) {
10051
+ if (lastRunId && !IsValidUUID(lastRunId)) {
10052
+ throw new Error('Invalid lastRunId: expected the UUID of a prior AI Agent Run.');
10053
+ }
10054
+ }
9433
10055
  /**
9434
10056
  * Initializes the agent run tracking by creating MJAIAgentRunEntityExtended and setting up context.
9435
10057
  *
@@ -9437,6 +10059,7 @@ The context is now within limits. Please retry your request with the recovered c
9437
10059
  * @param {ExecuteAgentParams} params - The execution parameters
9438
10060
  */
9439
10061
  async initializeAgentRun(params) {
10062
+ this.assertLastRunIdIsUUID(params.lastRunId);
9440
10063
  // Handle autoPopulateLastRunPayload if requested
9441
10064
  let modifiedParams = params;
9442
10065
  if (params.lastRunId && params.autoPopulateLastRunPayload) {
@@ -9445,7 +10068,9 @@ The context is now within limits. Please retry your request with the recovered c
9445
10068
  const lastRunResult = await rv.RunView({
9446
10069
  EntityName: 'MJ: AI Agent Runs',
9447
10070
  ExtraFilter: `ID='${params.lastRunId}'`,
9448
- ResultType: 'simple' // Avoid recursive loading
10071
+ ResultType: 'simple', // Avoid recursive loading
10072
+ // A stopped predecessor wrote its FinalPayload moments ago; read the database, not a cache.
10073
+ BypassCache: true
9449
10074
  }, params.contextUser);
9450
10075
  if (lastRunResult.Success && lastRunResult.Results.length > 0) {
9451
10076
  const lastRun = lastRunResult.Results[0];
@@ -9606,7 +10231,7 @@ The context is now within limits. Please retry your request with the recovered c
9606
10231
  // server-side DB provider can heartbeat via SQL; client/non-DB providers simply opt out.
9607
10232
  const runProvider = params.provider || this._activeProvider;
9608
10233
  if (runProvider instanceof DatabaseProviderBase && params.contextUser) {
9609
- AgentRunWatchdog.Instance.Track(this._agentRun.ID, runProvider, params.contextUser);
10234
+ AgentRunWatchdog.Instance.Track(this._agentRun.ID, runProvider, params.contextUser, this._runAbortController ?? undefined);
9610
10235
  }
9611
10236
  // Invoke callback if provided
9612
10237
  if (modifiedParams.onAgentRunCreated) {
@@ -12123,40 +12748,7 @@ The context is now within limits. Please retry your request with the recovered c
12123
12748
  // which (pre-guardrail) looped forever. Phrasing it in second person under the `user`
12124
12749
  // role keeps the memory while removing the false assistant-prose exemplar. The
12125
12750
  // human-facing narration is emitted separately via onProgress above.
12126
- let actionMessage;
12127
- if (actions.length === 1) {
12128
- const aa = actions[0];
12129
- actionMessage = `[You invoked the **${aa.name}** action`;
12130
- // Add parameters if they exist
12131
- if (aa.params && Object.keys(aa.params).length > 0) {
12132
- const paramsList = Object.entries(aa.params)
12133
- .map(([key, value]) => {
12134
- const displayValue = this.formatParamValueForMessage(value);
12135
- return `• **${key}**: ${displayValue}`;
12136
- })
12137
- .join('\n');
12138
- actionMessage += ` with parameters:\n${paramsList}\n]`;
12139
- }
12140
- else {
12141
- actionMessage += '.]';
12142
- }
12143
- }
12144
- else {
12145
- actionMessage = `[You invoked **${actions.length} actions** in parallel:\n\n` + actions.map((aa, index) => {
12146
- let actionText = `${index + 1}. **${aa.name}**`;
12147
- // Add parameters if they exist
12148
- if (aa.params && Object.keys(aa.params).length > 0) {
12149
- const paramsList = Object.entries(aa.params)
12150
- .map(([key, value]) => {
12151
- const displayValue = this.formatParamValueForMessage(value);
12152
- return ` • **${key}**: ${displayValue}`;
12153
- })
12154
- .join('\n');
12155
- actionText += `\n${paramsList}`;
12156
- }
12157
- return actionText;
12158
- }).join('\n\n') + '\n]';
12159
- }
12751
+ const actionMessage = this.buildActionInvocationMessage(actions);
12160
12752
  if (addConversationMessage) {
12161
12753
  // Record as a `user`-role environment annotation (no metadata - permanent record).
12162
12754
  // See the note above on why this is NOT an `assistant` turn.
@@ -12275,10 +12867,7 @@ The context is now within limits. Please retry your request with the recovered c
12275
12867
  // Check if any actions failed
12276
12868
  const failedActions = actionSummaries.filter(a => !a.success);
12277
12869
  // Add user message with the results — compact markdown instead of JSON
12278
- const header = failedActions.length > 0
12279
- ? `${failedActions.length} of ${actionSummaries.length} action(s) failed:`
12280
- : `Action results:`;
12281
- const resultsMessage = `${header}\n${this.formatActionResultsAsMarkdown(actionSummaries)}`;
12870
+ const resultsMessage = this.buildActionResultsMessage(actionSummaries);
12282
12871
  // Build metadata from AI Agent Actions configuration
12283
12872
  // If multiple actions, use the most restrictive (shortest) expiration settings
12284
12873
  let metadata;
@@ -12480,24 +13069,20 @@ The context is now within limits. Please retry your request with the recovered c
12480
13069
  });
12481
13070
  const timeoutMs = tool.TimeoutMs ?? defaultTimeout;
12482
13071
  const response = await ClientToolRequestManager.Instance.RequestClientTool(`ct_${Date.now()}_${Math.random().toString(36).substring(2, 9)}`, tool.Name, tool.Params, params.sessionID, agentRunID, timeoutMs, tool.Description);
12483
- await this.finalizeStepEntity(stepEntity, response.Success, response.ErrorMessage, { result: response.Result });
13072
+ await this.finalizeStepEntity(stepEntity, response.Success, response.ErrorMessage, {
13073
+ result: response.Result,
13074
+ media: response.Media?.map(m => ({ mimeType: m.MimeType, bytes: Math.floor((m.Base64.length * 3) / 4), width: m.Width, height: m.Height }))
13075
+ });
12484
13076
  results.push({
12485
13077
  ToolName: tool.Name,
12486
13078
  Success: response.Success,
12487
13079
  Result: response.Result,
13080
+ Media: response.Media,
12488
13081
  ErrorMessage: response.ErrorMessage
12489
13082
  });
12490
13083
  }
12491
13084
  // Format results as conversation message
12492
- const resultsMarkdown = this.formatClientToolResultsAsMarkdown(results);
12493
- params.conversationMessages.push({
12494
- role: 'user',
12495
- content: resultsMarkdown,
12496
- metadata: {
12497
- turnAdded: this._promptTurnCount,
12498
- messageType: 'client-tool-result'
12499
- }
12500
- });
13085
+ params.conversationMessages.push(BuildClientToolResultMessage(results, this._promptTurnCount, ResolveClientToolResultMaxChars(this._agentTypePromptParams)));
12501
13086
  // If the LLM already declared taskComplete=true alongside the client tools,
12502
13087
  // honor that intent now that tools have executed — no need for another LLM call.
12503
13088
  if (previousDecision.terminateAfterExecution) {
@@ -12518,29 +13103,6 @@ The context is now within limits. Please retry your request with the recovered c
12518
13103
  }
12519
13104
  return await this.executePromptStep(params, config, previousDecision, stepCount);
12520
13105
  }
12521
- /**
12522
- * Format client tool results as a compact markdown summary for the conversation.
12523
- */
12524
- formatClientToolResultsAsMarkdown(results) {
12525
- const failedCount = results.filter(r => !r.Success).length;
12526
- const header = failedCount > 0
12527
- ? `${failedCount} of ${results.length} client tool(s) failed:`
12528
- : 'Client tool results:';
12529
- const lines = results.map(r => {
12530
- const icon = r.Success ? '✓' : '✗';
12531
- let line = `${icon} **${r.ToolName}**: ${r.Success ? 'succeeded' : 'failed'}`;
12532
- if (r.ErrorMessage)
12533
- line += ` — ${r.ErrorMessage}`;
12534
- if (r.Success && r.Result != null) {
12535
- const resultStr = typeof r.Result === 'string' ? r.Result : JSON.stringify(r.Result);
12536
- if (resultStr.length <= 500) {
12537
- line += `\n Result: ${resultStr}`;
12538
- }
12539
- }
12540
- return line;
12541
- });
12542
- return `${header}\n${lines.join('\n')}`;
12543
- }
12544
13106
  /**
12545
13107
  * Executes a 'Skill' next step: activates one or more skills the LLM requested by name.
12546
13108
  * Activating a skill (1) appends its full `Instructions` to the conversation so they take
@@ -13447,12 +14009,10 @@ The context is now within limits. Please retry your request with the recovered c
13447
14009
  /**
13448
14010
  * Resolves the file storage account ID for this agent's file artifacts.
13449
14011
  *
13450
- * Resolution chain (first non-null wins):
13451
- * 1. Runtime override (`params.override?.storageAccountId`)
13452
- * 2. Agent-level (`params.agent.DefaultStorageAccountID`)
13453
- * 3. Category hierarchy — walks up the agent's category tree via `ParentID`
13454
- * 4. Agent Type-level (`agentType.DefaultStorageAccountID`)
13455
- * 5. System fallback — single active storage account, if only one exists
14012
+ * Delegates to {@link AIEngineBase.ResolveAgentStorageAccountID}, the shared golden path:
14013
+ * runtime override (`params.override?.storageAccountId`) → agent → category tree → agent type →
14014
+ * system fallback (the single active account, or the most recently updated one with a
14015
+ * once-per-agent warning when several are active).
13456
14016
  *
13457
14017
  * This method is `protected` so subclasses can override the resolution logic
13458
14018
  * for custom storage routing (e.g., routing by file type or tenant).
@@ -13461,68 +14021,7 @@ The context is now within limits. Please retry your request with the recovered c
13461
14021
  * @returns The resolved FileStorageAccount ID, or null if none configured
13462
14022
  */
13463
14023
  async getStorageAccountID(params) {
13464
- // 1. Runtime override — highest priority
13465
- if (params.override?.storageAccountId) {
13466
- return params.override.storageAccountId;
13467
- }
13468
- // 2. Agent-level override
13469
- if (params.agent.DefaultStorageAccountID) {
13470
- return params.agent.DefaultStorageAccountID;
13471
- }
13472
- // 3. Category tree walk
13473
- const categoryId = params.agent.CategoryID;
13474
- if (categoryId) {
13475
- try {
13476
- const categories = AIEngine.Instance.AgentCategories;
13477
- let currentId = categoryId;
13478
- const visited = new Set();
13479
- while (currentId && !visited.has(currentId)) {
13480
- visited.add(currentId);
13481
- const cat = categories.find(c => UUIDsEqual(c.ID, currentId));
13482
- if (!cat)
13483
- break;
13484
- if (cat.DefaultStorageAccountID)
13485
- return cat.DefaultStorageAccountID;
13486
- currentId = cat.ParentID;
13487
- }
13488
- }
13489
- catch (error) {
13490
- LogError(`Error resolving category storage account: ${error.message}`);
13491
- }
13492
- }
13493
- // 4. Agent Type-level default
13494
- const agentTypeId = params.agent.TypeID;
13495
- if (agentTypeId) {
13496
- const agentType = AIEngine.Instance.AgentTypes.find(at => UUIDsEqual(at.ID, agentTypeId));
13497
- if (agentType?.DefaultStorageAccountID) {
13498
- return agentType.DefaultStorageAccountID;
13499
- }
13500
- }
13501
- // 5. System fallback — use cached metadata (already loaded during engine Config)
13502
- const activeAccounts = FileStorageEngineBase.Instance.AccountsWithProviders
13503
- .filter(a => a.provider.IsActive);
13504
- if (activeAccounts.length === 0) {
13505
- // No storage configured — return null, inline base64 fallback handles it downstream
13506
- return null;
13507
- }
13508
- if (activeAccounts.length === 1) {
13509
- return activeAccounts[0].account.ID;
13510
- }
13511
- // 2+ active accounts but nothing configured at any level
13512
- const agentName = params.agent.Name || params.agent.ID;
13513
- const typeName = params.agent.TypeID
13514
- ? AIEngine.Instance.AgentTypes.find(at => UUIDsEqual(at.ID, params.agent.TypeID))?.Name || params.agent.TypeID
13515
- : 'unknown';
13516
- const categoryName = params.agent.CategoryID
13517
- ? AIEngine.Instance.AgentCategories.find(c => UUIDsEqual(c.ID, params.agent.CategoryID))?.Name || params.agent.CategoryID
13518
- : 'none';
13519
- const accountNames = activeAccounts.map(a => `'${a.account.Name}' (${a.provider.Name})`).join(', ');
13520
- throw new Error(`Multiple active file storage accounts detected (${accountNames}) but no DefaultStorageAccountID is configured ` +
13521
- `for agent '${agentName}', category '${categoryName}', or agent type '${typeName}'.\n` +
13522
- `To fix: Set DefaultStorageAccountID on one of the following (in order of priority):\n` +
13523
- ` 1. Agent '${agentName}' — for this specific agent\n` +
13524
- ` 2. Agent Category '${categoryName}' — for all agents in this category\n` +
13525
- ` 3. Agent Type '${typeName}' — for all agents of this type`);
14024
+ return AIEngine.Instance.ResolveAgentStorageAccountID(params.agent, params.override?.storageAccountId);
13526
14025
  }
13527
14026
  /**
13528
14027
  * Resolves the target user ID from a resolved assignment strategy.
@@ -14315,15 +14814,59 @@ The context is now within limits. Please retry your request with the recovered c
14315
14814
  this._agentRun.CompletedAt = new Date();
14316
14815
  this._agentRun.Success = false;
14317
14816
  this._agentRun.ErrorMessage = message;
14817
+ // Save() writes every column, so an in-memory null here would overwrite the reason the
14818
+ // Stop button wrote on the row. Record the reason the abort signal carried instead. The
14819
+ // signal is consulted first because most call sites pass a fixed message naming WHERE
14820
+ // the run was cancelled ('during prompt execution'), not WHY.
14821
+ const signalReason = this._runAbortController?.signal.reason;
14822
+ this._agentRun.CancellationReason = BaseAgent.CancellationReasonForAbort(typeof signalReason === 'string' ? signalReason : message);
14823
+ // A stop is not supposed to lose the work. The run row only learns its FinalPayload on a
14824
+ // normal finish, but every completed step already recorded the payload as it stood, so
14825
+ // the last of those is what the next turn (chained by LastRunID) starts from.
14826
+ if (!this._agentRun.FinalPayload) {
14827
+ const lastPayload = BaseAgent.LastStepPayload(this._agentRun.Steps);
14828
+ if (lastPayload) {
14829
+ this._agentRun.FinalPayload = lastPayload;
14830
+ }
14831
+ }
14318
14832
  // Calculate total tokens even for cancelled runs
14319
14833
  this.applyTokenStatsToRun(this._agentRun, this.calculateTokenStats());
14320
14834
  await this._agentRun.Save();
14835
+ AgentRunWatchdog.Instance.Untrack(this._agentRun.ID);
14321
14836
  }
14322
14837
  return {
14323
14838
  success: false,
14324
14839
  agentRun: this._agentRun
14325
14840
  };
14326
14841
  }
14842
+ /**
14843
+ * Maps the reason an abort signal carried to the run's `CancellationReason` value: the
14844
+ * watchdog's stop relay raises {@link AgentRunWatchdog.UserCancelAbortReason} for a row marked
14845
+ * `User Request`; the wall-clock guard's reason starts with {@link AgentTimeoutAbortReason};
14846
+ * anything else (an upstream caller's token, a shutdown) is `System`.
14847
+ */
14848
+ static CancellationReasonForAbort(reason) {
14849
+ if (reason === AgentRunWatchdog.UserCancelAbortReason) {
14850
+ return 'User Request';
14851
+ }
14852
+ if (reason && reason.startsWith(BaseAgent.AgentTimeoutAbortReason)) {
14853
+ return 'Timeout';
14854
+ }
14855
+ return 'System';
14856
+ }
14857
+ /**
14858
+ * The payload recorded at the end of the latest step that recorded one, as its serialized
14859
+ * string, or null when no step has. Steps are examined from the highest `StepNumber` down
14860
+ * (array order is spawn order, which a sub-step can interleave).
14861
+ */
14862
+ static LastStepPayload(steps) {
14863
+ if (!steps || steps.length === 0) {
14864
+ return null;
14865
+ }
14866
+ const ordered = [...steps].sort((a, b) => (b.StepNumber ?? 0) - (a.StepNumber ?? 0));
14867
+ const withPayload = ordered.find(s => typeof s.PayloadAtEnd === 'string' && s.PayloadAtEnd.length > 0);
14868
+ return withPayload?.PayloadAtEnd ?? null;
14869
+ }
14327
14870
  /**
14328
14871
  * Finalizes the agent run with success.
14329
14872
  *
@@ -14840,6 +15383,7 @@ The context is now within limits. Please retry your request with the recovered c
14840
15383
  }
14841
15384
  const outcome = await ConversationCompactionManager.CompactIfNeeded({
14842
15385
  ConversationId: params.conversationId,
15386
+ BranchID: params.ConversationBranchID ?? null,
14843
15387
  Agent: params.agent,
14844
15388
  AgentType: config?.agentType || null,
14845
15389
  Budget: budget,
@@ -15040,9 +15584,7 @@ The context is now within limits. Please retry your request with the recovered c
15040
15584
  * @protected
15041
15585
  */
15042
15586
  async compactMessage(message, metadata, params) {
15043
- const originalContent = typeof message.content === 'string'
15044
- ? message.content
15045
- : JSON.stringify(message.content);
15587
+ const originalContent = ReplaceMediaBlocksWithStubs(message.content);
15046
15588
  switch (metadata.compactMode) {
15047
15589
  case 'First N Chars': {
15048
15590
  const length = metadata.compactLength;
@@ -15153,12 +15695,15 @@ The context is now within limits. Please retry your request with the recovered c
15153
15695
  || messageType === 'loop-result';
15154
15696
  }
15155
15697
  estimateTokens(content, modelName) {
15156
- const text = typeof content === 'string'
15157
- ? content
15158
- : JSON.stringify(content);
15159
- // Use heuristic token estimation (fast, good enough for context management)
15160
- // Avoids heavy tokenizer dependencies while providing ~10-20% accuracy
15161
- return this.heuristicTokenCount(text);
15698
+ if (typeof content === 'string') {
15699
+ return this.heuristicTokenCount(content);
15700
+ }
15701
+ // Each media block counts as IMAGE_BLOCK_TOKEN_ESTIMATE, not by its base64 length.
15702
+ let tokens = 0;
15703
+ for (const block of content) {
15704
+ tokens += IsMediaBlock(block) ? IMAGE_BLOCK_TOKEN_ESTIMATE : this.heuristicTokenCount(block.content);
15705
+ }
15706
+ return tokens;
15162
15707
  }
15163
15708
  /**
15164
15709
  * Provides an improved heuristic token count when tokenizer is unavailable.