@just-every/ensemble 0.2.212 → 0.2.213

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/README.md +152 -91
  2. package/dist/cjs/core/ensemble_request.cjs +734 -333
  3. package/dist/cjs/core/ensemble_request.d.ts.map +1 -1
  4. package/dist/cjs/core/ensemble_request.js.map +1 -1
  5. package/dist/cjs/model_providers/base_provider.d.ts.map +1 -1
  6. package/dist/cjs/model_providers/base_provider.js.map +1 -1
  7. package/dist/cjs/model_providers/claude.cjs +72 -72
  8. package/dist/cjs/model_providers/claude.d.ts.map +1 -1
  9. package/dist/cjs/model_providers/claude.js.map +1 -1
  10. package/dist/cjs/model_providers/gemini.cjs +3 -0
  11. package/dist/cjs/model_providers/gemini.d.ts.map +1 -1
  12. package/dist/cjs/model_providers/gemini.js.map +1 -1
  13. package/dist/cjs/model_providers/openai.cjs +41 -112
  14. package/dist/cjs/model_providers/openai.d.ts.map +1 -1
  15. package/dist/cjs/model_providers/openai.js.map +1 -1
  16. package/dist/cjs/model_providers/openai_chat.cjs +55 -24
  17. package/dist/cjs/model_providers/openai_chat.d.ts.map +1 -1
  18. package/dist/cjs/model_providers/openai_chat.js.map +1 -1
  19. package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
  20. package/dist/cjs/types/types.d.ts +20 -2
  21. package/dist/cjs/types/types.d.ts.map +1 -1
  22. package/dist/cjs/utils/agent.cjs +4 -6
  23. package/dist/cjs/utils/agent.d.ts.map +1 -1
  24. package/dist/cjs/utils/agent.js.map +1 -1
  25. package/dist/cjs/utils/ensemble_result.cjs +43 -4
  26. package/dist/cjs/utils/ensemble_result.d.ts +10 -1
  27. package/dist/cjs/utils/ensemble_result.d.ts.map +1 -1
  28. package/dist/cjs/utils/ensemble_result.js.map +1 -1
  29. package/dist/cjs/utils/failure_detection.cjs +292 -0
  30. package/dist/cjs/utils/failure_detection.d.ts +51 -0
  31. package/dist/cjs/utils/failure_detection.d.ts.map +1 -0
  32. package/dist/cjs/utils/failure_detection.js.map +1 -0
  33. package/dist/cjs/utils/json_schema.cjs +490 -0
  34. package/dist/cjs/utils/json_schema.d.ts +10 -0
  35. package/dist/cjs/utils/json_schema.d.ts.map +1 -0
  36. package/dist/cjs/utils/json_schema.js.map +1 -0
  37. package/dist/cjs/utils/tool_execution_manager.cjs +28 -4
  38. package/dist/cjs/utils/tool_execution_manager.d.ts +1 -1
  39. package/dist/cjs/utils/tool_execution_manager.d.ts.map +1 -1
  40. package/dist/cjs/utils/tool_execution_manager.js.map +1 -1
  41. package/dist/cjs/utils/verification.cjs +26 -13
  42. package/dist/cjs/utils/verification.d.ts.map +1 -1
  43. package/dist/cjs/utils/verification.js.map +1 -1
  44. package/dist/core/ensemble_request.d.ts.map +1 -1
  45. package/dist/core/ensemble_request.js +734 -333
  46. package/dist/core/ensemble_request.js.map +1 -1
  47. package/dist/model_providers/base_provider.d.ts.map +1 -1
  48. package/dist/model_providers/base_provider.js.map +1 -1
  49. package/dist/model_providers/claude.d.ts.map +1 -1
  50. package/dist/model_providers/claude.js +72 -72
  51. package/dist/model_providers/claude.js.map +1 -1
  52. package/dist/model_providers/gemini.d.ts.map +1 -1
  53. package/dist/model_providers/gemini.js +3 -0
  54. package/dist/model_providers/gemini.js.map +1 -1
  55. package/dist/model_providers/openai.d.ts.map +1 -1
  56. package/dist/model_providers/openai.js +41 -112
  57. package/dist/model_providers/openai.js.map +1 -1
  58. package/dist/model_providers/openai_chat.d.ts.map +1 -1
  59. package/dist/model_providers/openai_chat.js +55 -24
  60. package/dist/model_providers/openai_chat.js.map +1 -1
  61. package/dist/tsconfig.tsbuildinfo +1 -1
  62. package/dist/types/types.d.ts +20 -2
  63. package/dist/types/types.d.ts.map +1 -1
  64. package/dist/utils/agent.d.ts.map +1 -1
  65. package/dist/utils/agent.js +4 -6
  66. package/dist/utils/agent.js.map +1 -1
  67. package/dist/utils/ensemble_result.d.ts +10 -1
  68. package/dist/utils/ensemble_result.d.ts.map +1 -1
  69. package/dist/utils/ensemble_result.js +43 -4
  70. package/dist/utils/ensemble_result.js.map +1 -1
  71. package/dist/utils/failure_detection.d.ts +51 -0
  72. package/dist/utils/failure_detection.d.ts.map +1 -0
  73. package/dist/utils/failure_detection.js +280 -0
  74. package/dist/utils/failure_detection.js.map +1 -0
  75. package/dist/utils/json_schema.d.ts +10 -0
  76. package/dist/utils/json_schema.d.ts.map +1 -0
  77. package/dist/utils/json_schema.js +486 -0
  78. package/dist/utils/json_schema.js.map +1 -0
  79. package/dist/utils/tool_execution_manager.d.ts +1 -1
  80. package/dist/utils/tool_execution_manager.d.ts.map +1 -1
  81. package/dist/utils/tool_execution_manager.js +28 -4
  82. package/dist/utils/tool_execution_manager.js.map +1 -1
  83. package/dist/utils/verification.d.ts.map +1 -1
  84. package/dist/utils/verification.js +26 -13
  85. package/dist/utils/verification.js.map +1 -1
  86. package/package.json +1 -1
@@ -13,9 +13,13 @@ const pause_controller_js_1 = require("../utils/pause_controller.cjs");
13
13
  const event_controller_js_1 = require("../utils/event_controller.cjs");
14
14
  const trace_context_js_1 = require("../utils/trace_context.cjs");
15
15
  const message_converter_js_1 = require("../utils/message_converter.cjs");
16
- const truncate_utils_js_1 = require("../utils/truncate_utils.cjs");
17
- const MAX_ERROR_ATTEMPTS = 5;
16
+ const failure_detection_js_1 = require("../utils/failure_detection.cjs");
17
+ const json_schema_js_1 = require("../utils/json_schema.cjs");
18
+ const running_tool_tracker_js_1 = require("../utils/running_tool_tracker.cjs");
19
+ const retry_handler_js_1 = require("../utils/retry_handler.cjs");
20
+ const DEFAULT_MAX_ERROR_RETRIES = 4;
18
21
  const DEFAULT_TERMINAL_TOOL_NAMES = new Set(['task_complete', 'task_fatal_error']);
22
+ const TOOL_FAILURE_FINALIZATION_TIMEOUT_MS = 50;
19
23
  const getTerminalToolNames = (agent) => {
20
24
  const toolNames = new Set(DEFAULT_TERMINAL_TOOL_NAMES);
21
25
  for (const name of agent.terminalToolNames ?? []) {
@@ -25,9 +29,81 @@ const getTerminalToolNames = (agent) => {
25
29
  }
26
30
  return toolNames;
27
31
  };
32
+ const hasTerminalTextContent = (content, expectsStructuredOutput) => {
33
+ if (typeof content !== 'string') {
34
+ return false;
35
+ }
36
+ return expectsStructuredOutput ? content.trim().length > 0 : content.length > 0;
37
+ };
38
+ const getMaxErrorRetries = (agent) => {
39
+ const configuredMaxRetries = agent.retryOptions?.maxRetries;
40
+ if (typeof configuredMaxRetries !== 'number' || Number.isNaN(configuredMaxRetries)) {
41
+ return DEFAULT_MAX_ERROR_RETRIES;
42
+ }
43
+ return Math.max(0, Math.floor(configuredMaxRetries));
44
+ };
45
+ const waitForRetryDelay = async (delayMs, abortSignal) => {
46
+ if (delayMs <= 0) {
47
+ return;
48
+ }
49
+ await new Promise((resolve, reject) => {
50
+ if (abortSignal?.aborted) {
51
+ reject(abortSignal.reason ?? new Error('Retry wait aborted'));
52
+ return;
53
+ }
54
+ const timeoutId = setTimeout(() => {
55
+ if (abortSignal && abortListener) {
56
+ abortSignal.removeEventListener('abort', abortListener);
57
+ }
58
+ resolve();
59
+ }, delayMs);
60
+ const abortListener = abortSignal
61
+ ? () => {
62
+ clearTimeout(timeoutId);
63
+ abortSignal.removeEventListener('abort', abortListener);
64
+ reject(abortSignal.reason ?? new Error('Retry wait aborted'));
65
+ }
66
+ : undefined;
67
+ if (abortSignal && abortListener) {
68
+ abortSignal.addEventListener('abort', abortListener, { once: true });
69
+ }
70
+ });
71
+ };
72
+ const getOuterRequestTimeoutMs = (agent) => {
73
+ const timeoutMs = agent.modelSettings?.timeout_ms;
74
+ if (typeof timeoutMs !== 'number' || Number.isNaN(timeoutMs) || timeoutMs <= 0) {
75
+ return undefined;
76
+ }
77
+ return Math.floor(timeoutMs);
78
+ };
79
+ const getRemainingRequestTimeoutMs = (requestTimeoutMs, requestStartedAt) => {
80
+ if (requestTimeoutMs === undefined || requestStartedAt === undefined) {
81
+ return undefined;
82
+ }
83
+ return Math.max(0, requestTimeoutMs - (Date.now() - requestStartedAt));
84
+ };
85
+ const createRequestTimeoutError = (model, timeoutMs) => {
86
+ const error = new Error(`Request generation for ${model} timed out after ${timeoutMs}ms`);
87
+ error.code = 'ETIMEDOUT';
88
+ error.recoverable = false;
89
+ return error;
90
+ };
91
+ const getFailureRetryOverrides = (agent) => ({
92
+ retryableErrors: agent.retryOptions?.additionalRetryableErrors,
93
+ retryableStatusCodes: agent.retryOptions?.additionalRetryableStatusCodes,
94
+ });
28
95
  (0, verification_js_1.setEnsembleRequestFunction)(ensembleRequest);
29
96
  (0, image_to_text_js_1.setEnsembleRequestFunction)(ensembleRequest);
30
97
  async function* ensembleRequest(messages, agent = {}) {
98
+ if (agent.jsonSchema && !agent.modelSettings?.json_schema) {
99
+ agent = {
100
+ ...agent,
101
+ modelSettings: {
102
+ ...agent.modelSettings,
103
+ json_schema: agent.jsonSchema,
104
+ },
105
+ };
106
+ }
31
107
  const conversationHistory = agent?.historyThread || messages;
32
108
  if (agent.instructions) {
33
109
  const alreadyHasInstructions = conversationHistory.some(msg => {
@@ -59,187 +135,228 @@ async function* ensembleRequest(messages, agent = {}) {
59
135
  compactionThreshold: 0.7,
60
136
  });
61
137
  const trace = (0, trace_context_js_1.createTraceContext)(agent, 'chat');
138
+ const lifecycle = new failure_detection_js_1.RequestLifecycleController();
139
+ const maxToolCalls = agent?.maxToolCalls ?? 200;
140
+ const maxRounds = agent?.maxToolCallRoundsPerTurn ?? Infinity;
141
+ const maxErrorRetries = getMaxErrorRetries(agent);
142
+ const maxErrorAttempts = maxErrorRetries + 1;
143
+ const outerRequestTimeoutMs = getOuterRequestTimeoutMs(agent);
144
+ const outerRequestStartedAt = outerRequestTimeoutMs !== undefined ? Date.now() : undefined;
145
+ const modelHistory = [];
146
+ let lastModelUsed;
62
147
  let totalToolCalls = 0;
63
148
  let toolCallRounds = 0;
64
149
  let errorRounds = 0;
150
+ let lastMessageContent = '';
65
151
  let turnStatus = 'completed';
66
152
  let turnEndReason = 'completed';
67
153
  let turnError;
68
- const maxToolCalls = agent?.maxToolCalls ?? 200;
69
- const maxRounds = agent?.maxToolCallRoundsPerTurn ?? Infinity;
70
- let hasToolCalls = false;
71
- let hasError = false;
72
- let lastMessageContent = '';
73
- const modelHistory = [];
154
+ let terminalFailure;
155
+ let terminalFailureEventEmitted = false;
156
+ let finalRound;
74
157
  await trace.emitTurnStart({
75
158
  input_messages: conversationHistory,
76
159
  });
77
160
  try {
78
- do {
79
- hasToolCalls = false;
80
- hasError = false;
81
- let terminalToolSucceededThisRound = false;
82
- let currentRoundRequestId;
83
- const currentRoundMessages = [];
84
- const currentRoundErrors = [];
85
- let currentRoundToolCalls = 0;
86
- let currentRoundRequestDuration;
87
- let currentRoundDurationWithTools;
88
- let currentRoundRequestCost;
89
- const terminalToolNames = getTerminalToolNames(agent);
161
+ const emitRoundAgentDone = async function* (round, model) {
162
+ if (!round.agentDoneEvent) {
163
+ return;
164
+ }
165
+ yield round.agentDoneEvent;
166
+ await (0, event_controller_js_1.emitEvent)(round.agentDoneEvent, round.agentDoneAgent ?? agent, model);
167
+ };
168
+ while (!terminalFailure) {
90
169
  const model = await (0, model_provider_js_1.getModelFromAgent)(agent, 'reasoning_mini', modelHistory);
170
+ const roundRequestId = (0, crypto_1.randomUUID)();
171
+ const startedStatusEvent = lifecycle.begin(roundRequestId);
91
172
  modelHistory.push(model);
92
- const stream = executeRound(model, agent, history, totalToolCalls, maxToolCalls, trace);
93
- try {
94
- for await (const event of stream) {
95
- yield event;
96
- switch (event.type) {
97
- case 'agent_start': {
98
- currentRoundRequestId = event.request_id;
99
- break;
100
- }
101
- case 'message_complete': {
102
- const messageEvent = event;
103
- if (messageEvent.content) {
104
- lastMessageContent = messageEvent.content;
105
- currentRoundMessages.push(messageEvent.content);
106
- }
107
- break;
108
- }
109
- case 'tool_start': {
110
- const toolEvent = event;
111
- if (toolEvent.tool_call) {
112
- const toolName = toolEvent.tool_call.function.name;
113
- currentRoundToolCalls += 1;
114
- await trace.emitToolStart(event.request_id, toolEvent.tool_call.id, {
115
- tool_name: toolName,
116
- arguments: toolEvent.tool_call.function.arguments,
117
- arguments_formatted: toolEvent.tool_call.function.arguments_formatted,
118
- });
119
- if (!terminalToolNames.has(toolName)) {
120
- hasToolCalls = true;
121
- }
122
- }
123
- ++totalToolCalls;
124
- break;
125
- }
126
- case 'tool_done': {
127
- const toolEvent = event;
128
- if (toolEvent.tool_call) {
129
- const toolName = toolEvent.tool_call.function.name;
130
- if (terminalToolNames.has(toolName) && !toolEvent.result?.error) {
131
- terminalToolSucceededThisRound = true;
132
- }
133
- await trace.emitToolDone(event.request_id, toolEvent.tool_call.id, {
134
- tool_name: toolName,
135
- call_id: toolEvent.result?.call_id,
136
- output: toolEvent.result?.output,
137
- error: toolEvent.result?.error,
138
- });
139
- }
140
- break;
141
- }
142
- case 'agent_done': {
143
- const agentDoneEvent = event;
144
- currentRoundRequestDuration = agentDoneEvent.request_duration;
145
- currentRoundDurationWithTools = agentDoneEvent.duration_with_tools;
146
- currentRoundRequestCost = agentDoneEvent.request_cost;
147
- break;
148
- }
149
- case 'error': {
150
- hasError = true;
151
- const errorEvent = event;
152
- if (errorEvent.error) {
153
- currentRoundErrors.push(String(errorEvent.error));
154
- }
155
- break;
156
- }
157
- }
173
+ lastModelUsed = model;
174
+ const round = yield* executeRound({
175
+ roundRequestId,
176
+ model,
177
+ agent,
178
+ history,
179
+ currentToolCalls: totalToolCalls,
180
+ maxToolCalls,
181
+ trace,
182
+ startedStatusEvent,
183
+ requestTimeoutMs: outerRequestTimeoutMs,
184
+ requestStartedAt: outerRequestStartedAt,
185
+ });
186
+ totalToolCalls += round.toolCallsStarted;
187
+ if (round.messages.length > 0) {
188
+ lastMessageContent = round.messages.at(-1) || lastMessageContent;
189
+ }
190
+ if (round.hasFollowupToolCalls) {
191
+ ++toolCallRounds;
192
+ }
193
+ const willRetryForError = (() => {
194
+ if (!round.failure) {
195
+ return false;
158
196
  }
197
+ ++errorRounds;
198
+ return !round.emittedTerminalOutput && round.failure.recoverable && errorRounds <= maxErrorRetries;
199
+ })();
200
+ const willContinueForTools = !round.failure &&
201
+ !round.terminalToolSucceeded &&
202
+ round.hasFollowupToolCalls &&
203
+ toolCallRounds < maxRounds &&
204
+ totalToolCalls < maxToolCalls;
205
+ let requestStatus = 'completed';
206
+ if (round.failure) {
207
+ requestStatus = willRetryForError ? 'error_retrying' : 'error';
159
208
  }
160
- catch (roundError) {
161
- hasError = true;
162
- const errorMessage = roundError instanceof Error ? roundError.message : String(roundError);
163
- currentRoundErrors.push(errorMessage);
164
- yield {
165
- type: 'error',
166
- request_id: currentRoundRequestId,
167
- error: errorMessage,
168
- recoverable: true,
169
- timestamp: new Date().toISOString(),
170
- };
209
+ else if (round.hasFollowupToolCalls && !round.terminalToolSucceeded) {
210
+ requestStatus = willContinueForTools ? 'waiting_for_followup_request' : 'tool_limit_reached';
211
+ }
212
+ await trace.emitRequestEnd(round.requestId, {
213
+ status: requestStatus,
214
+ will_continue: willRetryForError || willContinueForTools,
215
+ tool_calls: round.toolCallsStarted,
216
+ final_response: round.messages.length > 0 ? round.messages.join('\n') : undefined,
217
+ errors: round.errors.length > 0 ? round.errors : undefined,
218
+ request_duration_ms: round.requestDuration,
219
+ duration_with_tools_ms: round.durationWithTools,
220
+ request_cost: round.requestCost,
221
+ });
222
+ if (round.failure) {
223
+ const terminalRoundFailure = willRetryForError
224
+ ? round.failure
225
+ : {
226
+ ...round.failure,
227
+ recoverable: false,
228
+ terminal: true,
229
+ };
230
+ const errorEvent = (0, failure_detection_js_1.toErrorEvent)(terminalRoundFailure, {
231
+ request_id: round.requestId,
232
+ });
233
+ yield errorEvent;
234
+ await (0, event_controller_js_1.emitEvent)(errorEvent, agent, model);
235
+ if (willRetryForError) {
236
+ agent.retryOptions?.onRetry?.({
237
+ message: round.failure.error,
238
+ code: round.failure.code,
239
+ details: round.failure.details,
240
+ recoverable: round.failure.recoverable,
241
+ }, errorRounds);
242
+ const retryingEvent = lifecycle.retrying(round.failure, errorRounds, maxErrorAttempts);
243
+ if (retryingEvent) {
244
+ yield retryingEvent;
245
+ await (0, event_controller_js_1.emitEvent)(retryingEvent, agent, model);
246
+ }
247
+ const retryDelayMs = (0, retry_handler_js_1.calculateDelay)(errorRounds, agent.retryOptions);
248
+ const remainingTimeoutMs = getRemainingRequestTimeoutMs(outerRequestTimeoutMs, outerRequestStartedAt);
249
+ const boundedRetryDelayMs = remainingTimeoutMs === undefined
250
+ ? retryDelayMs
251
+ : remainingTimeoutMs < retryDelayMs
252
+ ? 0
253
+ : retryDelayMs;
254
+ yield* emitRoundAgentDone(round, model);
255
+ await waitForRetryDelay(boundedRetryDelayMs, agent.abortSignal);
256
+ continue;
257
+ }
258
+ terminalFailure = terminalRoundFailure;
259
+ terminalFailureEventEmitted = true;
260
+ finalRound = { round, model };
261
+ break;
171
262
  }
172
- if (terminalToolSucceededThisRound) {
173
- hasToolCalls = false;
174
- hasError = false;
263
+ if (round.terminalToolSucceeded) {
264
+ finalRound = { round, model };
265
+ break;
175
266
  }
176
- if (hasToolCalls) {
177
- ++toolCallRounds;
267
+ if (willContinueForTools) {
178
268
  if (agent.modelSettings?.tool_choice) {
269
+ agent = {
270
+ ...agent,
271
+ modelSettings: {
272
+ ...agent.modelSettings,
273
+ },
274
+ };
179
275
  delete agent.modelSettings.tool_choice;
180
276
  }
277
+ yield* emitRoundAgentDone(round, model);
278
+ continue;
181
279
  }
182
- if (hasError) {
183
- ++errorRounds;
184
- }
185
- const willRetryForError = hasError && errorRounds < MAX_ERROR_ATTEMPTS;
186
- const willContinueForTools = hasToolCalls && toolCallRounds < maxRounds && totalToolCalls < maxToolCalls;
187
- const willContinue = willRetryForError || willContinueForTools;
188
- let requestStatus = 'completed';
189
- if (hasError) {
190
- requestStatus = willContinue ? 'error_retrying' : 'error';
191
- }
192
- else if (hasToolCalls) {
193
- requestStatus = willContinue ? 'waiting_for_followup_request' : 'tool_limit_reached';
194
- }
195
- if (currentRoundRequestId) {
196
- await trace.emitRequestEnd(currentRoundRequestId, {
197
- status: requestStatus,
198
- will_continue: willContinue,
199
- tool_calls: currentRoundToolCalls,
200
- final_response: currentRoundMessages.length > 0 ? currentRoundMessages.join('\n') : undefined,
201
- errors: currentRoundErrors.length > 0 ? currentRoundErrors : undefined,
202
- request_duration_ms: currentRoundRequestDuration,
203
- duration_with_tools_ms: currentRoundDurationWithTools,
204
- request_cost: currentRoundRequestCost,
280
+ if (round.hasFollowupToolCalls && !round.terminalToolSucceeded) {
281
+ terminalFailure = (0, failure_detection_js_1.normalizeFailure)(new Error(toolCallRounds >= maxRounds
282
+ ? `Tool call rounds limit reached (${maxRounds}).`
283
+ : `Tool call limit reached (${maxToolCalls}).`), {
284
+ recoverable: false,
285
+ reason: toolCallRounds >= maxRounds
286
+ ? 'max_tool_call_rounds_reached'
287
+ : 'max_tool_calls_reached',
288
+ ...getFailureRetryOverrides(agent),
205
289
  });
290
+ finalRound = { round, model };
291
+ break;
206
292
  }
207
- } while ((hasError && errorRounds < MAX_ERROR_ATTEMPTS) ||
208
- (hasToolCalls && toolCallRounds < maxRounds && totalToolCalls < maxToolCalls));
209
- if (hasToolCalls && toolCallRounds >= maxRounds) {
210
- console.log('[ensembleRequest] Tool call rounds limit reached');
211
- turnEndReason = 'max_tool_call_rounds_reached';
293
+ finalRound = { round, model };
294
+ break;
212
295
  }
213
- else if (hasToolCalls && totalToolCalls >= maxToolCalls) {
214
- console.log('[ensembleRequest] Total tool calls limit reached');
215
- turnEndReason = 'max_tool_calls_reached';
296
+ if (!terminalFailure && agent.verifier && lastMessageContent) {
297
+ const verification = yield* performVerification(agent, lastMessageContent, await history.getMessages());
298
+ if (!verification.passed) {
299
+ terminalFailure = (0, failure_detection_js_1.normalizeFailure)(new Error(verification.error || 'Verification failed'), {
300
+ recoverable: false,
301
+ reason: 'verification_failed',
302
+ ...getFailureRetryOverrides(agent),
303
+ });
304
+ }
216
305
  }
217
- else if (hasError && errorRounds >= MAX_ERROR_ATTEMPTS) {
306
+ if (terminalFailure) {
218
307
  turnStatus = 'error';
219
- turnEndReason = 'max_error_attempts_reached';
308
+ turnEndReason = terminalFailure.reason || 'terminal_failure';
309
+ turnError = terminalFailure.error;
310
+ if (!terminalFailureEventEmitted) {
311
+ const errorEvent = (0, failure_detection_js_1.toErrorEvent)(terminalFailure, {
312
+ request_id: lifecycle.getRequestId(),
313
+ });
314
+ yield errorEvent;
315
+ await (0, event_controller_js_1.emitEvent)(errorEvent, agent, lastModelUsed);
316
+ }
317
+ const failedEvent = lifecycle.fail(terminalFailure, errorRounds || 1, maxErrorAttempts);
318
+ if (failedEvent) {
319
+ yield failedEvent;
320
+ await (0, event_controller_js_1.emitEvent)(failedEvent, agent, lastModelUsed);
321
+ }
220
322
  }
221
- if (agent?.verifier && lastMessageContent) {
222
- const verificationResult = await performVerification(agent, lastMessageContent, await history.getMessages());
223
- if (verificationResult) {
224
- for await (const event of verificationResult) {
225
- yield event;
226
- }
323
+ else {
324
+ const completedEvent = lifecycle.complete();
325
+ if (completedEvent) {
326
+ yield completedEvent;
327
+ await (0, event_controller_js_1.emitEvent)(completedEvent, agent, lastModelUsed);
227
328
  }
228
329
  }
330
+ if (finalRound) {
331
+ yield* emitRoundAgentDone(finalRound.round, finalRound.model);
332
+ }
229
333
  }
230
334
  catch (err) {
231
- const error = err;
335
+ if (!lifecycle.getRequestId()) {
336
+ const startedEvent = lifecycle.begin((0, crypto_1.randomUUID)());
337
+ if (startedEvent) {
338
+ yield startedEvent;
339
+ await (0, event_controller_js_1.emitEvent)(startedEvent, agent, lastModelUsed);
340
+ }
341
+ }
342
+ const failure = (0, failure_detection_js_1.normalizeFailure)(err, {
343
+ recoverable: false,
344
+ reason: 'exception',
345
+ ...getFailureRetryOverrides(agent),
346
+ });
232
347
  turnStatus = 'error';
233
348
  turnEndReason = 'exception';
234
- turnError = error.message || 'Unknown error';
235
- yield {
236
- type: 'error',
237
- error: error.message || 'Unknown error',
238
- code: error.code,
239
- details: error.details,
240
- recoverable: error.recoverable,
241
- timestamp: new Date().toISOString(),
242
- };
349
+ turnError = failure.error;
350
+ const errorEvent = (0, failure_detection_js_1.toErrorEvent)(failure, {
351
+ request_id: lifecycle.getRequestId(),
352
+ });
353
+ yield errorEvent;
354
+ await (0, event_controller_js_1.emitEvent)(errorEvent, agent, lastModelUsed);
355
+ const failedEvent = lifecycle.fail(failure, errorRounds || 1, maxErrorAttempts);
356
+ if (failedEvent) {
357
+ yield failedEvent;
358
+ await (0, event_controller_js_1.emitEvent)(failedEvent, agent, lastModelUsed);
359
+ }
243
360
  }
244
361
  finally {
245
362
  await trace.emitTurnEnd(turnStatus, turnEndReason, {
@@ -254,14 +371,29 @@ async function* ensembleRequest(messages, agent = {}) {
254
371
  };
255
372
  }
256
373
  }
257
- async function* executeRound(model, agent, history, currentToolCalls, maxToolCalls, trace) {
258
- const requestId = (0, crypto_1.randomUUID)();
374
+ async function* executeRound(options) {
375
+ const { roundRequestId, model, agent, history, currentToolCalls, maxToolCalls, trace, startedStatusEvent } = options;
259
376
  const startTime = Date.now();
260
377
  let totalCost = 0;
261
378
  let messages = await history.getMessages(model);
379
+ let roundAgentDefinition = agent;
380
+ let requestGuard;
381
+ let toolExecutionGuard;
382
+ let roundAgent = agent;
383
+ let provider;
384
+ let stream;
385
+ const roundSummary = {
386
+ requestId: roundRequestId,
387
+ messages: [],
388
+ errors: [],
389
+ toolCallsStarted: 0,
390
+ hasFollowupToolCalls: false,
391
+ emittedTerminalOutput: false,
392
+ terminalToolSucceeded: false,
393
+ };
262
394
  const agentStartEvent = {
263
395
  type: 'agent_start',
264
- request_id: requestId,
396
+ request_id: roundRequestId,
265
397
  input: 'content' in messages[0] && typeof messages[0].content === 'string' ? messages[0].content : undefined,
266
398
  timestamp: new Date().toISOString(),
267
399
  agent: {
@@ -278,199 +410,447 @@ async function* executeRound(model, agent, history, currentToolCalls, maxToolCal
278
410
  };
279
411
  yield agentStartEvent;
280
412
  await (0, event_controller_js_1.emitEvent)(agentStartEvent, agent, model);
281
- if (agent.onRequest) {
282
- [agent, messages] = await agent.onRequest(agent, messages);
413
+ try {
414
+ if (roundAgentDefinition.onRequest) {
415
+ const [nextAgent, nextMessages] = await roundAgentDefinition.onRequest(roundAgentDefinition, messages);
416
+ roundAgentDefinition = nextAgent;
417
+ messages = nextMessages;
418
+ }
419
+ const remainingTimeoutMs = getRemainingRequestTimeoutMs(options.requestTimeoutMs, options.requestStartedAt);
420
+ const needsRequestGuard = Boolean(roundAgentDefinition.abortSignal || remainingTimeoutMs !== undefined);
421
+ if (needsRequestGuard) {
422
+ if (options.requestTimeoutMs !== undefined && remainingTimeoutMs !== undefined && remainingTimeoutMs <= 0) {
423
+ throw createRequestTimeoutError(model, options.requestTimeoutMs);
424
+ }
425
+ requestGuard = (0, failure_detection_js_1.createOperationGuard)({
426
+ operationName: `Request generation for ${model}`,
427
+ abortSignal: roundAgentDefinition.abortSignal,
428
+ timeoutMs: remainingTimeoutMs,
429
+ });
430
+ roundAgent = {
431
+ ...roundAgentDefinition,
432
+ abortSignal: requestGuard.signal,
433
+ };
434
+ }
435
+ else {
436
+ roundAgent = roundAgentDefinition;
437
+ }
438
+ await (0, pause_controller_js_1.waitWhilePaused)(100, roundAgent.abortSignal);
439
+ toolExecutionGuard = (0, failure_detection_js_1.createOperationGuard)({
440
+ operationName: `Tool execution for ${model}`,
441
+ abortSignal: roundAgent.abortSignal,
442
+ });
443
+ if (startedStatusEvent) {
444
+ yield startedStatusEvent;
445
+ await (0, event_controller_js_1.emitEvent)(startedStatusEvent, roundAgent, model);
446
+ }
447
+ provider = (0, model_provider_js_1.getModelProvider)(model);
448
+ await trace.emitRequestStart(roundRequestId, {
449
+ agent_id: roundAgent.agent_id,
450
+ provider: provider.provider_id,
451
+ model,
452
+ payload: {
453
+ messages,
454
+ model_settings: roundAgent.modelSettings,
455
+ tool_names: roundAgent.tools?.map(tool => tool.definition.function.name) || [],
456
+ },
457
+ });
458
+ const rawStream = provider.createResponseStream(messages, model, roundAgent, roundRequestId);
459
+ stream = (0, failure_detection_js_1.streamWithAbortAndTimeout)(rawStream, {
460
+ abortSignal: requestGuard?.signal,
461
+ });
283
462
  }
284
- await (0, pause_controller_js_1.waitWhilePaused)(100, agent.abortSignal);
285
- const provider = (0, model_provider_js_1.getModelProvider)(model);
286
- await trace.emitRequestStart(requestId, {
287
- agent_id: agent.agent_id,
288
- provider: provider.provider_id,
289
- model,
290
- payload: {
291
- messages,
292
- model_settings: agent.modelSettings,
293
- tool_names: agent.tools?.map(tool => tool.definition.function.name) || [],
294
- },
295
- });
296
- const stream = 'createResponseStreamWithRetry' in provider
297
- ? provider.createResponseStreamWithRetry(messages, model, agent, requestId)
298
- : provider.createResponseStream(messages, model, agent, requestId);
299
- const toolPromises = [];
463
+ catch (error) {
464
+ requestGuard?.cleanup();
465
+ toolExecutionGuard?.cleanup();
466
+ const failure = (0, failure_detection_js_1.normalizeFailure)(error, {
467
+ reason: 'request_setup_failed',
468
+ ...getFailureRetryOverrides(agent),
469
+ });
470
+ roundSummary.failure = failure;
471
+ roundSummary.errors.push(failure.error);
472
+ roundSummary.requestDuration = Date.now() - startTime;
473
+ roundSummary.durationWithTools = roundSummary.requestDuration;
474
+ roundSummary.agentDoneEvent = {
475
+ type: 'agent_done',
476
+ request_id: roundRequestId,
477
+ request_duration: roundSummary.requestDuration,
478
+ duration_with_tools: roundSummary.durationWithTools,
479
+ timestamp: new Date().toISOString(),
480
+ };
481
+ roundSummary.agentDoneAgent = roundAgentDefinition;
482
+ return roundSummary;
483
+ }
484
+ const terminalToolNames = getTerminalToolNames(roundAgent);
485
+ const expectsStructuredOutput = Boolean(roundAgent.modelSettings?.json_schema?.schema);
486
+ const structuredOutputSchema = roundAgent.modelSettings?.json_schema?.strict === true
487
+ ? roundAgent.modelSettings.json_schema.schema
488
+ : undefined;
489
+ const toolExecutions = [];
300
490
  const toolCallFormattedArgs = new Map();
301
491
  const toolEventBuffer = [];
302
492
  let sawToolCallThisRound = false;
303
- agent.onToolEvent = async (event) => {
493
+ let sawTerminalProviderOutcome = false;
494
+ roundAgent.onToolEvent = async (event) => {
304
495
  toolEventBuffer.push(event);
305
496
  };
306
- for await (let event of stream) {
307
- event = { ...event, request_id: requestId };
308
- if (event.type === 'tool_start') {
309
- const toolEvent = event;
310
- if (toolEvent.tool_call) {
311
- const toolCall = toolEvent.tool_call;
312
- let argumentsFormatted;
313
- try {
314
- const tool = agent.tools?.find(t => t.definition.function.name === toolCall.function.name);
315
- if (tool && 'definition' in tool && tool.definition.function.parameters.properties) {
316
- const parsedArgs = JSON.parse(toolCall.function.arguments || '{}');
317
- if (typeof parsedArgs === 'object' && parsedArgs !== null && !Array.isArray(parsedArgs)) {
318
- const paramNames = Object.keys(tool.definition.function.parameters.properties);
319
- const orderedArgs = {};
320
- for (const param of paramNames) {
321
- if (param in parsedArgs) {
322
- orderedArgs[param] = parsedArgs[param];
323
- }
324
- }
325
- argumentsFormatted = JSON.stringify(orderedArgs, null, 2);
326
- }
327
- }
328
- }
329
- catch (error) {
330
- console.debug('Failed to format tool arguments:', error);
497
+ const finalizeToolResults = async function* (mode) {
498
+ const waitForPendingExecutions = async (executions, timeoutMs) => {
499
+ if (executions.length === 0) {
500
+ return;
501
+ }
502
+ const completionPromise = Promise.all(executions.map(execution => execution.promise.then(() => undefined)));
503
+ if (timeoutMs === undefined) {
504
+ await completionPromise;
505
+ return;
506
+ }
507
+ await Promise.race([
508
+ completionPromise,
509
+ new Promise(resolve => setTimeout(resolve, timeoutMs)),
510
+ ]);
511
+ };
512
+ const waitForAllExecutions = async (executions, abortSignal) => {
513
+ if (executions.length === 0) {
514
+ return true;
515
+ }
516
+ const completionPromise = Promise.all(executions.map(execution => execution.promise.then(() => undefined))).then(() => true);
517
+ if (!abortSignal) {
518
+ return completionPromise;
519
+ }
520
+ if (abortSignal.aborted) {
521
+ return false;
522
+ }
523
+ return new Promise(resolve => {
524
+ const abortListener = () => {
525
+ abortSignal.removeEventListener('abort', abortListener);
526
+ resolve(false);
527
+ };
528
+ completionPromise.then(completed => {
529
+ abortSignal.removeEventListener('abort', abortListener);
530
+ resolve(completed);
531
+ });
532
+ abortSignal.addEventListener('abort', abortListener, { once: true });
533
+ });
534
+ };
535
+ let finalizationMode = mode;
536
+ if (finalizationMode === 'wait_all') {
537
+ const completedAllExecutions = await waitForAllExecutions(toolExecutions.filter(execution => !execution.settled), requestGuard?.signal);
538
+ if (!completedAllExecutions) {
539
+ finalizationMode = 'bounded_failure';
540
+ }
541
+ }
542
+ if (finalizationMode === 'bounded_failure') {
543
+ toolExecutionGuard?.abort(roundSummary.failure?.error
544
+ ? new Error(roundSummary.failure.error)
545
+ : new Error('Request finalized after terminal provider failure.'));
546
+ await waitForPendingExecutions(toolExecutions.filter(execution => !execution.settled), TOOL_FAILURE_FINALIZATION_TIMEOUT_MS);
547
+ for (const execution of toolExecutions) {
548
+ if (!execution.settled) {
549
+ running_tool_tracker_js_1.runningToolTracker.abortRunningTool(execution.toolCall.id || execution.toolCall.call_id || '');
331
550
  }
332
- if (argumentsFormatted) {
333
- toolCallFormattedArgs.set(toolCall.id, argumentsFormatted);
551
+ }
552
+ await waitForPendingExecutions(toolExecutions.filter(execution => !execution.settled), TOOL_FAILURE_FINALIZATION_TIMEOUT_MS);
553
+ for (const execution of toolExecutions) {
554
+ const runningToolId = execution.toolCall.id || execution.toolCall.call_id || '';
555
+ if (execution.settled) {
556
+ const leakedRunningTool = runningToolId
557
+ ? running_tool_tracker_js_1.runningToolTracker.getRunningTool(runningToolId)
558
+ : undefined;
559
+ if (leakedRunningTool) {
560
+ const failureResult = execution.result ?? createToolFinalizationFailureResult(execution.toolCall);
561
+ await running_tool_tracker_js_1.runningToolTracker.failRunningTool(runningToolId, failureResult.error || 'Tool execution failed during bounded finalization.');
562
+ }
334
563
  }
335
- const modifiedEvent = {
336
- ...event,
337
- tool_call: {
338
- ...toolCall,
339
- function: {
340
- ...toolCall.function,
341
- arguments_formatted: argumentsFormatted,
342
- },
564
+ }
565
+ }
566
+ const toolResults = finalizationMode === 'wait_all'
567
+ ? await Promise.all(toolExecutions.map(execution => execution.promise))
568
+ : toolExecutions.flatMap(execution => (execution.settled && execution.result ? [execution.result] : []));
569
+ for (const toolResult of toolResults) {
570
+ const toolName = toolResult.toolCall.function.name;
571
+ const isTerminalTool = terminalToolNames.has(toolName);
572
+ const formattedArgs = toolCallFormattedArgs.get(toolResult.toolCall.id);
573
+ const toolCallWithFormattedArgs = formattedArgs
574
+ ? {
575
+ ...toolResult.toolCall,
576
+ function: {
577
+ ...toolResult.toolCall.function,
578
+ arguments_formatted: formattedArgs,
343
579
  },
580
+ }
581
+ : toolResult.toolCall;
582
+ const toolDoneEvent = {
583
+ type: 'tool_done',
584
+ request_id: roundRequestId,
585
+ tool_call: toolCallWithFormattedArgs,
586
+ result: {
587
+ call_id: toolResult.call_id || toolResult.id,
588
+ output: toolResult.output,
589
+ error: toolResult.error,
590
+ },
591
+ };
592
+ if (isTerminalTool && !toolResult.error) {
593
+ roundSummary.terminalToolSucceeded = true;
594
+ }
595
+ yield toolDoneEvent;
596
+ await (0, event_controller_js_1.emitEvent)(toolDoneEvent, roundAgent, model);
597
+ await trace.emitToolDone(roundRequestId, toolResult.toolCall.id, {
598
+ tool_name: toolName,
599
+ call_id: toolResult.call_id,
600
+ output: toolResult.output,
601
+ error: toolResult.error,
602
+ });
603
+ if (!isTerminalTool) {
604
+ const functionOutput = (0, message_converter_js_1.convertToFunctionCallOutput)(toolResult, model, 'completed');
605
+ history.add(functionOutput);
606
+ yield {
607
+ type: 'response_output',
608
+ message: functionOutput,
609
+ request_id: roundRequestId,
344
610
  };
345
- event = modifiedEvent;
346
611
  }
347
612
  }
348
- yield event;
349
- await (0, event_controller_js_1.emitEvent)(event, agent, model);
350
- switch (event.type) {
351
- case 'cost_update': {
352
- const costEvent = event;
353
- if (costEvent.usage?.cost) {
354
- totalCost += costEvent.usage.cost;
613
+ for (const bufferedEvent of toolEventBuffer) {
614
+ yield { ...bufferedEvent, request_id: roundRequestId };
615
+ }
616
+ };
617
+ try {
618
+ for await (let event of stream) {
619
+ event = { ...event, request_id: roundRequestId };
620
+ if (event.type === 'error') {
621
+ const failure = (0, failure_detection_js_1.normalizeFailure)(event, {
622
+ error: event.error,
623
+ recoverable: event.recoverable,
624
+ code: event.code,
625
+ details: event.details,
626
+ ...getFailureRetryOverrides(agent),
627
+ });
628
+ roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, failure);
629
+ roundSummary.errors.push(failure.error);
630
+ continue;
631
+ }
632
+ if (event.type === 'message_complete' && structuredOutputSchema) {
633
+ const messageEvent = event;
634
+ if (hasTerminalTextContent(messageEvent.content, true)) {
635
+ const validationResult = (0, json_schema_js_1.validateJsonResponseContent)(messageEvent.content, structuredOutputSchema);
636
+ if (!validationResult.ok && 'error' in validationResult) {
637
+ const failure = (0, failure_detection_js_1.normalizeFailure)(new Error(validationResult.error), {
638
+ recoverable: false,
639
+ reason: 'structured_output_validation_failed',
640
+ });
641
+ roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, failure);
642
+ roundSummary.errors.push(failure.error);
643
+ continue;
644
+ }
355
645
  }
356
- break;
357
646
  }
358
- case 'message_complete': {
647
+ if (event.type === 'tool_start') {
648
+ const toolEvent = event;
649
+ if (toolEvent.tool_call) {
650
+ const toolCall = toolEvent.tool_call;
651
+ let argumentsFormatted;
652
+ try {
653
+ const tool = roundAgent.tools?.find(t => t.definition.function.name === toolCall.function.name);
654
+ if (tool?.definition.function.parameters.properties) {
655
+ const parsedArgs = JSON.parse(toolCall.function.arguments || '{}');
656
+ if (typeof parsedArgs === 'object' && parsedArgs !== null && !Array.isArray(parsedArgs)) {
657
+ const paramNames = Object.keys(tool.definition.function.parameters.properties);
658
+ const orderedArgs = {};
659
+ for (const param of paramNames) {
660
+ if (param in parsedArgs) {
661
+ orderedArgs[param] = parsedArgs[param];
662
+ }
663
+ }
664
+ argumentsFormatted = JSON.stringify(orderedArgs, null, 2);
665
+ }
666
+ }
667
+ }
668
+ catch (error) {
669
+ console.debug('Failed to format tool arguments:', error);
670
+ }
671
+ if (argumentsFormatted) {
672
+ toolCallFormattedArgs.set(toolCall.id, argumentsFormatted);
673
+ }
674
+ event = {
675
+ ...event,
676
+ tool_call: {
677
+ ...toolCall,
678
+ function: {
679
+ ...toolCall.function,
680
+ arguments_formatted: argumentsFormatted,
681
+ },
682
+ },
683
+ };
684
+ }
685
+ }
686
+ if (event.type === 'message_complete') {
359
687
  const messageEvent = event;
360
- if (sawToolCallThisRound) {
688
+ if (hasTerminalTextContent(messageEvent.content, expectsStructuredOutput)) {
689
+ sawTerminalProviderOutcome = true;
690
+ }
691
+ }
692
+ else if (event.type === 'tool_start' || event.type === 'file_complete') {
693
+ sawTerminalProviderOutcome = true;
694
+ }
695
+ yield event;
696
+ await (0, event_controller_js_1.emitEvent)(event, roundAgent, model);
697
+ switch (event.type) {
698
+ case 'cost_update': {
699
+ const costEvent = event;
700
+ if (costEvent.usage?.cost) {
701
+ totalCost += costEvent.usage.cost;
702
+ }
361
703
  break;
362
704
  }
363
- if (messageEvent.thinking_content ||
364
- (!messageEvent.content && messageEvent.message_id)) {
365
- const thinkingMessage = (0, message_converter_js_1.convertToThinkingMessage)(messageEvent, model);
366
- if (agent.onThinking) {
367
- await agent.onThinking(thinkingMessage);
705
+ case 'message_complete': {
706
+ const messageEvent = event;
707
+ if (sawToolCallThisRound) {
708
+ break;
368
709
  }
369
- history.add(thinkingMessage);
370
- yield {
371
- type: 'response_output',
372
- message: thinkingMessage,
373
- request_id: requestId,
374
- };
710
+ if (messageEvent.thinking_content ||
711
+ (!messageEvent.content && messageEvent.message_id)) {
712
+ const thinkingMessage = (0, message_converter_js_1.convertToThinkingMessage)(messageEvent, model);
713
+ if (roundAgent.onThinking) {
714
+ await roundAgent.onThinking(thinkingMessage);
715
+ }
716
+ history.add(thinkingMessage);
717
+ yield {
718
+ type: 'response_output',
719
+ message: thinkingMessage,
720
+ request_id: roundRequestId,
721
+ };
722
+ }
723
+ if (hasTerminalTextContent(messageEvent.content, expectsStructuredOutput)) {
724
+ roundSummary.emittedTerminalOutput = true;
725
+ roundSummary.messages.push(messageEvent.content);
726
+ const contentMessage = (0, message_converter_js_1.convertToOutputMessage)(messageEvent, model, 'completed');
727
+ if (roundAgent.onResponse) {
728
+ await roundAgent.onResponse(contentMessage);
729
+ }
730
+ history.add(contentMessage);
731
+ yield {
732
+ type: 'response_output',
733
+ message: contentMessage,
734
+ request_id: roundRequestId,
735
+ };
736
+ }
737
+ break;
738
+ }
739
+ case 'file_complete': {
740
+ roundSummary.emittedTerminalOutput = true;
741
+ break;
375
742
  }
376
- if (messageEvent.content) {
377
- const contentMessage = (0, message_converter_js_1.convertToOutputMessage)(messageEvent, model, 'completed');
378
- if (agent.onResponse) {
379
- await agent.onResponse(contentMessage);
743
+ case 'tool_start': {
744
+ const toolEvent = event;
745
+ if (!toolEvent.tool_call) {
746
+ break;
747
+ }
748
+ if (!sawToolCallThisRound) {
749
+ roundSummary.emittedTerminalOutput = false;
750
+ roundSummary.messages = [];
380
751
  }
381
- history.add(contentMessage);
752
+ sawToolCallThisRound = true;
753
+ const remainingCalls = maxToolCalls - currentToolCalls - roundSummary.toolCallsStarted;
754
+ if (remainingCalls <= 0) {
755
+ console.warn(`Tool call limit reached (${maxToolCalls}). Skipping tool calls.`);
756
+ const failure = (0, failure_detection_js_1.normalizeFailure)(new Error(`Tool call limit reached (${maxToolCalls}). Cannot execute tool ${toolEvent.tool_call.function.name}.`), {
757
+ recoverable: false,
758
+ reason: 'max_tool_calls_reached',
759
+ ...getFailureRetryOverrides(agent),
760
+ });
761
+ roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, failure);
762
+ if (!roundSummary.errors.includes(failure.error)) {
763
+ roundSummary.errors.push(failure.error);
764
+ }
765
+ break;
766
+ }
767
+ const toolCall = toolEvent.tool_call;
768
+ const functionCall = (0, message_converter_js_1.convertToFunctionCall)(toolCall, model, 'completed');
769
+ history.add(functionCall);
382
770
  yield {
383
771
  type: 'response_output',
384
- message: contentMessage,
385
- request_id: requestId,
772
+ message: functionCall,
773
+ request_id: roundRequestId,
386
774
  };
387
- }
388
- break;
389
- }
390
- case 'tool_start': {
391
- const toolEvent = event;
392
- if (!toolEvent.tool_call) {
393
- break;
394
- }
395
- sawToolCallThisRound = true;
396
- const remainingCalls = maxToolCalls - currentToolCalls;
397
- if (remainingCalls <= 0) {
398
- console.warn(`Tool call limit reached (${maxToolCalls}). Skipping tool calls.`);
775
+ ++roundSummary.toolCallsStarted;
776
+ if (!terminalToolNames.has(toolCall.function.name)) {
777
+ roundSummary.hasFollowupToolCalls = true;
778
+ }
779
+ await trace.emitToolStart(roundRequestId, toolCall.id, {
780
+ tool_name: toolCall.function.name,
781
+ arguments: toolCall.function.arguments,
782
+ arguments_formatted: toolCall.function.arguments_formatted,
783
+ });
784
+ const trackedExecution = {
785
+ toolCall,
786
+ promise: processToolCall(toolCall, {
787
+ ...roundAgent,
788
+ abortSignal: toolExecutionGuard?.signal ?? roundAgent.abortSignal,
789
+ }),
790
+ settled: false,
791
+ };
792
+ trackedExecution.promise = trackedExecution.promise.then(result => {
793
+ if (!trackedExecution.settled) {
794
+ trackedExecution.settled = true;
795
+ trackedExecution.result = result;
796
+ }
797
+ return trackedExecution.result ?? result;
798
+ });
799
+ toolExecutions.push(trackedExecution);
399
800
  break;
400
801
  }
401
- const toolCall = toolEvent.tool_call;
402
- const functionCall = (0, message_converter_js_1.convertToFunctionCall)(toolCall, model, 'completed');
403
- toolPromises.push(processToolCall(toolCall, agent));
404
- history.add(functionCall);
405
- yield {
406
- type: 'response_output',
407
- message: functionCall,
408
- request_id: requestId,
409
- };
410
- break;
411
- }
412
- case 'error': {
413
- console.error('[executeRound] Error event:', (0, truncate_utils_js_1.truncateLargeValues)(event.error));
414
- break;
415
802
  }
416
803
  }
417
804
  }
418
- const request_duration = Date.now() - startTime;
419
- const toolResults = await Promise.all(toolPromises);
420
- const terminalToolNames = getTerminalToolNames(agent);
421
- for (const toolResult of toolResults) {
422
- const toolName = toolResult.toolCall.function.name;
423
- const isTerminalTool = terminalToolNames.has(toolName);
424
- const formattedArgs = toolCallFormattedArgs.get(toolResult.toolCall.id);
425
- const toolCallWithFormattedArgs = formattedArgs
426
- ? {
427
- ...toolResult.toolCall,
428
- function: {
429
- ...toolResult.toolCall.function,
430
- arguments_formatted: formattedArgs,
431
- },
432
- }
433
- : toolResult.toolCall;
434
- const toolDoneEvent = {
435
- type: 'tool_done',
436
- request_id: requestId,
437
- tool_call: toolCallWithFormattedArgs,
438
- result: {
439
- call_id: toolResult.call_id || toolResult.id,
440
- output: toolResult.output,
441
- error: toolResult.error,
442
- },
443
- };
444
- yield toolDoneEvent;
445
- await (0, event_controller_js_1.emitEvent)(toolDoneEvent, agent, model);
446
- if (!isTerminalTool) {
447
- const functionOutput = (0, message_converter_js_1.convertToFunctionCallOutput)(toolResult, model, 'completed');
448
- history.add(functionOutput);
449
- yield {
450
- type: 'response_output',
451
- message: functionOutput,
452
- request_id: requestId,
453
- };
805
+ catch (error) {
806
+ const streamFailure = (0, failure_detection_js_1.normalizeFailure)(error, {
807
+ reason: 'request_stream_failed',
808
+ ...getFailureRetryOverrides(agent),
809
+ });
810
+ roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, streamFailure);
811
+ roundSummary.errors.push(streamFailure.error);
812
+ }
813
+ if (!sawTerminalProviderOutcome && !roundSummary.failure) {
814
+ const emptyResponseFailure = (0, failure_detection_js_1.normalizeFailure)(new Error(`Provider ${provider.provider_id} ended the stream without any terminal content, tool calls, files, or errors.`), {
815
+ recoverable: false,
816
+ reason: 'empty_provider_response',
817
+ ...getFailureRetryOverrides(agent),
818
+ });
819
+ roundSummary.failure = emptyResponseFailure;
820
+ roundSummary.errors.push(emptyResponseFailure.error);
821
+ }
822
+ roundSummary.requestDuration = Date.now() - startTime;
823
+ const shouldUseBoundedFailureFinalization = Boolean(roundSummary.failure?.terminal);
824
+ yield* finalizeToolResults(shouldUseBoundedFailureFinalization ? 'bounded_failure' : 'wait_all');
825
+ if (requestGuard?.signal.aborted) {
826
+ const abortFailure = (0, failure_detection_js_1.normalizeFailure)(requestGuard.signal.reason, {
827
+ reason: 'request_stream_failed',
828
+ ...getFailureRetryOverrides(agent),
829
+ });
830
+ roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, abortFailure);
831
+ if (!roundSummary.errors.includes(abortFailure.error)) {
832
+ roundSummary.errors.push(abortFailure.error);
454
833
  }
455
834
  }
456
- const duration_with_tools = Date.now() - startTime;
457
- const agentDoneEvent = {
835
+ roundSummary.durationWithTools = Date.now() - startTime;
836
+ roundSummary.requestCost = totalCost > 0 ? totalCost : undefined;
837
+ roundSummary.agentDoneEvent = {
458
838
  type: 'agent_done',
459
- request_id: requestId,
460
- request_cost: totalCost > 0 ? totalCost : undefined,
461
- request_duration,
462
- duration_with_tools,
839
+ request_id: roundRequestId,
840
+ request_cost: roundSummary.requestCost,
841
+ request_duration: roundSummary.requestDuration,
842
+ duration_with_tools: roundSummary.durationWithTools,
463
843
  timestamp: new Date().toISOString(),
464
844
  };
465
- yield agentDoneEvent;
466
- await (0, event_controller_js_1.emitEvent)(agentDoneEvent, agent, model);
467
- for (const bufferedEvent of toolEventBuffer) {
468
- yield { ...bufferedEvent, request_id: requestId };
469
- }
845
+ roundSummary.agentDoneAgent = roundAgent;
846
+ requestGuard?.cleanup();
847
+ toolExecutionGuard?.cleanup();
848
+ return roundSummary;
470
849
  }
471
850
  async function* performVerification(agent, output, messages, attempt = 0) {
472
- if (!agent.verifier)
473
- return;
851
+ if (!agent.verifier) {
852
+ return { passed: true };
853
+ }
474
854
  const maxAttempts = agent.maxVerificationAttempts || 2;
475
855
  const verification = await (0, verification_js_1.verifyOutput)(agent.verifier, output, messages);
476
856
  if (verification.status === 'pass') {
@@ -478,7 +858,7 @@ async function* performVerification(agent, output, messages, attempt = 0) {
478
858
  type: 'message_delta',
479
859
  content: '\n\n✓ Output verified',
480
860
  };
481
- return;
861
+ return { passed: true };
482
862
  }
483
863
  if (attempt < maxAttempts - 1) {
484
864
  yield {
@@ -507,27 +887,37 @@ async function* performVerification(agent, output, messages, attempt = 0) {
507
887
  const retryStream = ensembleRequest(retryMessages, retryAgent);
508
888
  let retryOutput = '';
509
889
  for await (const event of retryStream) {
890
+ if (event.type === 'operation_status' || event.type === 'error') {
891
+ continue;
892
+ }
510
893
  yield event;
511
894
  if (event.type === 'message_complete' && 'content' in event) {
512
895
  retryOutput = event.content;
513
896
  }
514
897
  }
515
898
  if (retryOutput) {
516
- yield* performVerification(agent, retryOutput, messages, attempt + 1);
899
+ return yield* performVerification(agent, retryOutput, messages, attempt + 1);
517
900
  }
518
- }
519
- else {
520
- yield {
521
- type: 'message_delta',
522
- content: `\n\n❌ Verification failed after ${maxAttempts} attempts: ${verification.reason}`,
901
+ return {
902
+ passed: false,
903
+ error: 'Verification retry did not produce a final response.',
523
904
  };
524
905
  }
906
+ const failureMessage = `Verification failed after ${maxAttempts} attempts: ${verification.reason}`;
907
+ yield {
908
+ type: 'message_delta',
909
+ content: `\n\n❌ ${failureMessage}`,
910
+ };
911
+ return {
912
+ passed: false,
913
+ error: failureMessage,
914
+ };
525
915
  }
526
916
  async function processToolCall(toolCall, agent) {
527
- if (agent.onToolCall) {
528
- await agent.onToolCall(toolCall);
529
- }
530
917
  try {
918
+ if (agent.onToolCall) {
919
+ await agent.onToolCall(toolCall);
920
+ }
531
921
  if (!agent.tools) {
532
922
  throw new Error('No tools available for agent');
533
923
  }
@@ -535,7 +925,7 @@ async function processToolCall(toolCall, agent) {
535
925
  if (!tool || !('function' in tool)) {
536
926
  throw new Error(`Tool ${toolCall.function.name} not found`);
537
927
  }
538
- const rawResult = await (0, tool_execution_manager_js_1.handleToolCall)(toolCall, tool, agent);
928
+ const rawResult = await (0, tool_execution_manager_js_1.handleToolCall)(toolCall, tool, agent, agent.abortSignal);
539
929
  const processedResult = await (0, tool_result_processor_js_1.processToolResult)(toolCall, rawResult, agent, tool.allowSummary);
540
930
  const toolCallResult = {
541
931
  toolCall,
@@ -549,21 +939,32 @@ async function processToolCall(toolCall, agent) {
549
939
  return toolCallResult;
550
940
  }
551
941
  catch (error) {
552
- const errorOutput = error instanceof Error
553
- ? `Tool execution failed: ${error.message}`
554
- : `Tool execution failed: ${String(error)}`;
555
- const toolCallResult = {
556
- toolCall,
557
- id: toolCall.id,
558
- call_id: toolCall.call_id || toolCall.id,
559
- error: errorOutput,
560
- };
942
+ const toolCallResult = createToolFailureResult(toolCall, error);
561
943
  if (agent.onToolError) {
562
- await agent.onToolError(toolCallResult);
944
+ try {
945
+ await agent.onToolError(toolCallResult);
946
+ }
947
+ catch (hookError) {
948
+ console.error('[processToolCall] onToolError hook failed:', hookError);
949
+ }
563
950
  }
564
951
  return toolCallResult;
565
952
  }
566
953
  }
954
+ function createToolFailureResult(toolCall, error) {
955
+ const errorOutput = error instanceof Error
956
+ ? `Tool execution failed: ${error.message}`
957
+ : `Tool execution failed: ${String(error)}`;
958
+ return {
959
+ toolCall,
960
+ id: toolCall.id,
961
+ call_id: toolCall.call_id || toolCall.id,
962
+ error: errorOutput,
963
+ };
964
+ }
965
+ function createToolFinalizationFailureResult(toolCall) {
966
+ return createToolFailureResult(toolCall, 'Tool did not finish before request finalization after a terminal provider failure.');
967
+ }
567
968
  function mergeHistoryThread(mainHistory, thread, startIndex) {
568
969
  const newMessages = thread.slice(startIndex);
569
970
  mainHistory.push(...newMessages);