@just-every/ensemble 0.2.212 → 0.2.213
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -91
- package/dist/cjs/core/ensemble_request.cjs +734 -333
- package/dist/cjs/core/ensemble_request.d.ts.map +1 -1
- package/dist/cjs/core/ensemble_request.js.map +1 -1
- package/dist/cjs/model_providers/base_provider.d.ts.map +1 -1
- package/dist/cjs/model_providers/base_provider.js.map +1 -1
- package/dist/cjs/model_providers/claude.cjs +72 -72
- package/dist/cjs/model_providers/claude.d.ts.map +1 -1
- package/dist/cjs/model_providers/claude.js.map +1 -1
- package/dist/cjs/model_providers/gemini.cjs +3 -0
- package/dist/cjs/model_providers/gemini.d.ts.map +1 -1
- package/dist/cjs/model_providers/gemini.js.map +1 -1
- package/dist/cjs/model_providers/openai.cjs +41 -112
- package/dist/cjs/model_providers/openai.d.ts.map +1 -1
- package/dist/cjs/model_providers/openai.js.map +1 -1
- package/dist/cjs/model_providers/openai_chat.cjs +55 -24
- package/dist/cjs/model_providers/openai_chat.d.ts.map +1 -1
- package/dist/cjs/model_providers/openai_chat.js.map +1 -1
- package/dist/cjs/tsconfig.cjs.tsbuildinfo +1 -1
- package/dist/cjs/types/types.d.ts +20 -2
- package/dist/cjs/types/types.d.ts.map +1 -1
- package/dist/cjs/utils/agent.cjs +4 -6
- package/dist/cjs/utils/agent.d.ts.map +1 -1
- package/dist/cjs/utils/agent.js.map +1 -1
- package/dist/cjs/utils/ensemble_result.cjs +43 -4
- package/dist/cjs/utils/ensemble_result.d.ts +10 -1
- package/dist/cjs/utils/ensemble_result.d.ts.map +1 -1
- package/dist/cjs/utils/ensemble_result.js.map +1 -1
- package/dist/cjs/utils/failure_detection.cjs +292 -0
- package/dist/cjs/utils/failure_detection.d.ts +51 -0
- package/dist/cjs/utils/failure_detection.d.ts.map +1 -0
- package/dist/cjs/utils/failure_detection.js.map +1 -0
- package/dist/cjs/utils/json_schema.cjs +490 -0
- package/dist/cjs/utils/json_schema.d.ts +10 -0
- package/dist/cjs/utils/json_schema.d.ts.map +1 -0
- package/dist/cjs/utils/json_schema.js.map +1 -0
- package/dist/cjs/utils/tool_execution_manager.cjs +28 -4
- package/dist/cjs/utils/tool_execution_manager.d.ts +1 -1
- package/dist/cjs/utils/tool_execution_manager.d.ts.map +1 -1
- package/dist/cjs/utils/tool_execution_manager.js.map +1 -1
- package/dist/cjs/utils/verification.cjs +26 -13
- package/dist/cjs/utils/verification.d.ts.map +1 -1
- package/dist/cjs/utils/verification.js.map +1 -1
- package/dist/core/ensemble_request.d.ts.map +1 -1
- package/dist/core/ensemble_request.js +734 -333
- package/dist/core/ensemble_request.js.map +1 -1
- package/dist/model_providers/base_provider.d.ts.map +1 -1
- package/dist/model_providers/base_provider.js.map +1 -1
- package/dist/model_providers/claude.d.ts.map +1 -1
- package/dist/model_providers/claude.js +72 -72
- package/dist/model_providers/claude.js.map +1 -1
- package/dist/model_providers/gemini.d.ts.map +1 -1
- package/dist/model_providers/gemini.js +3 -0
- package/dist/model_providers/gemini.js.map +1 -1
- package/dist/model_providers/openai.d.ts.map +1 -1
- package/dist/model_providers/openai.js +41 -112
- package/dist/model_providers/openai.js.map +1 -1
- package/dist/model_providers/openai_chat.d.ts.map +1 -1
- package/dist/model_providers/openai_chat.js +55 -24
- package/dist/model_providers/openai_chat.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types/types.d.ts +20 -2
- package/dist/types/types.d.ts.map +1 -1
- package/dist/utils/agent.d.ts.map +1 -1
- package/dist/utils/agent.js +4 -6
- package/dist/utils/agent.js.map +1 -1
- package/dist/utils/ensemble_result.d.ts +10 -1
- package/dist/utils/ensemble_result.d.ts.map +1 -1
- package/dist/utils/ensemble_result.js +43 -4
- package/dist/utils/ensemble_result.js.map +1 -1
- package/dist/utils/failure_detection.d.ts +51 -0
- package/dist/utils/failure_detection.d.ts.map +1 -0
- package/dist/utils/failure_detection.js +280 -0
- package/dist/utils/failure_detection.js.map +1 -0
- package/dist/utils/json_schema.d.ts +10 -0
- package/dist/utils/json_schema.d.ts.map +1 -0
- package/dist/utils/json_schema.js +486 -0
- package/dist/utils/json_schema.js.map +1 -0
- package/dist/utils/tool_execution_manager.d.ts +1 -1
- package/dist/utils/tool_execution_manager.d.ts.map +1 -1
- package/dist/utils/tool_execution_manager.js +28 -4
- package/dist/utils/tool_execution_manager.js.map +1 -1
- package/dist/utils/verification.d.ts.map +1 -1
- package/dist/utils/verification.js +26 -13
- package/dist/utils/verification.js.map +1 -1
- package/package.json +1 -1
|
@@ -13,9 +13,13 @@ const pause_controller_js_1 = require("../utils/pause_controller.cjs");
|
|
|
13
13
|
const event_controller_js_1 = require("../utils/event_controller.cjs");
|
|
14
14
|
const trace_context_js_1 = require("../utils/trace_context.cjs");
|
|
15
15
|
const message_converter_js_1 = require("../utils/message_converter.cjs");
|
|
16
|
-
const
|
|
17
|
-
const
|
|
16
|
+
const failure_detection_js_1 = require("../utils/failure_detection.cjs");
|
|
17
|
+
const json_schema_js_1 = require("../utils/json_schema.cjs");
|
|
18
|
+
const running_tool_tracker_js_1 = require("../utils/running_tool_tracker.cjs");
|
|
19
|
+
const retry_handler_js_1 = require("../utils/retry_handler.cjs");
|
|
20
|
+
const DEFAULT_MAX_ERROR_RETRIES = 4;
|
|
18
21
|
const DEFAULT_TERMINAL_TOOL_NAMES = new Set(['task_complete', 'task_fatal_error']);
|
|
22
|
+
const TOOL_FAILURE_FINALIZATION_TIMEOUT_MS = 50;
|
|
19
23
|
const getTerminalToolNames = (agent) => {
|
|
20
24
|
const toolNames = new Set(DEFAULT_TERMINAL_TOOL_NAMES);
|
|
21
25
|
for (const name of agent.terminalToolNames ?? []) {
|
|
@@ -25,9 +29,81 @@ const getTerminalToolNames = (agent) => {
|
|
|
25
29
|
}
|
|
26
30
|
return toolNames;
|
|
27
31
|
};
|
|
32
|
+
const hasTerminalTextContent = (content, expectsStructuredOutput) => {
|
|
33
|
+
if (typeof content !== 'string') {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
return expectsStructuredOutput ? content.trim().length > 0 : content.length > 0;
|
|
37
|
+
};
|
|
38
|
+
const getMaxErrorRetries = (agent) => {
|
|
39
|
+
const configuredMaxRetries = agent.retryOptions?.maxRetries;
|
|
40
|
+
if (typeof configuredMaxRetries !== 'number' || Number.isNaN(configuredMaxRetries)) {
|
|
41
|
+
return DEFAULT_MAX_ERROR_RETRIES;
|
|
42
|
+
}
|
|
43
|
+
return Math.max(0, Math.floor(configuredMaxRetries));
|
|
44
|
+
};
|
|
45
|
+
const waitForRetryDelay = async (delayMs, abortSignal) => {
|
|
46
|
+
if (delayMs <= 0) {
|
|
47
|
+
return;
|
|
48
|
+
}
|
|
49
|
+
await new Promise((resolve, reject) => {
|
|
50
|
+
if (abortSignal?.aborted) {
|
|
51
|
+
reject(abortSignal.reason ?? new Error('Retry wait aborted'));
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const timeoutId = setTimeout(() => {
|
|
55
|
+
if (abortSignal && abortListener) {
|
|
56
|
+
abortSignal.removeEventListener('abort', abortListener);
|
|
57
|
+
}
|
|
58
|
+
resolve();
|
|
59
|
+
}, delayMs);
|
|
60
|
+
const abortListener = abortSignal
|
|
61
|
+
? () => {
|
|
62
|
+
clearTimeout(timeoutId);
|
|
63
|
+
abortSignal.removeEventListener('abort', abortListener);
|
|
64
|
+
reject(abortSignal.reason ?? new Error('Retry wait aborted'));
|
|
65
|
+
}
|
|
66
|
+
: undefined;
|
|
67
|
+
if (abortSignal && abortListener) {
|
|
68
|
+
abortSignal.addEventListener('abort', abortListener, { once: true });
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
};
|
|
72
|
+
const getOuterRequestTimeoutMs = (agent) => {
|
|
73
|
+
const timeoutMs = agent.modelSettings?.timeout_ms;
|
|
74
|
+
if (typeof timeoutMs !== 'number' || Number.isNaN(timeoutMs) || timeoutMs <= 0) {
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
77
|
+
return Math.floor(timeoutMs);
|
|
78
|
+
};
|
|
79
|
+
const getRemainingRequestTimeoutMs = (requestTimeoutMs, requestStartedAt) => {
|
|
80
|
+
if (requestTimeoutMs === undefined || requestStartedAt === undefined) {
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
return Math.max(0, requestTimeoutMs - (Date.now() - requestStartedAt));
|
|
84
|
+
};
|
|
85
|
+
const createRequestTimeoutError = (model, timeoutMs) => {
|
|
86
|
+
const error = new Error(`Request generation for ${model} timed out after ${timeoutMs}ms`);
|
|
87
|
+
error.code = 'ETIMEDOUT';
|
|
88
|
+
error.recoverable = false;
|
|
89
|
+
return error;
|
|
90
|
+
};
|
|
91
|
+
const getFailureRetryOverrides = (agent) => ({
|
|
92
|
+
retryableErrors: agent.retryOptions?.additionalRetryableErrors,
|
|
93
|
+
retryableStatusCodes: agent.retryOptions?.additionalRetryableStatusCodes,
|
|
94
|
+
});
|
|
28
95
|
(0, verification_js_1.setEnsembleRequestFunction)(ensembleRequest);
|
|
29
96
|
(0, image_to_text_js_1.setEnsembleRequestFunction)(ensembleRequest);
|
|
30
97
|
async function* ensembleRequest(messages, agent = {}) {
|
|
98
|
+
if (agent.jsonSchema && !agent.modelSettings?.json_schema) {
|
|
99
|
+
agent = {
|
|
100
|
+
...agent,
|
|
101
|
+
modelSettings: {
|
|
102
|
+
...agent.modelSettings,
|
|
103
|
+
json_schema: agent.jsonSchema,
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
}
|
|
31
107
|
const conversationHistory = agent?.historyThread || messages;
|
|
32
108
|
if (agent.instructions) {
|
|
33
109
|
const alreadyHasInstructions = conversationHistory.some(msg => {
|
|
@@ -59,187 +135,228 @@ async function* ensembleRequest(messages, agent = {}) {
|
|
|
59
135
|
compactionThreshold: 0.7,
|
|
60
136
|
});
|
|
61
137
|
const trace = (0, trace_context_js_1.createTraceContext)(agent, 'chat');
|
|
138
|
+
const lifecycle = new failure_detection_js_1.RequestLifecycleController();
|
|
139
|
+
const maxToolCalls = agent?.maxToolCalls ?? 200;
|
|
140
|
+
const maxRounds = agent?.maxToolCallRoundsPerTurn ?? Infinity;
|
|
141
|
+
const maxErrorRetries = getMaxErrorRetries(agent);
|
|
142
|
+
const maxErrorAttempts = maxErrorRetries + 1;
|
|
143
|
+
const outerRequestTimeoutMs = getOuterRequestTimeoutMs(agent);
|
|
144
|
+
const outerRequestStartedAt = outerRequestTimeoutMs !== undefined ? Date.now() : undefined;
|
|
145
|
+
const modelHistory = [];
|
|
146
|
+
let lastModelUsed;
|
|
62
147
|
let totalToolCalls = 0;
|
|
63
148
|
let toolCallRounds = 0;
|
|
64
149
|
let errorRounds = 0;
|
|
150
|
+
let lastMessageContent = '';
|
|
65
151
|
let turnStatus = 'completed';
|
|
66
152
|
let turnEndReason = 'completed';
|
|
67
153
|
let turnError;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
let
|
|
71
|
-
let hasError = false;
|
|
72
|
-
let lastMessageContent = '';
|
|
73
|
-
const modelHistory = [];
|
|
154
|
+
let terminalFailure;
|
|
155
|
+
let terminalFailureEventEmitted = false;
|
|
156
|
+
let finalRound;
|
|
74
157
|
await trace.emitTurnStart({
|
|
75
158
|
input_messages: conversationHistory,
|
|
76
159
|
});
|
|
77
160
|
try {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
let currentRoundRequestDuration;
|
|
87
|
-
let currentRoundDurationWithTools;
|
|
88
|
-
let currentRoundRequestCost;
|
|
89
|
-
const terminalToolNames = getTerminalToolNames(agent);
|
|
161
|
+
const emitRoundAgentDone = async function* (round, model) {
|
|
162
|
+
if (!round.agentDoneEvent) {
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
yield round.agentDoneEvent;
|
|
166
|
+
await (0, event_controller_js_1.emitEvent)(round.agentDoneEvent, round.agentDoneAgent ?? agent, model);
|
|
167
|
+
};
|
|
168
|
+
while (!terminalFailure) {
|
|
90
169
|
const model = await (0, model_provider_js_1.getModelFromAgent)(agent, 'reasoning_mini', modelHistory);
|
|
170
|
+
const roundRequestId = (0, crypto_1.randomUUID)();
|
|
171
|
+
const startedStatusEvent = lifecycle.begin(roundRequestId);
|
|
91
172
|
modelHistory.push(model);
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
tool_name: toolName,
|
|
116
|
-
arguments: toolEvent.tool_call.function.arguments,
|
|
117
|
-
arguments_formatted: toolEvent.tool_call.function.arguments_formatted,
|
|
118
|
-
});
|
|
119
|
-
if (!terminalToolNames.has(toolName)) {
|
|
120
|
-
hasToolCalls = true;
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
++totalToolCalls;
|
|
124
|
-
break;
|
|
125
|
-
}
|
|
126
|
-
case 'tool_done': {
|
|
127
|
-
const toolEvent = event;
|
|
128
|
-
if (toolEvent.tool_call) {
|
|
129
|
-
const toolName = toolEvent.tool_call.function.name;
|
|
130
|
-
if (terminalToolNames.has(toolName) && !toolEvent.result?.error) {
|
|
131
|
-
terminalToolSucceededThisRound = true;
|
|
132
|
-
}
|
|
133
|
-
await trace.emitToolDone(event.request_id, toolEvent.tool_call.id, {
|
|
134
|
-
tool_name: toolName,
|
|
135
|
-
call_id: toolEvent.result?.call_id,
|
|
136
|
-
output: toolEvent.result?.output,
|
|
137
|
-
error: toolEvent.result?.error,
|
|
138
|
-
});
|
|
139
|
-
}
|
|
140
|
-
break;
|
|
141
|
-
}
|
|
142
|
-
case 'agent_done': {
|
|
143
|
-
const agentDoneEvent = event;
|
|
144
|
-
currentRoundRequestDuration = agentDoneEvent.request_duration;
|
|
145
|
-
currentRoundDurationWithTools = agentDoneEvent.duration_with_tools;
|
|
146
|
-
currentRoundRequestCost = agentDoneEvent.request_cost;
|
|
147
|
-
break;
|
|
148
|
-
}
|
|
149
|
-
case 'error': {
|
|
150
|
-
hasError = true;
|
|
151
|
-
const errorEvent = event;
|
|
152
|
-
if (errorEvent.error) {
|
|
153
|
-
currentRoundErrors.push(String(errorEvent.error));
|
|
154
|
-
}
|
|
155
|
-
break;
|
|
156
|
-
}
|
|
157
|
-
}
|
|
173
|
+
lastModelUsed = model;
|
|
174
|
+
const round = yield* executeRound({
|
|
175
|
+
roundRequestId,
|
|
176
|
+
model,
|
|
177
|
+
agent,
|
|
178
|
+
history,
|
|
179
|
+
currentToolCalls: totalToolCalls,
|
|
180
|
+
maxToolCalls,
|
|
181
|
+
trace,
|
|
182
|
+
startedStatusEvent,
|
|
183
|
+
requestTimeoutMs: outerRequestTimeoutMs,
|
|
184
|
+
requestStartedAt: outerRequestStartedAt,
|
|
185
|
+
});
|
|
186
|
+
totalToolCalls += round.toolCallsStarted;
|
|
187
|
+
if (round.messages.length > 0) {
|
|
188
|
+
lastMessageContent = round.messages.at(-1) || lastMessageContent;
|
|
189
|
+
}
|
|
190
|
+
if (round.hasFollowupToolCalls) {
|
|
191
|
+
++toolCallRounds;
|
|
192
|
+
}
|
|
193
|
+
const willRetryForError = (() => {
|
|
194
|
+
if (!round.failure) {
|
|
195
|
+
return false;
|
|
158
196
|
}
|
|
197
|
+
++errorRounds;
|
|
198
|
+
return !round.emittedTerminalOutput && round.failure.recoverable && errorRounds <= maxErrorRetries;
|
|
199
|
+
})();
|
|
200
|
+
const willContinueForTools = !round.failure &&
|
|
201
|
+
!round.terminalToolSucceeded &&
|
|
202
|
+
round.hasFollowupToolCalls &&
|
|
203
|
+
toolCallRounds < maxRounds &&
|
|
204
|
+
totalToolCalls < maxToolCalls;
|
|
205
|
+
let requestStatus = 'completed';
|
|
206
|
+
if (round.failure) {
|
|
207
|
+
requestStatus = willRetryForError ? 'error_retrying' : 'error';
|
|
159
208
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
209
|
+
else if (round.hasFollowupToolCalls && !round.terminalToolSucceeded) {
|
|
210
|
+
requestStatus = willContinueForTools ? 'waiting_for_followup_request' : 'tool_limit_reached';
|
|
211
|
+
}
|
|
212
|
+
await trace.emitRequestEnd(round.requestId, {
|
|
213
|
+
status: requestStatus,
|
|
214
|
+
will_continue: willRetryForError || willContinueForTools,
|
|
215
|
+
tool_calls: round.toolCallsStarted,
|
|
216
|
+
final_response: round.messages.length > 0 ? round.messages.join('\n') : undefined,
|
|
217
|
+
errors: round.errors.length > 0 ? round.errors : undefined,
|
|
218
|
+
request_duration_ms: round.requestDuration,
|
|
219
|
+
duration_with_tools_ms: round.durationWithTools,
|
|
220
|
+
request_cost: round.requestCost,
|
|
221
|
+
});
|
|
222
|
+
if (round.failure) {
|
|
223
|
+
const terminalRoundFailure = willRetryForError
|
|
224
|
+
? round.failure
|
|
225
|
+
: {
|
|
226
|
+
...round.failure,
|
|
227
|
+
recoverable: false,
|
|
228
|
+
terminal: true,
|
|
229
|
+
};
|
|
230
|
+
const errorEvent = (0, failure_detection_js_1.toErrorEvent)(terminalRoundFailure, {
|
|
231
|
+
request_id: round.requestId,
|
|
232
|
+
});
|
|
233
|
+
yield errorEvent;
|
|
234
|
+
await (0, event_controller_js_1.emitEvent)(errorEvent, agent, model);
|
|
235
|
+
if (willRetryForError) {
|
|
236
|
+
agent.retryOptions?.onRetry?.({
|
|
237
|
+
message: round.failure.error,
|
|
238
|
+
code: round.failure.code,
|
|
239
|
+
details: round.failure.details,
|
|
240
|
+
recoverable: round.failure.recoverable,
|
|
241
|
+
}, errorRounds);
|
|
242
|
+
const retryingEvent = lifecycle.retrying(round.failure, errorRounds, maxErrorAttempts);
|
|
243
|
+
if (retryingEvent) {
|
|
244
|
+
yield retryingEvent;
|
|
245
|
+
await (0, event_controller_js_1.emitEvent)(retryingEvent, agent, model);
|
|
246
|
+
}
|
|
247
|
+
const retryDelayMs = (0, retry_handler_js_1.calculateDelay)(errorRounds, agent.retryOptions);
|
|
248
|
+
const remainingTimeoutMs = getRemainingRequestTimeoutMs(outerRequestTimeoutMs, outerRequestStartedAt);
|
|
249
|
+
const boundedRetryDelayMs = remainingTimeoutMs === undefined
|
|
250
|
+
? retryDelayMs
|
|
251
|
+
: remainingTimeoutMs < retryDelayMs
|
|
252
|
+
? 0
|
|
253
|
+
: retryDelayMs;
|
|
254
|
+
yield* emitRoundAgentDone(round, model);
|
|
255
|
+
await waitForRetryDelay(boundedRetryDelayMs, agent.abortSignal);
|
|
256
|
+
continue;
|
|
257
|
+
}
|
|
258
|
+
terminalFailure = terminalRoundFailure;
|
|
259
|
+
terminalFailureEventEmitted = true;
|
|
260
|
+
finalRound = { round, model };
|
|
261
|
+
break;
|
|
171
262
|
}
|
|
172
|
-
if (
|
|
173
|
-
|
|
174
|
-
|
|
263
|
+
if (round.terminalToolSucceeded) {
|
|
264
|
+
finalRound = { round, model };
|
|
265
|
+
break;
|
|
175
266
|
}
|
|
176
|
-
if (
|
|
177
|
-
++toolCallRounds;
|
|
267
|
+
if (willContinueForTools) {
|
|
178
268
|
if (agent.modelSettings?.tool_choice) {
|
|
269
|
+
agent = {
|
|
270
|
+
...agent,
|
|
271
|
+
modelSettings: {
|
|
272
|
+
...agent.modelSettings,
|
|
273
|
+
},
|
|
274
|
+
};
|
|
179
275
|
delete agent.modelSettings.tool_choice;
|
|
180
276
|
}
|
|
277
|
+
yield* emitRoundAgentDone(round, model);
|
|
278
|
+
continue;
|
|
181
279
|
}
|
|
182
|
-
if (
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
}
|
|
192
|
-
else if (hasToolCalls) {
|
|
193
|
-
requestStatus = willContinue ? 'waiting_for_followup_request' : 'tool_limit_reached';
|
|
194
|
-
}
|
|
195
|
-
if (currentRoundRequestId) {
|
|
196
|
-
await trace.emitRequestEnd(currentRoundRequestId, {
|
|
197
|
-
status: requestStatus,
|
|
198
|
-
will_continue: willContinue,
|
|
199
|
-
tool_calls: currentRoundToolCalls,
|
|
200
|
-
final_response: currentRoundMessages.length > 0 ? currentRoundMessages.join('\n') : undefined,
|
|
201
|
-
errors: currentRoundErrors.length > 0 ? currentRoundErrors : undefined,
|
|
202
|
-
request_duration_ms: currentRoundRequestDuration,
|
|
203
|
-
duration_with_tools_ms: currentRoundDurationWithTools,
|
|
204
|
-
request_cost: currentRoundRequestCost,
|
|
280
|
+
if (round.hasFollowupToolCalls && !round.terminalToolSucceeded) {
|
|
281
|
+
terminalFailure = (0, failure_detection_js_1.normalizeFailure)(new Error(toolCallRounds >= maxRounds
|
|
282
|
+
? `Tool call rounds limit reached (${maxRounds}).`
|
|
283
|
+
: `Tool call limit reached (${maxToolCalls}).`), {
|
|
284
|
+
recoverable: false,
|
|
285
|
+
reason: toolCallRounds >= maxRounds
|
|
286
|
+
? 'max_tool_call_rounds_reached'
|
|
287
|
+
: 'max_tool_calls_reached',
|
|
288
|
+
...getFailureRetryOverrides(agent),
|
|
205
289
|
});
|
|
290
|
+
finalRound = { round, model };
|
|
291
|
+
break;
|
|
206
292
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
if (hasToolCalls && toolCallRounds >= maxRounds) {
|
|
210
|
-
console.log('[ensembleRequest] Tool call rounds limit reached');
|
|
211
|
-
turnEndReason = 'max_tool_call_rounds_reached';
|
|
293
|
+
finalRound = { round, model };
|
|
294
|
+
break;
|
|
212
295
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
296
|
+
if (!terminalFailure && agent.verifier && lastMessageContent) {
|
|
297
|
+
const verification = yield* performVerification(agent, lastMessageContent, await history.getMessages());
|
|
298
|
+
if (!verification.passed) {
|
|
299
|
+
terminalFailure = (0, failure_detection_js_1.normalizeFailure)(new Error(verification.error || 'Verification failed'), {
|
|
300
|
+
recoverable: false,
|
|
301
|
+
reason: 'verification_failed',
|
|
302
|
+
...getFailureRetryOverrides(agent),
|
|
303
|
+
});
|
|
304
|
+
}
|
|
216
305
|
}
|
|
217
|
-
|
|
306
|
+
if (terminalFailure) {
|
|
218
307
|
turnStatus = 'error';
|
|
219
|
-
turnEndReason = '
|
|
308
|
+
turnEndReason = terminalFailure.reason || 'terminal_failure';
|
|
309
|
+
turnError = terminalFailure.error;
|
|
310
|
+
if (!terminalFailureEventEmitted) {
|
|
311
|
+
const errorEvent = (0, failure_detection_js_1.toErrorEvent)(terminalFailure, {
|
|
312
|
+
request_id: lifecycle.getRequestId(),
|
|
313
|
+
});
|
|
314
|
+
yield errorEvent;
|
|
315
|
+
await (0, event_controller_js_1.emitEvent)(errorEvent, agent, lastModelUsed);
|
|
316
|
+
}
|
|
317
|
+
const failedEvent = lifecycle.fail(terminalFailure, errorRounds || 1, maxErrorAttempts);
|
|
318
|
+
if (failedEvent) {
|
|
319
|
+
yield failedEvent;
|
|
320
|
+
await (0, event_controller_js_1.emitEvent)(failedEvent, agent, lastModelUsed);
|
|
321
|
+
}
|
|
220
322
|
}
|
|
221
|
-
|
|
222
|
-
const
|
|
223
|
-
if (
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
}
|
|
323
|
+
else {
|
|
324
|
+
const completedEvent = lifecycle.complete();
|
|
325
|
+
if (completedEvent) {
|
|
326
|
+
yield completedEvent;
|
|
327
|
+
await (0, event_controller_js_1.emitEvent)(completedEvent, agent, lastModelUsed);
|
|
227
328
|
}
|
|
228
329
|
}
|
|
330
|
+
if (finalRound) {
|
|
331
|
+
yield* emitRoundAgentDone(finalRound.round, finalRound.model);
|
|
332
|
+
}
|
|
229
333
|
}
|
|
230
334
|
catch (err) {
|
|
231
|
-
|
|
335
|
+
if (!lifecycle.getRequestId()) {
|
|
336
|
+
const startedEvent = lifecycle.begin((0, crypto_1.randomUUID)());
|
|
337
|
+
if (startedEvent) {
|
|
338
|
+
yield startedEvent;
|
|
339
|
+
await (0, event_controller_js_1.emitEvent)(startedEvent, agent, lastModelUsed);
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
const failure = (0, failure_detection_js_1.normalizeFailure)(err, {
|
|
343
|
+
recoverable: false,
|
|
344
|
+
reason: 'exception',
|
|
345
|
+
...getFailureRetryOverrides(agent),
|
|
346
|
+
});
|
|
232
347
|
turnStatus = 'error';
|
|
233
348
|
turnEndReason = 'exception';
|
|
234
|
-
turnError =
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
349
|
+
turnError = failure.error;
|
|
350
|
+
const errorEvent = (0, failure_detection_js_1.toErrorEvent)(failure, {
|
|
351
|
+
request_id: lifecycle.getRequestId(),
|
|
352
|
+
});
|
|
353
|
+
yield errorEvent;
|
|
354
|
+
await (0, event_controller_js_1.emitEvent)(errorEvent, agent, lastModelUsed);
|
|
355
|
+
const failedEvent = lifecycle.fail(failure, errorRounds || 1, maxErrorAttempts);
|
|
356
|
+
if (failedEvent) {
|
|
357
|
+
yield failedEvent;
|
|
358
|
+
await (0, event_controller_js_1.emitEvent)(failedEvent, agent, lastModelUsed);
|
|
359
|
+
}
|
|
243
360
|
}
|
|
244
361
|
finally {
|
|
245
362
|
await trace.emitTurnEnd(turnStatus, turnEndReason, {
|
|
@@ -254,14 +371,29 @@ async function* ensembleRequest(messages, agent = {}) {
|
|
|
254
371
|
};
|
|
255
372
|
}
|
|
256
373
|
}
|
|
257
|
-
async function* executeRound(
|
|
258
|
-
const
|
|
374
|
+
async function* executeRound(options) {
|
|
375
|
+
const { roundRequestId, model, agent, history, currentToolCalls, maxToolCalls, trace, startedStatusEvent } = options;
|
|
259
376
|
const startTime = Date.now();
|
|
260
377
|
let totalCost = 0;
|
|
261
378
|
let messages = await history.getMessages(model);
|
|
379
|
+
let roundAgentDefinition = agent;
|
|
380
|
+
let requestGuard;
|
|
381
|
+
let toolExecutionGuard;
|
|
382
|
+
let roundAgent = agent;
|
|
383
|
+
let provider;
|
|
384
|
+
let stream;
|
|
385
|
+
const roundSummary = {
|
|
386
|
+
requestId: roundRequestId,
|
|
387
|
+
messages: [],
|
|
388
|
+
errors: [],
|
|
389
|
+
toolCallsStarted: 0,
|
|
390
|
+
hasFollowupToolCalls: false,
|
|
391
|
+
emittedTerminalOutput: false,
|
|
392
|
+
terminalToolSucceeded: false,
|
|
393
|
+
};
|
|
262
394
|
const agentStartEvent = {
|
|
263
395
|
type: 'agent_start',
|
|
264
|
-
request_id:
|
|
396
|
+
request_id: roundRequestId,
|
|
265
397
|
input: 'content' in messages[0] && typeof messages[0].content === 'string' ? messages[0].content : undefined,
|
|
266
398
|
timestamp: new Date().toISOString(),
|
|
267
399
|
agent: {
|
|
@@ -278,199 +410,447 @@ async function* executeRound(model, agent, history, currentToolCalls, maxToolCal
|
|
|
278
410
|
};
|
|
279
411
|
yield agentStartEvent;
|
|
280
412
|
await (0, event_controller_js_1.emitEvent)(agentStartEvent, agent, model);
|
|
281
|
-
|
|
282
|
-
|
|
413
|
+
try {
|
|
414
|
+
if (roundAgentDefinition.onRequest) {
|
|
415
|
+
const [nextAgent, nextMessages] = await roundAgentDefinition.onRequest(roundAgentDefinition, messages);
|
|
416
|
+
roundAgentDefinition = nextAgent;
|
|
417
|
+
messages = nextMessages;
|
|
418
|
+
}
|
|
419
|
+
const remainingTimeoutMs = getRemainingRequestTimeoutMs(options.requestTimeoutMs, options.requestStartedAt);
|
|
420
|
+
const needsRequestGuard = Boolean(roundAgentDefinition.abortSignal || remainingTimeoutMs !== undefined);
|
|
421
|
+
if (needsRequestGuard) {
|
|
422
|
+
if (options.requestTimeoutMs !== undefined && remainingTimeoutMs !== undefined && remainingTimeoutMs <= 0) {
|
|
423
|
+
throw createRequestTimeoutError(model, options.requestTimeoutMs);
|
|
424
|
+
}
|
|
425
|
+
requestGuard = (0, failure_detection_js_1.createOperationGuard)({
|
|
426
|
+
operationName: `Request generation for ${model}`,
|
|
427
|
+
abortSignal: roundAgentDefinition.abortSignal,
|
|
428
|
+
timeoutMs: remainingTimeoutMs,
|
|
429
|
+
});
|
|
430
|
+
roundAgent = {
|
|
431
|
+
...roundAgentDefinition,
|
|
432
|
+
abortSignal: requestGuard.signal,
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
else {
|
|
436
|
+
roundAgent = roundAgentDefinition;
|
|
437
|
+
}
|
|
438
|
+
await (0, pause_controller_js_1.waitWhilePaused)(100, roundAgent.abortSignal);
|
|
439
|
+
toolExecutionGuard = (0, failure_detection_js_1.createOperationGuard)({
|
|
440
|
+
operationName: `Tool execution for ${model}`,
|
|
441
|
+
abortSignal: roundAgent.abortSignal,
|
|
442
|
+
});
|
|
443
|
+
if (startedStatusEvent) {
|
|
444
|
+
yield startedStatusEvent;
|
|
445
|
+
await (0, event_controller_js_1.emitEvent)(startedStatusEvent, roundAgent, model);
|
|
446
|
+
}
|
|
447
|
+
provider = (0, model_provider_js_1.getModelProvider)(model);
|
|
448
|
+
await trace.emitRequestStart(roundRequestId, {
|
|
449
|
+
agent_id: roundAgent.agent_id,
|
|
450
|
+
provider: provider.provider_id,
|
|
451
|
+
model,
|
|
452
|
+
payload: {
|
|
453
|
+
messages,
|
|
454
|
+
model_settings: roundAgent.modelSettings,
|
|
455
|
+
tool_names: roundAgent.tools?.map(tool => tool.definition.function.name) || [],
|
|
456
|
+
},
|
|
457
|
+
});
|
|
458
|
+
const rawStream = provider.createResponseStream(messages, model, roundAgent, roundRequestId);
|
|
459
|
+
stream = (0, failure_detection_js_1.streamWithAbortAndTimeout)(rawStream, {
|
|
460
|
+
abortSignal: requestGuard?.signal,
|
|
461
|
+
});
|
|
283
462
|
}
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
463
|
+
catch (error) {
|
|
464
|
+
requestGuard?.cleanup();
|
|
465
|
+
toolExecutionGuard?.cleanup();
|
|
466
|
+
const failure = (0, failure_detection_js_1.normalizeFailure)(error, {
|
|
467
|
+
reason: 'request_setup_failed',
|
|
468
|
+
...getFailureRetryOverrides(agent),
|
|
469
|
+
});
|
|
470
|
+
roundSummary.failure = failure;
|
|
471
|
+
roundSummary.errors.push(failure.error);
|
|
472
|
+
roundSummary.requestDuration = Date.now() - startTime;
|
|
473
|
+
roundSummary.durationWithTools = roundSummary.requestDuration;
|
|
474
|
+
roundSummary.agentDoneEvent = {
|
|
475
|
+
type: 'agent_done',
|
|
476
|
+
request_id: roundRequestId,
|
|
477
|
+
request_duration: roundSummary.requestDuration,
|
|
478
|
+
duration_with_tools: roundSummary.durationWithTools,
|
|
479
|
+
timestamp: new Date().toISOString(),
|
|
480
|
+
};
|
|
481
|
+
roundSummary.agentDoneAgent = roundAgentDefinition;
|
|
482
|
+
return roundSummary;
|
|
483
|
+
}
|
|
484
|
+
const terminalToolNames = getTerminalToolNames(roundAgent);
|
|
485
|
+
const expectsStructuredOutput = Boolean(roundAgent.modelSettings?.json_schema?.schema);
|
|
486
|
+
const structuredOutputSchema = roundAgent.modelSettings?.json_schema?.strict === true
|
|
487
|
+
? roundAgent.modelSettings.json_schema.schema
|
|
488
|
+
: undefined;
|
|
489
|
+
const toolExecutions = [];
|
|
300
490
|
const toolCallFormattedArgs = new Map();
|
|
301
491
|
const toolEventBuffer = [];
|
|
302
492
|
let sawToolCallThisRound = false;
|
|
303
|
-
|
|
493
|
+
let sawTerminalProviderOutcome = false;
|
|
494
|
+
roundAgent.onToolEvent = async (event) => {
|
|
304
495
|
toolEventBuffer.push(event);
|
|
305
496
|
};
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
497
|
+
const finalizeToolResults = async function* (mode) {
|
|
498
|
+
const waitForPendingExecutions = async (executions, timeoutMs) => {
|
|
499
|
+
if (executions.length === 0) {
|
|
500
|
+
return;
|
|
501
|
+
}
|
|
502
|
+
const completionPromise = Promise.all(executions.map(execution => execution.promise.then(() => undefined)));
|
|
503
|
+
if (timeoutMs === undefined) {
|
|
504
|
+
await completionPromise;
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
507
|
+
await Promise.race([
|
|
508
|
+
completionPromise,
|
|
509
|
+
new Promise(resolve => setTimeout(resolve, timeoutMs)),
|
|
510
|
+
]);
|
|
511
|
+
};
|
|
512
|
+
const waitForAllExecutions = async (executions, abortSignal) => {
|
|
513
|
+
if (executions.length === 0) {
|
|
514
|
+
return true;
|
|
515
|
+
}
|
|
516
|
+
const completionPromise = Promise.all(executions.map(execution => execution.promise.then(() => undefined))).then(() => true);
|
|
517
|
+
if (!abortSignal) {
|
|
518
|
+
return completionPromise;
|
|
519
|
+
}
|
|
520
|
+
if (abortSignal.aborted) {
|
|
521
|
+
return false;
|
|
522
|
+
}
|
|
523
|
+
return new Promise(resolve => {
|
|
524
|
+
const abortListener = () => {
|
|
525
|
+
abortSignal.removeEventListener('abort', abortListener);
|
|
526
|
+
resolve(false);
|
|
527
|
+
};
|
|
528
|
+
completionPromise.then(completed => {
|
|
529
|
+
abortSignal.removeEventListener('abort', abortListener);
|
|
530
|
+
resolve(completed);
|
|
531
|
+
});
|
|
532
|
+
abortSignal.addEventListener('abort', abortListener, { once: true });
|
|
533
|
+
});
|
|
534
|
+
};
|
|
535
|
+
let finalizationMode = mode;
|
|
536
|
+
if (finalizationMode === 'wait_all') {
|
|
537
|
+
const completedAllExecutions = await waitForAllExecutions(toolExecutions.filter(execution => !execution.settled), requestGuard?.signal);
|
|
538
|
+
if (!completedAllExecutions) {
|
|
539
|
+
finalizationMode = 'bounded_failure';
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
if (finalizationMode === 'bounded_failure') {
|
|
543
|
+
toolExecutionGuard?.abort(roundSummary.failure?.error
|
|
544
|
+
? new Error(roundSummary.failure.error)
|
|
545
|
+
: new Error('Request finalized after terminal provider failure.'));
|
|
546
|
+
await waitForPendingExecutions(toolExecutions.filter(execution => !execution.settled), TOOL_FAILURE_FINALIZATION_TIMEOUT_MS);
|
|
547
|
+
for (const execution of toolExecutions) {
|
|
548
|
+
if (!execution.settled) {
|
|
549
|
+
running_tool_tracker_js_1.runningToolTracker.abortRunningTool(execution.toolCall.id || execution.toolCall.call_id || '');
|
|
331
550
|
}
|
|
332
|
-
|
|
333
|
-
|
|
551
|
+
}
|
|
552
|
+
await waitForPendingExecutions(toolExecutions.filter(execution => !execution.settled), TOOL_FAILURE_FINALIZATION_TIMEOUT_MS);
|
|
553
|
+
for (const execution of toolExecutions) {
|
|
554
|
+
const runningToolId = execution.toolCall.id || execution.toolCall.call_id || '';
|
|
555
|
+
if (execution.settled) {
|
|
556
|
+
const leakedRunningTool = runningToolId
|
|
557
|
+
? running_tool_tracker_js_1.runningToolTracker.getRunningTool(runningToolId)
|
|
558
|
+
: undefined;
|
|
559
|
+
if (leakedRunningTool) {
|
|
560
|
+
const failureResult = execution.result ?? createToolFinalizationFailureResult(execution.toolCall);
|
|
561
|
+
await running_tool_tracker_js_1.runningToolTracker.failRunningTool(runningToolId, failureResult.error || 'Tool execution failed during bounded finalization.');
|
|
562
|
+
}
|
|
334
563
|
}
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
const toolResults = finalizationMode === 'wait_all'
|
|
567
|
+
? await Promise.all(toolExecutions.map(execution => execution.promise))
|
|
568
|
+
: toolExecutions.flatMap(execution => (execution.settled && execution.result ? [execution.result] : []));
|
|
569
|
+
for (const toolResult of toolResults) {
|
|
570
|
+
const toolName = toolResult.toolCall.function.name;
|
|
571
|
+
const isTerminalTool = terminalToolNames.has(toolName);
|
|
572
|
+
const formattedArgs = toolCallFormattedArgs.get(toolResult.toolCall.id);
|
|
573
|
+
const toolCallWithFormattedArgs = formattedArgs
|
|
574
|
+
? {
|
|
575
|
+
...toolResult.toolCall,
|
|
576
|
+
function: {
|
|
577
|
+
...toolResult.toolCall.function,
|
|
578
|
+
arguments_formatted: formattedArgs,
|
|
343
579
|
},
|
|
580
|
+
}
|
|
581
|
+
: toolResult.toolCall;
|
|
582
|
+
const toolDoneEvent = {
|
|
583
|
+
type: 'tool_done',
|
|
584
|
+
request_id: roundRequestId,
|
|
585
|
+
tool_call: toolCallWithFormattedArgs,
|
|
586
|
+
result: {
|
|
587
|
+
call_id: toolResult.call_id || toolResult.id,
|
|
588
|
+
output: toolResult.output,
|
|
589
|
+
error: toolResult.error,
|
|
590
|
+
},
|
|
591
|
+
};
|
|
592
|
+
if (isTerminalTool && !toolResult.error) {
|
|
593
|
+
roundSummary.terminalToolSucceeded = true;
|
|
594
|
+
}
|
|
595
|
+
yield toolDoneEvent;
|
|
596
|
+
await (0, event_controller_js_1.emitEvent)(toolDoneEvent, roundAgent, model);
|
|
597
|
+
await trace.emitToolDone(roundRequestId, toolResult.toolCall.id, {
|
|
598
|
+
tool_name: toolName,
|
|
599
|
+
call_id: toolResult.call_id,
|
|
600
|
+
output: toolResult.output,
|
|
601
|
+
error: toolResult.error,
|
|
602
|
+
});
|
|
603
|
+
if (!isTerminalTool) {
|
|
604
|
+
const functionOutput = (0, message_converter_js_1.convertToFunctionCallOutput)(toolResult, model, 'completed');
|
|
605
|
+
history.add(functionOutput);
|
|
606
|
+
yield {
|
|
607
|
+
type: 'response_output',
|
|
608
|
+
message: functionOutput,
|
|
609
|
+
request_id: roundRequestId,
|
|
344
610
|
};
|
|
345
|
-
event = modifiedEvent;
|
|
346
611
|
}
|
|
347
612
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
613
|
+
for (const bufferedEvent of toolEventBuffer) {
|
|
614
|
+
yield { ...bufferedEvent, request_id: roundRequestId };
|
|
615
|
+
}
|
|
616
|
+
};
|
|
617
|
+
try {
|
|
618
|
+
for await (let event of stream) {
|
|
619
|
+
event = { ...event, request_id: roundRequestId };
|
|
620
|
+
if (event.type === 'error') {
|
|
621
|
+
const failure = (0, failure_detection_js_1.normalizeFailure)(event, {
|
|
622
|
+
error: event.error,
|
|
623
|
+
recoverable: event.recoverable,
|
|
624
|
+
code: event.code,
|
|
625
|
+
details: event.details,
|
|
626
|
+
...getFailureRetryOverrides(agent),
|
|
627
|
+
});
|
|
628
|
+
roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, failure);
|
|
629
|
+
roundSummary.errors.push(failure.error);
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
if (event.type === 'message_complete' && structuredOutputSchema) {
|
|
633
|
+
const messageEvent = event;
|
|
634
|
+
if (hasTerminalTextContent(messageEvent.content, true)) {
|
|
635
|
+
const validationResult = (0, json_schema_js_1.validateJsonResponseContent)(messageEvent.content, structuredOutputSchema);
|
|
636
|
+
if (!validationResult.ok && 'error' in validationResult) {
|
|
637
|
+
const failure = (0, failure_detection_js_1.normalizeFailure)(new Error(validationResult.error), {
|
|
638
|
+
recoverable: false,
|
|
639
|
+
reason: 'structured_output_validation_failed',
|
|
640
|
+
});
|
|
641
|
+
roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, failure);
|
|
642
|
+
roundSummary.errors.push(failure.error);
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
355
645
|
}
|
|
356
|
-
break;
|
|
357
646
|
}
|
|
358
|
-
|
|
647
|
+
if (event.type === 'tool_start') {
|
|
648
|
+
const toolEvent = event;
|
|
649
|
+
if (toolEvent.tool_call) {
|
|
650
|
+
const toolCall = toolEvent.tool_call;
|
|
651
|
+
let argumentsFormatted;
|
|
652
|
+
try {
|
|
653
|
+
const tool = roundAgent.tools?.find(t => t.definition.function.name === toolCall.function.name);
|
|
654
|
+
if (tool?.definition.function.parameters.properties) {
|
|
655
|
+
const parsedArgs = JSON.parse(toolCall.function.arguments || '{}');
|
|
656
|
+
if (typeof parsedArgs === 'object' && parsedArgs !== null && !Array.isArray(parsedArgs)) {
|
|
657
|
+
const paramNames = Object.keys(tool.definition.function.parameters.properties);
|
|
658
|
+
const orderedArgs = {};
|
|
659
|
+
for (const param of paramNames) {
|
|
660
|
+
if (param in parsedArgs) {
|
|
661
|
+
orderedArgs[param] = parsedArgs[param];
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
argumentsFormatted = JSON.stringify(orderedArgs, null, 2);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
catch (error) {
|
|
669
|
+
console.debug('Failed to format tool arguments:', error);
|
|
670
|
+
}
|
|
671
|
+
if (argumentsFormatted) {
|
|
672
|
+
toolCallFormattedArgs.set(toolCall.id, argumentsFormatted);
|
|
673
|
+
}
|
|
674
|
+
event = {
|
|
675
|
+
...event,
|
|
676
|
+
tool_call: {
|
|
677
|
+
...toolCall,
|
|
678
|
+
function: {
|
|
679
|
+
...toolCall.function,
|
|
680
|
+
arguments_formatted: argumentsFormatted,
|
|
681
|
+
},
|
|
682
|
+
},
|
|
683
|
+
};
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
if (event.type === 'message_complete') {
|
|
359
687
|
const messageEvent = event;
|
|
360
|
-
if (
|
|
688
|
+
if (hasTerminalTextContent(messageEvent.content, expectsStructuredOutput)) {
|
|
689
|
+
sawTerminalProviderOutcome = true;
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
else if (event.type === 'tool_start' || event.type === 'file_complete') {
|
|
693
|
+
sawTerminalProviderOutcome = true;
|
|
694
|
+
}
|
|
695
|
+
yield event;
|
|
696
|
+
await (0, event_controller_js_1.emitEvent)(event, roundAgent, model);
|
|
697
|
+
switch (event.type) {
|
|
698
|
+
case 'cost_update': {
|
|
699
|
+
const costEvent = event;
|
|
700
|
+
if (costEvent.usage?.cost) {
|
|
701
|
+
totalCost += costEvent.usage.cost;
|
|
702
|
+
}
|
|
361
703
|
break;
|
|
362
704
|
}
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
await agent.onThinking(thinkingMessage);
|
|
705
|
+
case 'message_complete': {
|
|
706
|
+
const messageEvent = event;
|
|
707
|
+
if (sawToolCallThisRound) {
|
|
708
|
+
break;
|
|
368
709
|
}
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
710
|
+
if (messageEvent.thinking_content ||
|
|
711
|
+
(!messageEvent.content && messageEvent.message_id)) {
|
|
712
|
+
const thinkingMessage = (0, message_converter_js_1.convertToThinkingMessage)(messageEvent, model);
|
|
713
|
+
if (roundAgent.onThinking) {
|
|
714
|
+
await roundAgent.onThinking(thinkingMessage);
|
|
715
|
+
}
|
|
716
|
+
history.add(thinkingMessage);
|
|
717
|
+
yield {
|
|
718
|
+
type: 'response_output',
|
|
719
|
+
message: thinkingMessage,
|
|
720
|
+
request_id: roundRequestId,
|
|
721
|
+
};
|
|
722
|
+
}
|
|
723
|
+
if (hasTerminalTextContent(messageEvent.content, expectsStructuredOutput)) {
|
|
724
|
+
roundSummary.emittedTerminalOutput = true;
|
|
725
|
+
roundSummary.messages.push(messageEvent.content);
|
|
726
|
+
const contentMessage = (0, message_converter_js_1.convertToOutputMessage)(messageEvent, model, 'completed');
|
|
727
|
+
if (roundAgent.onResponse) {
|
|
728
|
+
await roundAgent.onResponse(contentMessage);
|
|
729
|
+
}
|
|
730
|
+
history.add(contentMessage);
|
|
731
|
+
yield {
|
|
732
|
+
type: 'response_output',
|
|
733
|
+
message: contentMessage,
|
|
734
|
+
request_id: roundRequestId,
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
break;
|
|
738
|
+
}
|
|
739
|
+
case 'file_complete': {
|
|
740
|
+
roundSummary.emittedTerminalOutput = true;
|
|
741
|
+
break;
|
|
375
742
|
}
|
|
376
|
-
|
|
377
|
-
const
|
|
378
|
-
if (
|
|
379
|
-
|
|
743
|
+
case 'tool_start': {
|
|
744
|
+
const toolEvent = event;
|
|
745
|
+
if (!toolEvent.tool_call) {
|
|
746
|
+
break;
|
|
747
|
+
}
|
|
748
|
+
if (!sawToolCallThisRound) {
|
|
749
|
+
roundSummary.emittedTerminalOutput = false;
|
|
750
|
+
roundSummary.messages = [];
|
|
380
751
|
}
|
|
381
|
-
|
|
752
|
+
sawToolCallThisRound = true;
|
|
753
|
+
const remainingCalls = maxToolCalls - currentToolCalls - roundSummary.toolCallsStarted;
|
|
754
|
+
if (remainingCalls <= 0) {
|
|
755
|
+
console.warn(`Tool call limit reached (${maxToolCalls}). Skipping tool calls.`);
|
|
756
|
+
const failure = (0, failure_detection_js_1.normalizeFailure)(new Error(`Tool call limit reached (${maxToolCalls}). Cannot execute tool ${toolEvent.tool_call.function.name}.`), {
|
|
757
|
+
recoverable: false,
|
|
758
|
+
reason: 'max_tool_calls_reached',
|
|
759
|
+
...getFailureRetryOverrides(agent),
|
|
760
|
+
});
|
|
761
|
+
roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, failure);
|
|
762
|
+
if (!roundSummary.errors.includes(failure.error)) {
|
|
763
|
+
roundSummary.errors.push(failure.error);
|
|
764
|
+
}
|
|
765
|
+
break;
|
|
766
|
+
}
|
|
767
|
+
const toolCall = toolEvent.tool_call;
|
|
768
|
+
const functionCall = (0, message_converter_js_1.convertToFunctionCall)(toolCall, model, 'completed');
|
|
769
|
+
history.add(functionCall);
|
|
382
770
|
yield {
|
|
383
771
|
type: 'response_output',
|
|
384
|
-
message:
|
|
385
|
-
request_id:
|
|
772
|
+
message: functionCall,
|
|
773
|
+
request_id: roundRequestId,
|
|
386
774
|
};
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
775
|
+
++roundSummary.toolCallsStarted;
|
|
776
|
+
if (!terminalToolNames.has(toolCall.function.name)) {
|
|
777
|
+
roundSummary.hasFollowupToolCalls = true;
|
|
778
|
+
}
|
|
779
|
+
await trace.emitToolStart(roundRequestId, toolCall.id, {
|
|
780
|
+
tool_name: toolCall.function.name,
|
|
781
|
+
arguments: toolCall.function.arguments,
|
|
782
|
+
arguments_formatted: toolCall.function.arguments_formatted,
|
|
783
|
+
});
|
|
784
|
+
const trackedExecution = {
|
|
785
|
+
toolCall,
|
|
786
|
+
promise: processToolCall(toolCall, {
|
|
787
|
+
...roundAgent,
|
|
788
|
+
abortSignal: toolExecutionGuard?.signal ?? roundAgent.abortSignal,
|
|
789
|
+
}),
|
|
790
|
+
settled: false,
|
|
791
|
+
};
|
|
792
|
+
trackedExecution.promise = trackedExecution.promise.then(result => {
|
|
793
|
+
if (!trackedExecution.settled) {
|
|
794
|
+
trackedExecution.settled = true;
|
|
795
|
+
trackedExecution.result = result;
|
|
796
|
+
}
|
|
797
|
+
return trackedExecution.result ?? result;
|
|
798
|
+
});
|
|
799
|
+
toolExecutions.push(trackedExecution);
|
|
399
800
|
break;
|
|
400
801
|
}
|
|
401
|
-
const toolCall = toolEvent.tool_call;
|
|
402
|
-
const functionCall = (0, message_converter_js_1.convertToFunctionCall)(toolCall, model, 'completed');
|
|
403
|
-
toolPromises.push(processToolCall(toolCall, agent));
|
|
404
|
-
history.add(functionCall);
|
|
405
|
-
yield {
|
|
406
|
-
type: 'response_output',
|
|
407
|
-
message: functionCall,
|
|
408
|
-
request_id: requestId,
|
|
409
|
-
};
|
|
410
|
-
break;
|
|
411
|
-
}
|
|
412
|
-
case 'error': {
|
|
413
|
-
console.error('[executeRound] Error event:', (0, truncate_utils_js_1.truncateLargeValues)(event.error));
|
|
414
|
-
break;
|
|
415
802
|
}
|
|
416
803
|
}
|
|
417
804
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
if (!isTerminalTool) {
|
|
447
|
-
const functionOutput = (0, message_converter_js_1.convertToFunctionCallOutput)(toolResult, model, 'completed');
|
|
448
|
-
history.add(functionOutput);
|
|
449
|
-
yield {
|
|
450
|
-
type: 'response_output',
|
|
451
|
-
message: functionOutput,
|
|
452
|
-
request_id: requestId,
|
|
453
|
-
};
|
|
805
|
+
catch (error) {
|
|
806
|
+
const streamFailure = (0, failure_detection_js_1.normalizeFailure)(error, {
|
|
807
|
+
reason: 'request_stream_failed',
|
|
808
|
+
...getFailureRetryOverrides(agent),
|
|
809
|
+
});
|
|
810
|
+
roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, streamFailure);
|
|
811
|
+
roundSummary.errors.push(streamFailure.error);
|
|
812
|
+
}
|
|
813
|
+
if (!sawTerminalProviderOutcome && !roundSummary.failure) {
|
|
814
|
+
const emptyResponseFailure = (0, failure_detection_js_1.normalizeFailure)(new Error(`Provider ${provider.provider_id} ended the stream without any terminal content, tool calls, files, or errors.`), {
|
|
815
|
+
recoverable: false,
|
|
816
|
+
reason: 'empty_provider_response',
|
|
817
|
+
...getFailureRetryOverrides(agent),
|
|
818
|
+
});
|
|
819
|
+
roundSummary.failure = emptyResponseFailure;
|
|
820
|
+
roundSummary.errors.push(emptyResponseFailure.error);
|
|
821
|
+
}
|
|
822
|
+
roundSummary.requestDuration = Date.now() - startTime;
|
|
823
|
+
const shouldUseBoundedFailureFinalization = Boolean(roundSummary.failure?.terminal);
|
|
824
|
+
yield* finalizeToolResults(shouldUseBoundedFailureFinalization ? 'bounded_failure' : 'wait_all');
|
|
825
|
+
if (requestGuard?.signal.aborted) {
|
|
826
|
+
const abortFailure = (0, failure_detection_js_1.normalizeFailure)(requestGuard.signal.reason, {
|
|
827
|
+
reason: 'request_stream_failed',
|
|
828
|
+
...getFailureRetryOverrides(agent),
|
|
829
|
+
});
|
|
830
|
+
roundSummary.failure = (0, failure_detection_js_1.selectMoreSevereFailure)(roundSummary.failure, abortFailure);
|
|
831
|
+
if (!roundSummary.errors.includes(abortFailure.error)) {
|
|
832
|
+
roundSummary.errors.push(abortFailure.error);
|
|
454
833
|
}
|
|
455
834
|
}
|
|
456
|
-
|
|
457
|
-
|
|
835
|
+
roundSummary.durationWithTools = Date.now() - startTime;
|
|
836
|
+
roundSummary.requestCost = totalCost > 0 ? totalCost : undefined;
|
|
837
|
+
roundSummary.agentDoneEvent = {
|
|
458
838
|
type: 'agent_done',
|
|
459
|
-
request_id:
|
|
460
|
-
request_cost:
|
|
461
|
-
request_duration,
|
|
462
|
-
duration_with_tools,
|
|
839
|
+
request_id: roundRequestId,
|
|
840
|
+
request_cost: roundSummary.requestCost,
|
|
841
|
+
request_duration: roundSummary.requestDuration,
|
|
842
|
+
duration_with_tools: roundSummary.durationWithTools,
|
|
463
843
|
timestamp: new Date().toISOString(),
|
|
464
844
|
};
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
}
|
|
845
|
+
roundSummary.agentDoneAgent = roundAgent;
|
|
846
|
+
requestGuard?.cleanup();
|
|
847
|
+
toolExecutionGuard?.cleanup();
|
|
848
|
+
return roundSummary;
|
|
470
849
|
}
|
|
471
850
|
async function* performVerification(agent, output, messages, attempt = 0) {
|
|
472
|
-
if (!agent.verifier)
|
|
473
|
-
return;
|
|
851
|
+
if (!agent.verifier) {
|
|
852
|
+
return { passed: true };
|
|
853
|
+
}
|
|
474
854
|
const maxAttempts = agent.maxVerificationAttempts || 2;
|
|
475
855
|
const verification = await (0, verification_js_1.verifyOutput)(agent.verifier, output, messages);
|
|
476
856
|
if (verification.status === 'pass') {
|
|
@@ -478,7 +858,7 @@ async function* performVerification(agent, output, messages, attempt = 0) {
|
|
|
478
858
|
type: 'message_delta',
|
|
479
859
|
content: '\n\n✓ Output verified',
|
|
480
860
|
};
|
|
481
|
-
return;
|
|
861
|
+
return { passed: true };
|
|
482
862
|
}
|
|
483
863
|
if (attempt < maxAttempts - 1) {
|
|
484
864
|
yield {
|
|
@@ -507,27 +887,37 @@ async function* performVerification(agent, output, messages, attempt = 0) {
|
|
|
507
887
|
const retryStream = ensembleRequest(retryMessages, retryAgent);
|
|
508
888
|
let retryOutput = '';
|
|
509
889
|
for await (const event of retryStream) {
|
|
890
|
+
if (event.type === 'operation_status' || event.type === 'error') {
|
|
891
|
+
continue;
|
|
892
|
+
}
|
|
510
893
|
yield event;
|
|
511
894
|
if (event.type === 'message_complete' && 'content' in event) {
|
|
512
895
|
retryOutput = event.content;
|
|
513
896
|
}
|
|
514
897
|
}
|
|
515
898
|
if (retryOutput) {
|
|
516
|
-
yield* performVerification(agent, retryOutput, messages, attempt + 1);
|
|
899
|
+
return yield* performVerification(agent, retryOutput, messages, attempt + 1);
|
|
517
900
|
}
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
type: 'message_delta',
|
|
522
|
-
content: `\n\n❌ Verification failed after ${maxAttempts} attempts: ${verification.reason}`,
|
|
901
|
+
return {
|
|
902
|
+
passed: false,
|
|
903
|
+
error: 'Verification retry did not produce a final response.',
|
|
523
904
|
};
|
|
524
905
|
}
|
|
906
|
+
const failureMessage = `Verification failed after ${maxAttempts} attempts: ${verification.reason}`;
|
|
907
|
+
yield {
|
|
908
|
+
type: 'message_delta',
|
|
909
|
+
content: `\n\n❌ ${failureMessage}`,
|
|
910
|
+
};
|
|
911
|
+
return {
|
|
912
|
+
passed: false,
|
|
913
|
+
error: failureMessage,
|
|
914
|
+
};
|
|
525
915
|
}
|
|
526
916
|
async function processToolCall(toolCall, agent) {
|
|
527
|
-
if (agent.onToolCall) {
|
|
528
|
-
await agent.onToolCall(toolCall);
|
|
529
|
-
}
|
|
530
917
|
try {
|
|
918
|
+
if (agent.onToolCall) {
|
|
919
|
+
await agent.onToolCall(toolCall);
|
|
920
|
+
}
|
|
531
921
|
if (!agent.tools) {
|
|
532
922
|
throw new Error('No tools available for agent');
|
|
533
923
|
}
|
|
@@ -535,7 +925,7 @@ async function processToolCall(toolCall, agent) {
|
|
|
535
925
|
if (!tool || !('function' in tool)) {
|
|
536
926
|
throw new Error(`Tool ${toolCall.function.name} not found`);
|
|
537
927
|
}
|
|
538
|
-
const rawResult = await (0, tool_execution_manager_js_1.handleToolCall)(toolCall, tool, agent);
|
|
928
|
+
const rawResult = await (0, tool_execution_manager_js_1.handleToolCall)(toolCall, tool, agent, agent.abortSignal);
|
|
539
929
|
const processedResult = await (0, tool_result_processor_js_1.processToolResult)(toolCall, rawResult, agent, tool.allowSummary);
|
|
540
930
|
const toolCallResult = {
|
|
541
931
|
toolCall,
|
|
@@ -549,21 +939,32 @@ async function processToolCall(toolCall, agent) {
|
|
|
549
939
|
return toolCallResult;
|
|
550
940
|
}
|
|
551
941
|
catch (error) {
|
|
552
|
-
const
|
|
553
|
-
? `Tool execution failed: ${error.message}`
|
|
554
|
-
: `Tool execution failed: ${String(error)}`;
|
|
555
|
-
const toolCallResult = {
|
|
556
|
-
toolCall,
|
|
557
|
-
id: toolCall.id,
|
|
558
|
-
call_id: toolCall.call_id || toolCall.id,
|
|
559
|
-
error: errorOutput,
|
|
560
|
-
};
|
|
942
|
+
const toolCallResult = createToolFailureResult(toolCall, error);
|
|
561
943
|
if (agent.onToolError) {
|
|
562
|
-
|
|
944
|
+
try {
|
|
945
|
+
await agent.onToolError(toolCallResult);
|
|
946
|
+
}
|
|
947
|
+
catch (hookError) {
|
|
948
|
+
console.error('[processToolCall] onToolError hook failed:', hookError);
|
|
949
|
+
}
|
|
563
950
|
}
|
|
564
951
|
return toolCallResult;
|
|
565
952
|
}
|
|
566
953
|
}
|
|
954
|
+
function createToolFailureResult(toolCall, error) {
|
|
955
|
+
const errorOutput = error instanceof Error
|
|
956
|
+
? `Tool execution failed: ${error.message}`
|
|
957
|
+
: `Tool execution failed: ${String(error)}`;
|
|
958
|
+
return {
|
|
959
|
+
toolCall,
|
|
960
|
+
id: toolCall.id,
|
|
961
|
+
call_id: toolCall.call_id || toolCall.id,
|
|
962
|
+
error: errorOutput,
|
|
963
|
+
};
|
|
964
|
+
}
|
|
965
|
+
function createToolFinalizationFailureResult(toolCall) {
|
|
966
|
+
return createToolFailureResult(toolCall, 'Tool did not finish before request finalization after a terminal provider failure.');
|
|
967
|
+
}
|
|
567
968
|
function mergeHistoryThread(mainHistory, thread, startIndex) {
|
|
568
969
|
const newMessages = thread.slice(startIndex);
|
|
569
970
|
mainHistory.push(...newMessages);
|