neoagent 3.2.1-beta.7 → 3.2.1-beta.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/server/http/routes.js +7 -0
- package/server/public/.last_build_id +1 -1
- package/server/public/flutter_bootstrap.js +2 -2
- package/server/public/main.dart.js +5 -5
- package/server/services/ai/loop/agent_engine_core.js +21 -1
- package/server/services/ai/loop/completion_judge.js +91 -13
- package/server/services/ai/loop/conversation_loop.js +122 -129
- package/server/services/ai/model_failure_cache.js +47 -8
- package/server/services/ai/providers/google.js +101 -95
- package/server/services/ai/taskAnalysis.js +15 -0
- package/server/services/ai/terminal_reply.js +13 -1
- package/server/services/ai/toolEvidence.js +354 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "neoagent",
|
|
3
|
-
"version": "3.2.1-beta.
|
|
3
|
+
"version": "3.2.1-beta.9",
|
|
4
4
|
"description": "Self-hosted AI agent for long-running tasks, automation, messaging, device control, and local memory",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"main": "server/index.js",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"dependencies": {
|
|
69
69
|
"@anthropic-ai/sdk": "^0.39.0",
|
|
70
70
|
"@bitwarden/cli": "^2026.6.0",
|
|
71
|
-
"@google/
|
|
71
|
+
"@google/genai": "^2.13.0",
|
|
72
72
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
73
73
|
"@remotion/cli": "^4.0.459",
|
|
74
74
|
"@slidev/cli": "^52.18.0",
|
package/server/http/routes.js
CHANGED
|
@@ -56,6 +56,13 @@ function registerApiRoutes(app) {
|
|
|
56
56
|
if (isBillingEnabled()) {
|
|
57
57
|
app.use('/api/billing/webhook', require('../routes/billing_webhook'));
|
|
58
58
|
app.use('/api/billing', require('../routes/billing'));
|
|
59
|
+
} else {
|
|
60
|
+
// The Flutter client probes this public endpoint to decide whether to show
|
|
61
|
+
// billing. Return an explicit disabled state instead of generating a noisy
|
|
62
|
+
// 404 on every client startup.
|
|
63
|
+
app.get('/api/billing/plans', (_req, res) => {
|
|
64
|
+
res.json({ enabled: false, plans: null });
|
|
65
|
+
});
|
|
59
66
|
}
|
|
60
67
|
|
|
61
68
|
setupTelnyxWebhook(app);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
1a325740c55a320441979a83b65c758d
|
|
@@ -33,10 +33,10 @@ addEventListener("message", eventListener);
|
|
|
33
33
|
if (!window._flutter) {
|
|
34
34
|
window._flutter = {};
|
|
35
35
|
}
|
|
36
|
-
_flutter.buildConfig = {"engineRevision":"
|
|
36
|
+
_flutter.buildConfig = {"engineRevision":"0cd610717bde95fd88343c64f81c11ba4e5c0010","builds":[{"compileTarget":"dart2js","renderer":"canvaskit","mainJsPath":"main.dart.js"},{}]};
|
|
37
37
|
|
|
38
38
|
_flutter.loader.load({
|
|
39
39
|
serviceWorkerSettings: {
|
|
40
|
-
serviceWorkerVersion: "
|
|
40
|
+
serviceWorkerVersion: "393614478" /* Flutter's service worker is deprecated and will be removed in a future Flutter release. */
|
|
41
41
|
}
|
|
42
42
|
});
|
|
@@ -44367,7 +44367,7 @@ $S:4}
|
|
|
44367
44367
|
A.bnI.prototype={
|
|
44368
44368
|
$1(a){var s=A.fp().b
|
|
44369
44369
|
s=s==null?null:s.canvasKitBaseUrl
|
|
44370
|
-
return(s==null?"https://www.gstatic.com/flutter-canvaskit/
|
|
44370
|
+
return(s==null?"https://www.gstatic.com/flutter-canvaskit/0cd610717bde95fd88343c64f81c11ba4e5c0010/":s)+a},
|
|
44371
44371
|
$S:42}
|
|
44372
44372
|
A.a2F.prototype={
|
|
44373
44373
|
gF(a){var s=this.a
|
|
@@ -144506,7 +144506,7 @@ if(r){r=s.d
|
|
|
144506
144506
|
r===$&&A.b()
|
|
144507
144507
|
p.push(A.ju(q,A.jx(!1,new A.Y(B.wP,A.cU(new A.cH(B.kE,new A.acU(r,q),q),q,q),q),!1,B.J,!0),q,q,0,0,0,q))}if(!s.ay){r=s.e
|
|
144508
144508
|
r===$&&A.b()
|
|
144509
|
-
r=B.b.t("
|
|
144509
|
+
r=B.b.t("ms0ggj2e-2c398a2").length!==0&&r.b}else r=!1
|
|
144510
144510
|
if(r){r=s.d
|
|
144511
144511
|
r===$&&A.b()
|
|
144512
144512
|
r=r.aU&&!r.ag?84:0
|
|
@@ -150393,7 +150393,7 @@ $S:0}
|
|
|
150393
150393
|
A.a3j.prototype={}
|
|
150394
150394
|
A.WC.prototype={
|
|
150395
150395
|
r9(a){var s=this
|
|
150396
|
-
if(B.b.t("
|
|
150396
|
+
if(B.b.t("ms0ggj2e-2c398a2").length===0||s.a!=null)return
|
|
150397
150397
|
s.Co()
|
|
150398
150398
|
s.a=A.mL(B.Yh,new A.bmK(s))},
|
|
150399
150399
|
Co(){var s=0,r=A.l(t.H),q,p=2,o=[],n=this,m,l,k,j,i,h,g,f
|
|
@@ -150411,7 +150411,7 @@ if(!t.f.b(k)){s=1
|
|
|
150411
150411
|
break}i=J.a0(k,"buildId")
|
|
150412
150412
|
h=i==null?null:B.b.t(J.q(i))
|
|
150413
150413
|
j=h==null?"":h
|
|
150414
|
-
if(J.bv(j)===0||J.e(j,"
|
|
150414
|
+
if(J.bv(j)===0||J.e(j,"ms0ggj2e-2c398a2")){s=1
|
|
150415
150415
|
break}n.b=!0
|
|
150416
150416
|
n.H()
|
|
150417
150417
|
p=2
|
|
@@ -150428,7 +150428,7 @@ case 2:return A.i(o.at(-1),r)}})
|
|
|
150428
150428
|
return A.k($async$Co,r)},
|
|
150429
150429
|
wV(){var s=0,r=A.l(t.H),q,p=2,o=[],n=this,m,l,k,j,i,h,g,f,e,d,c,b,a,a0,a1
|
|
150430
150430
|
var $async$wV=A.h(function(a2,a3){if(a2===1){o.push(a3)
|
|
150431
|
-
s=p}for(;;)switch(s){case 0:if(B.b.t("
|
|
150431
|
+
s=p}for(;;)switch(s){case 0:if(B.b.t("ms0ggj2e-2c398a2").length===0||n.c){s=1
|
|
150432
150432
|
break}n.c=!0
|
|
150433
150433
|
n.H()
|
|
150434
150434
|
p=4
|
|
@@ -87,6 +87,7 @@ const {
|
|
|
87
87
|
} = require('../messagingFallback');
|
|
88
88
|
const { isDeferredWorkReply } = require('../terminal_reply');
|
|
89
89
|
const {
|
|
90
|
+
assessResearchAdequacy,
|
|
90
91
|
summarizeToolExecutions,
|
|
91
92
|
} = require('../toolEvidence');
|
|
92
93
|
const {
|
|
@@ -793,6 +794,11 @@ class AgentEngine {
|
|
|
793
794
|
}) {
|
|
794
795
|
const runMeta = options?.runId ? this.getRunMeta(options.runId) : null;
|
|
795
796
|
const goalContext = resolveRunGoalContext(runMeta, analysis, plan);
|
|
797
|
+
const researchAdequacy = assessResearchAdequacy({
|
|
798
|
+
analysis,
|
|
799
|
+
goalContext,
|
|
800
|
+
toolExecutions,
|
|
801
|
+
});
|
|
796
802
|
const response = await this.requestStructuredJson({
|
|
797
803
|
provider,
|
|
798
804
|
providerName,
|
|
@@ -808,6 +814,8 @@ class AgentEngine {
|
|
|
808
814
|
lastReply,
|
|
809
815
|
iteration,
|
|
810
816
|
maxIterations,
|
|
817
|
+
analysis,
|
|
818
|
+
researchAdequacy,
|
|
811
819
|
}),
|
|
812
820
|
maxTokens: 500,
|
|
813
821
|
normalize: (raw) => normalizeCompletionDecision(raw, 'continue'),
|
|
@@ -817,9 +825,15 @@ class AgentEngine {
|
|
|
817
825
|
phase: 'completion_decision',
|
|
818
826
|
});
|
|
819
827
|
return {
|
|
820
|
-
decision: enforceTerminalReplyDecision(response.value, lastReply
|
|
828
|
+
decision: enforceTerminalReplyDecision(response.value, lastReply, {
|
|
829
|
+
analysis,
|
|
830
|
+
goalContext,
|
|
831
|
+
toolExecutions,
|
|
832
|
+
researchAdequacy,
|
|
833
|
+
}),
|
|
821
834
|
usage: response.usage,
|
|
822
835
|
raw: response.raw,
|
|
836
|
+
researchAdequacy,
|
|
823
837
|
};
|
|
824
838
|
}
|
|
825
839
|
|
|
@@ -905,6 +919,12 @@ class AgentEngine {
|
|
|
905
919
|
goalContext,
|
|
906
920
|
toolExecutions,
|
|
907
921
|
iteration,
|
|
922
|
+
analysis,
|
|
923
|
+
researchAdequacy: assessResearchAdequacy({
|
|
924
|
+
analysis,
|
|
925
|
+
goalContext,
|
|
926
|
+
toolExecutions,
|
|
927
|
+
}),
|
|
908
928
|
}),
|
|
909
929
|
maxTokens: 200,
|
|
910
930
|
normalize: normalizeChurnAssessment,
|
|
@@ -2,8 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
const { normalizeCompletionConfidence } = require('../completion');
|
|
4
4
|
const { normalizeOutgoingMessage } = require('../messagingFallback');
|
|
5
|
-
const { isDeferredWorkReply } = require('../terminal_reply');
|
|
6
5
|
const {
|
|
6
|
+
isDeferredWorkReply,
|
|
7
|
+
isTerminalQuestionOrBlockerReply,
|
|
8
|
+
} = require('../terminal_reply');
|
|
9
|
+
const {
|
|
10
|
+
assessResearchAdequacy,
|
|
11
|
+
formatResearchAdequacyGuidance,
|
|
7
12
|
summarizeAvailableTools,
|
|
8
13
|
summarizeToolExecutions,
|
|
9
14
|
} = require('../toolEvidence');
|
|
@@ -195,8 +200,15 @@ function buildCompletionDecisionPrompt({
|
|
|
195
200
|
lastReply,
|
|
196
201
|
iteration,
|
|
197
202
|
maxIterations,
|
|
203
|
+
analysis = null,
|
|
204
|
+
researchAdequacy = null,
|
|
198
205
|
}) {
|
|
199
206
|
const draftReply = normalizeOutgoingMessage(lastReply) || '';
|
|
207
|
+
const adequacy = researchAdequacy || assessResearchAdequacy({
|
|
208
|
+
analysis,
|
|
209
|
+
goalContext,
|
|
210
|
+
toolExecutions,
|
|
211
|
+
});
|
|
200
212
|
const lines = [
|
|
201
213
|
'Return JSON only.',
|
|
202
214
|
'Decide whether this run should continue autonomously or stop now.',
|
|
@@ -211,13 +223,29 @@ function buildCompletionDecisionPrompt({
|
|
|
211
223
|
'- A tool-specific API error, timeout, rate limit, or missing result inside this run is usually "continue", not "blocked", if any other available tool could still make progress.',
|
|
212
224
|
'- Repeated read-only inspection that has already established the relevant object is absent or unchanged is not progress. Accept a concise complete/blocker reply instead of requiring more searching.',
|
|
213
225
|
`- If completion_confidence_required is ${goalContext.effectiveCompletionConfidence} and the latest draft depends on unverified assumptions, use "continue" so the run can gather evidence, inspect state, or narrow the reply.`,
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
: triggerSource === 'messaging'
|
|
217
|
-
? '- For messaging, do not stop on a partial status message. Continue unless the task is actually complete or externally blocked.'
|
|
218
|
-
: '- Do not stop just because you wrote a status update. Continue unless the task is actually complete or externally blocked.',
|
|
226
|
+
'- When research intensity is light or deep, use "continue" until the required primary/source coverage and target coverage are met, or the draft is an explicit blocker naming exactly what could not be verified.',
|
|
227
|
+
'- Search snippets, memory, and model priors are leads, not completion evidence. Prefer opened/fetched primary sources before "complete".',
|
|
219
228
|
];
|
|
220
229
|
|
|
230
|
+
if (adequacy.intensity !== 'none') {
|
|
231
|
+
lines.push(
|
|
232
|
+
`- Research intensity for this run is ${adequacy.intensity}. Current coverage: primary=${adequacy.primarySourceCount}/${adequacy.requiredPrimarySources}, secondary=${adequacy.secondarySourceCount}, targets_covered=${adequacy.coveredTargets.length}/${Math.max(adequacy.requiredTargetCoverage, adequacy.targets.length)}.`,
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
if (adequacy.adequate === false) {
|
|
236
|
+
lines.push(
|
|
237
|
+
`- Research is still incomplete (${adequacy.reason}). Use "continue" unless the latest draft is an explicit blocker naming the exact missing evidence.`,
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
if (triggerSource === 'messaging' && messagingSent) {
|
|
242
|
+
lines.push('- A final reply was already delivered via send_message. Use "complete" unless concrete task work remains.');
|
|
243
|
+
} else if (triggerSource === 'messaging') {
|
|
244
|
+
lines.push('- For messaging, do not stop on a partial status message. Continue unless the task is actually complete or externally blocked.');
|
|
245
|
+
} else {
|
|
246
|
+
lines.push('- Do not stop just because you wrote a status update. Continue unless the task is actually complete or externally blocked.');
|
|
247
|
+
}
|
|
248
|
+
|
|
221
249
|
lines.push(
|
|
222
250
|
goalContext.effectiveGoal ? `Goal: ${goalContext.effectiveGoal}` : '',
|
|
223
251
|
goalContext.persistedGoalPrompt,
|
|
@@ -228,6 +256,16 @@ function buildCompletionDecisionPrompt({
|
|
|
228
256
|
`Current iteration: ${iteration} of ${maxIterations}.`,
|
|
229
257
|
`Available tools in this run: ${summarizeAvailableTools(tools) || 'none'}`,
|
|
230
258
|
`Recent tool evidence:\n${summarizeToolExecutions(toolExecutions, 8) || 'none'}`,
|
|
259
|
+
adequacy.intensity !== 'none'
|
|
260
|
+
? `Research adequacy: intensity=${adequacy.intensity}; adequate=${adequacy.adequate}; reason=${adequacy.reason}`
|
|
261
|
+
: '',
|
|
262
|
+
adequacy.targets.length
|
|
263
|
+
? `Research targets: ${adequacy.targets.join('; ')}`
|
|
264
|
+
: '',
|
|
265
|
+
adequacy.uncoveredTargets.length
|
|
266
|
+
? `Uncovered research targets: ${adequacy.uncoveredTargets.join('; ')}`
|
|
267
|
+
: '',
|
|
268
|
+
formatResearchAdequacyGuidance(adequacy),
|
|
231
269
|
`Latest draft reply:\n${draftReply || '(empty)'}`,
|
|
232
270
|
);
|
|
233
271
|
return lines.filter(Boolean).join('\n');
|
|
@@ -242,14 +280,39 @@ function normalizeCompletionDecision(raw, fallbackStatus = 'continue') {
|
|
|
242
280
|
};
|
|
243
281
|
}
|
|
244
282
|
|
|
245
|
-
function enforceTerminalReplyDecision(decision, lastReply) {
|
|
246
|
-
if (
|
|
247
|
-
return
|
|
283
|
+
function enforceTerminalReplyDecision(decision, lastReply, options = {}) {
|
|
284
|
+
if (isDeferredWorkReply(lastReply)) {
|
|
285
|
+
return {
|
|
286
|
+
status: 'continue',
|
|
287
|
+
reason: 'The latest reply only announces or promises unfinished work; the run must continue or return a concrete blocker.',
|
|
288
|
+
};
|
|
248
289
|
}
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
290
|
+
if (decision?.status === 'continue' && isTerminalQuestionOrBlockerReply(lastReply)) {
|
|
291
|
+
return {
|
|
292
|
+
status: 'blocked',
|
|
293
|
+
reason: 'The latest reply asks for user input or states a concrete blocker, so the run must wait instead of repeating it.',
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
const researchAdequacy = options.researchAdequacy
|
|
298
|
+
|| assessResearchAdequacy({
|
|
299
|
+
analysis: options.analysis || null,
|
|
300
|
+
goalContext: options.goalContext || null,
|
|
301
|
+
toolExecutions: options.toolExecutions || [],
|
|
302
|
+
});
|
|
303
|
+
if (
|
|
304
|
+
researchAdequacy
|
|
305
|
+
&& researchAdequacy.adequate === false
|
|
306
|
+
&& (decision?.status === 'complete' || decision?.status === 'blocked')
|
|
307
|
+
&& !isTerminalQuestionOrBlockerReply(lastReply)
|
|
308
|
+
) {
|
|
309
|
+
return {
|
|
310
|
+
status: 'continue',
|
|
311
|
+
reason: researchAdequacy.reason
|
|
312
|
+
|| 'Research evidence is still incomplete for the requested targets; continue gathering sources before finishing.',
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
return decision;
|
|
253
316
|
}
|
|
254
317
|
|
|
255
318
|
// Intentionally lightweight (200-token cap, self-contained) so the model can
|
|
@@ -260,7 +323,14 @@ function buildChurnAssessmentPrompt({
|
|
|
260
323
|
goalContext,
|
|
261
324
|
toolExecutions,
|
|
262
325
|
iteration,
|
|
326
|
+
analysis = null,
|
|
327
|
+
researchAdequacy = null,
|
|
263
328
|
}) {
|
|
329
|
+
const adequacy = researchAdequacy || assessResearchAdequacy({
|
|
330
|
+
analysis,
|
|
331
|
+
goalContext,
|
|
332
|
+
toolExecutions,
|
|
333
|
+
});
|
|
264
334
|
const lines = [
|
|
265
335
|
'Return JSON only.',
|
|
266
336
|
'Self-assess your current loop state — are you making genuine progress or spinning?',
|
|
@@ -274,11 +344,19 @@ function buildChurnAssessmentPrompt({
|
|
|
274
344
|
: '',
|
|
275
345
|
`Iteration: ${iteration}`,
|
|
276
346
|
`Recent tool evidence:\n${summarizeToolExecutions(toolExecutions, 6) || 'none'}`,
|
|
347
|
+
adequacy.intensity !== 'none'
|
|
348
|
+
? `Research adequacy: intensity=${adequacy.intensity}; adequate=${adequacy.adequate}; covered=${adequacy.coveredTargets.length}/${Math.max(adequacy.requiredTargetCoverage, adequacy.targets.length)}; primary=${adequacy.primarySourceCount}/${adequacy.requiredPrimarySources}.`
|
|
349
|
+
: '',
|
|
350
|
+
adequacy.uncoveredTargets.length
|
|
351
|
+
? `Still uncovered research targets: ${adequacy.uncoveredTargets.join('; ')}.`
|
|
352
|
+
: '',
|
|
277
353
|
'',
|
|
278
354
|
'Assessment rules:',
|
|
279
355
|
'- "progressing": You are systematically gathering necessary context and the next concrete action is already determined — you know exactly what to do next.',
|
|
280
356
|
'- "churn": You are re-reading/re-searching information already in context, or exploring without a clear next concrete step. Accept the nudge and act.',
|
|
281
357
|
'- "blocked": No concrete action is available in this run. You have all the evidence needed to deliver a truthful final answer or a specific external blocker.',
|
|
358
|
+
'- For multi-target research, keep "progressing" while uncovered targets remain and a fresh primary source can still be opened. Do not mark "blocked" just because you have partial notes.',
|
|
359
|
+
'- Re-querying the same snippet source for an already covered target is "churn". Opening a different primary source for an uncovered target is "progressing".',
|
|
282
360
|
];
|
|
283
361
|
return lines.filter(Boolean).join('\n');
|
|
284
362
|
}
|
|
@@ -422,13 +422,20 @@ async function getFailureFallbackModelId(
|
|
|
422
422
|
preferredFallbackId = null,
|
|
423
423
|
failureError = null,
|
|
424
424
|
signal = null,
|
|
425
|
+
excludedModelIds = [],
|
|
425
426
|
) {
|
|
426
427
|
const { getSupportedModels } = require('../models');
|
|
427
428
|
const aiSettings = getAiSettings(userId, agentId);
|
|
428
429
|
const models = await getSupportedModels(userId, agentId, { signal });
|
|
430
|
+
const excluded = new Set(
|
|
431
|
+
[...excludedModelIds]
|
|
432
|
+
.map((id) => String(id || '').trim())
|
|
433
|
+
.filter(Boolean),
|
|
434
|
+
);
|
|
429
435
|
const availableModels = models.filter(
|
|
430
436
|
(model) => model.available !== false
|
|
431
|
-
&& !isModelCoolingDown(userId, agentId, model.id)
|
|
437
|
+
&& !isModelCoolingDown(userId, agentId, model.id)
|
|
438
|
+
&& !excluded.has(model.id),
|
|
432
439
|
);
|
|
433
440
|
const configuredEnabledIds = Array.isArray(aiSettings.enabled_models)
|
|
434
441
|
? aiSettings.enabled_models.map((id) => String(id).trim()).filter(Boolean)
|
|
@@ -438,15 +445,26 @@ async function getFailureFallbackModelId(
|
|
|
438
445
|
? availableModels.filter((model) => enabledIds.includes(model.id))
|
|
439
446
|
: availableModels;
|
|
440
447
|
const fallbackSearchPool = pool;
|
|
441
|
-
const currentModel = resolveModelSelection(
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
//
|
|
445
|
-
//
|
|
446
|
-
|
|
447
|
-
|
|
448
|
+
const currentModel = resolveModelSelection(models, currentModelId);
|
|
449
|
+
|
|
450
|
+
// Provider-wide failures (credentials, throttling, or service outages) are
|
|
451
|
+
// likely to affect another model on the same provider. Prefer a different
|
|
452
|
+
// provider before consulting the configured fallback.
|
|
453
|
+
const failureStatus = Number(
|
|
454
|
+
failureError?.status
|
|
455
|
+
?? failureError?.statusCode
|
|
456
|
+
?? failureError?.response?.status,
|
|
457
|
+
);
|
|
458
|
+
const isProviderScopedFailure = (
|
|
459
|
+
failureStatus === 401
|
|
460
|
+
|| failureStatus === 403
|
|
461
|
+
|| failureStatus === 429
|
|
462
|
+
|| (failureStatus >= 500 && failureStatus < 600)
|
|
463
|
+
|| /rate.?limit|free-models-per|service unavailable|provider unavailable|authentication|api key/i
|
|
464
|
+
.test(String(failureError?.message || ''))
|
|
465
|
+
);
|
|
448
466
|
|
|
449
|
-
if (preferredFallbackId && !
|
|
467
|
+
if (preferredFallbackId && !isProviderScopedFailure) {
|
|
450
468
|
const preferred = resolveModelSelection(fallbackSearchPool, preferredFallbackId)
|
|
451
469
|
|| resolveModelSelection(availableModels, preferredFallbackId);
|
|
452
470
|
if (preferred && preferred.id !== currentModel?.id) return preferred.id;
|
|
@@ -458,8 +476,7 @@ async function getFailureFallbackModelId(
|
|
|
458
476
|
if (differentProvider) return differentProvider.id;
|
|
459
477
|
}
|
|
460
478
|
|
|
461
|
-
// If no different-provider model exists, still try the preferred fallback
|
|
462
|
-
// even on rate limits (it's better than nothing).
|
|
479
|
+
// If no different-provider model exists, still try the preferred fallback.
|
|
463
480
|
if (preferredFallbackId) {
|
|
464
481
|
const preferred = resolveModelSelection(fallbackSearchPool, preferredFallbackId)
|
|
465
482
|
|| resolveModelSelection(availableModels, preferredFallbackId);
|
|
@@ -491,6 +508,7 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
491
508
|
let model = null;
|
|
492
509
|
let modelSelectionId = null;
|
|
493
510
|
let providerName = null;
|
|
511
|
+
const modelTurnFailedModelSelectionIds = new Set();
|
|
494
512
|
let messages = [];
|
|
495
513
|
let iteration = 0;
|
|
496
514
|
let totalTokens = 0;
|
|
@@ -606,8 +624,17 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
606
624
|
}
|
|
607
625
|
db.prepare('UPDATE agent_runs SET model = ?, updated_at = datetime(\'now\') WHERE id = ?')
|
|
608
626
|
.run(modelSelectionId, runId);
|
|
609
|
-
const
|
|
627
|
+
const recordFailedModel = (failedSelectionId, error, excludedModels) => {
|
|
628
|
+
excludedModels.add(failedSelectionId);
|
|
610
629
|
recordModelFailure(userId, agentId, failedSelectionId, error);
|
|
630
|
+
};
|
|
631
|
+
const switchToFallbackModel = async (
|
|
632
|
+
failedSelectionId,
|
|
633
|
+
error,
|
|
634
|
+
phase,
|
|
635
|
+
excludedModels,
|
|
636
|
+
) => {
|
|
637
|
+
recordFailedModel(failedSelectionId, error, excludedModels);
|
|
611
638
|
const fallbackModelId = await getFailureFallbackModelId(
|
|
612
639
|
userId,
|
|
613
640
|
agentId,
|
|
@@ -615,9 +642,21 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
615
642
|
aiSettings.fallback_model_id,
|
|
616
643
|
error,
|
|
617
644
|
engine.getRunMeta(runId)?.abortController?.signal,
|
|
645
|
+
excludedModels,
|
|
618
646
|
);
|
|
619
647
|
if (!fallbackModelId || fallbackModelId === failedSelectionId) return false;
|
|
620
|
-
|
|
648
|
+
const failureSummary = summarizeForLog(error?.message || error, 180);
|
|
649
|
+
console.log(
|
|
650
|
+
`[Engine] ${phase} failed on ${failedSelectionId}: ${failureSummary}; attempting fallback to: ${fallbackModelId}`
|
|
651
|
+
);
|
|
652
|
+
engine.recordRunEvent(userId, runId, 'model_fallback', {
|
|
653
|
+
phase,
|
|
654
|
+
failedModel: failedSelectionId,
|
|
655
|
+
fallbackModel: fallbackModelId,
|
|
656
|
+
errorCode: error?.code || null,
|
|
657
|
+
errorStatus: error?.status || error?.statusCode || error?.response?.status || null,
|
|
658
|
+
error: failureSummary,
|
|
659
|
+
}, { agentId });
|
|
621
660
|
engine.emit(userId, 'run:interim', {
|
|
622
661
|
runId,
|
|
623
662
|
message: `Model service failed on ${failedSelectionId}; retrying with ${fallbackModelId}.`,
|
|
@@ -647,17 +686,28 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
647
686
|
return true;
|
|
648
687
|
};
|
|
649
688
|
const runWithModelFallback = async (phase, fn) => {
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
689
|
+
let recoveries = 0;
|
|
690
|
+
const failedModels = new Set();
|
|
691
|
+
while (true) {
|
|
692
|
+
try {
|
|
693
|
+
const result = await fn();
|
|
694
|
+
recordModelSuccess(userId, agentId, modelSelectionId);
|
|
695
|
+
return result;
|
|
696
|
+
} catch (err) {
|
|
697
|
+
if (recoveries >= loopPolicy.maxModelFailureRecoveries) {
|
|
698
|
+
recordFailedModel(modelSelectionId, err, failedModels);
|
|
699
|
+
throw err;
|
|
700
|
+
}
|
|
701
|
+
const failedSelectionId = modelSelectionId;
|
|
702
|
+
const switched = await switchToFallbackModel(
|
|
703
|
+
failedSelectionId,
|
|
704
|
+
err,
|
|
705
|
+
phase,
|
|
706
|
+
failedModels,
|
|
707
|
+
);
|
|
708
|
+
if (!switched) throw err;
|
|
709
|
+
recoveries += 1;
|
|
710
|
+
}
|
|
661
711
|
}
|
|
662
712
|
};
|
|
663
713
|
|
|
@@ -1483,102 +1533,69 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
1483
1533
|
let responseModel = model;
|
|
1484
1534
|
let streamContent = '';
|
|
1485
1535
|
|
|
1486
|
-
const tryModelCall = async (
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
messages,
|
|
1493
|
-
tools,
|
|
1494
|
-
options: {
|
|
1495
|
-
...options,
|
|
1496
|
-
userId,
|
|
1497
|
-
agentId,
|
|
1498
|
-
runId,
|
|
1499
|
-
phase: 'model_turn',
|
|
1500
|
-
signal: engine.getRunMeta(runId)?.abortController?.signal,
|
|
1501
|
-
},
|
|
1502
|
-
runId,
|
|
1503
|
-
iteration,
|
|
1504
|
-
});
|
|
1505
|
-
response = modelCall.response;
|
|
1506
|
-
responseModel = modelCall.responseModel;
|
|
1507
|
-
streamContent = modelCall.streamContent;
|
|
1508
|
-
recordModelSuccess(userId, agentId, modelSelectionId);
|
|
1509
|
-
} catch (err) {
|
|
1510
|
-
console.error(`[Engine] Model call failed (${model}):`, err.message);
|
|
1511
|
-
recordModelFailure(userId, agentId, modelSelectionId, err);
|
|
1512
|
-
const fallbackModelId = retryForFallback
|
|
1513
|
-
? await getFailureFallbackModelId(
|
|
1514
|
-
userId,
|
|
1515
|
-
agentId,
|
|
1516
|
-
modelSelectionId,
|
|
1517
|
-
aiSettings.fallback_model_id,
|
|
1518
|
-
err,
|
|
1519
|
-
engine.getRunMeta(runId)?.abortController?.signal,
|
|
1520
|
-
)
|
|
1521
|
-
: null;
|
|
1522
|
-
if (fallbackModelId) {
|
|
1523
|
-
const failedModel = model;
|
|
1524
|
-
console.log(`[Engine] Attempting fallback to: ${fallbackModelId}`);
|
|
1525
|
-
const fallback = await getProviderForUser(
|
|
1526
|
-
userId,
|
|
1527
|
-
userMessage,
|
|
1528
|
-
triggerType === 'subagent',
|
|
1529
|
-
fallbackModelId,
|
|
1530
|
-
{
|
|
1531
|
-
...providerStatusConfig,
|
|
1532
|
-
signal: engine.getRunMeta(runId)?.abortController?.signal,
|
|
1533
|
-
}
|
|
1534
|
-
);
|
|
1535
|
-
provider = fallback.provider;
|
|
1536
|
-
model = fallback.model;
|
|
1537
|
-
modelSelectionId = fallback.modelSelectionId;
|
|
1538
|
-
providerName = fallback.providerName;
|
|
1539
|
-
db.prepare('UPDATE agent_runs SET model = ?, updated_at = datetime(\'now\') WHERE id = ?')
|
|
1540
|
-
.run(modelSelectionId, runId);
|
|
1541
|
-
Object.assign(engine.getRunMeta(runId) || {}, {
|
|
1542
|
-
model,
|
|
1543
|
-
modelSelectionId,
|
|
1544
|
-
providerName,
|
|
1545
|
-
});
|
|
1546
|
-
|
|
1547
|
-
const retryMessages = sanitizeConversationMessages([
|
|
1548
|
-
...messages,
|
|
1549
|
-
{
|
|
1550
|
-
role: 'system',
|
|
1551
|
-
content: buildModelFailureLoopPrompt({
|
|
1552
|
-
failedModel,
|
|
1553
|
-
nextModel: model,
|
|
1554
|
-
errorMessage: err.message
|
|
1555
|
-
})
|
|
1556
|
-
}
|
|
1557
|
-
]);
|
|
1558
|
-
|
|
1559
|
-
const fallbackCall = await engine.requestModelResponse({
|
|
1536
|
+
const tryModelCall = async () => {
|
|
1537
|
+
let requestMessages = messages;
|
|
1538
|
+
let requestPhase = 'model_turn';
|
|
1539
|
+
while (true) {
|
|
1540
|
+
try {
|
|
1541
|
+
const modelCall = await engine.requestModelResponse({
|
|
1560
1542
|
provider,
|
|
1561
1543
|
providerName,
|
|
1562
1544
|
model,
|
|
1563
|
-
messages:
|
|
1545
|
+
messages: requestMessages,
|
|
1564
1546
|
tools,
|
|
1565
1547
|
options: {
|
|
1566
1548
|
...options,
|
|
1567
1549
|
userId,
|
|
1568
1550
|
agentId,
|
|
1569
1551
|
runId,
|
|
1570
|
-
phase:
|
|
1552
|
+
phase: requestPhase,
|
|
1571
1553
|
signal: engine.getRunMeta(runId)?.abortController?.signal,
|
|
1572
1554
|
},
|
|
1573
1555
|
runId,
|
|
1574
1556
|
iteration,
|
|
1575
1557
|
});
|
|
1576
|
-
response =
|
|
1577
|
-
responseModel =
|
|
1578
|
-
streamContent =
|
|
1558
|
+
response = modelCall.response;
|
|
1559
|
+
responseModel = modelCall.responseModel;
|
|
1560
|
+
streamContent = modelCall.streamContent;
|
|
1579
1561
|
recordModelSuccess(userId, agentId, modelSelectionId);
|
|
1580
|
-
|
|
1581
|
-
|
|
1562
|
+
return;
|
|
1563
|
+
} catch (err) {
|
|
1564
|
+
console.error(`[Engine] Model call failed (${model}):`, err.message);
|
|
1565
|
+
const runSignal = engine.getRunMeta(runId)?.abortController?.signal;
|
|
1566
|
+
if (isAbortError(err) || runSignal?.aborted) throw err;
|
|
1567
|
+
if (modelFailureRecoveries >= loopPolicy.maxModelFailureRecoveries) {
|
|
1568
|
+
recordFailedModel(
|
|
1569
|
+
modelSelectionId,
|
|
1570
|
+
err,
|
|
1571
|
+
modelTurnFailedModelSelectionIds,
|
|
1572
|
+
);
|
|
1573
|
+
throw err;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
const failedModel = model;
|
|
1577
|
+
const switched = await switchToFallbackModel(
|
|
1578
|
+
modelSelectionId,
|
|
1579
|
+
err,
|
|
1580
|
+
'model turn',
|
|
1581
|
+
modelTurnFailedModelSelectionIds,
|
|
1582
|
+
);
|
|
1583
|
+
if (!switched) throw err;
|
|
1584
|
+
|
|
1585
|
+
modelFailureRecoveries += 1;
|
|
1586
|
+
failedStepCount += 1;
|
|
1587
|
+
requestPhase = 'model_turn_fallback';
|
|
1588
|
+
requestMessages = sanitizeConversationMessages([
|
|
1589
|
+
...messages,
|
|
1590
|
+
{
|
|
1591
|
+
role: 'system',
|
|
1592
|
+
content: buildModelFailureLoopPrompt({
|
|
1593
|
+
failedModel,
|
|
1594
|
+
nextModel: model,
|
|
1595
|
+
errorMessage: err.message
|
|
1596
|
+
})
|
|
1597
|
+
}
|
|
1598
|
+
]);
|
|
1582
1599
|
}
|
|
1583
1600
|
}
|
|
1584
1601
|
};
|
|
@@ -1601,30 +1618,6 @@ async function runConversation(engine, userId, userMessage, options = {}, _model
|
|
|
1601
1618
|
continue;
|
|
1602
1619
|
}
|
|
1603
1620
|
if (lifecycleControl?.action === 'stop' || lifecycleControl?.action === 'interrupt') break;
|
|
1604
|
-
const modelError = String(err?.message || 'Model call failed');
|
|
1605
|
-
|
|
1606
|
-
if (modelFailureRecoveries < loopPolicy.maxModelFailureRecoveries) {
|
|
1607
|
-
const failedModel = model;
|
|
1608
|
-
const switched = await switchToFallbackModel(modelSelectionId, err, 'model turn');
|
|
1609
|
-
if (!switched) throw err;
|
|
1610
|
-
modelFailureRecoveries += 1;
|
|
1611
|
-
failedStepCount += 1;
|
|
1612
|
-
messages.push({
|
|
1613
|
-
role: 'system',
|
|
1614
|
-
content: buildModelFailureLoopPrompt({
|
|
1615
|
-
failedModel,
|
|
1616
|
-
nextModel: model,
|
|
1617
|
-
errorMessage: modelError
|
|
1618
|
-
})
|
|
1619
|
-
});
|
|
1620
|
-
engine.emit(userId, 'run:interim', {
|
|
1621
|
-
runId,
|
|
1622
|
-
message: 'Model call failed; adapting and retrying autonomously.',
|
|
1623
|
-
phase: 'recovering'
|
|
1624
|
-
});
|
|
1625
|
-
continue;
|
|
1626
|
-
}
|
|
1627
|
-
|
|
1628
1621
|
throw err;
|
|
1629
1622
|
}
|
|
1630
1623
|
|