@probelabs/probe 0.6.0-rc295 → 0.6.0-rc297
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/bin/binaries/{probe-v0.6.0-rc295-aarch64-apple-darwin.tar.gz → probe-v0.6.0-rc297-aarch64-apple-darwin.tar.gz} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc295-aarch64-unknown-linux-musl.tar.gz → probe-v0.6.0-rc297-aarch64-unknown-linux-musl.tar.gz} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc295-x86_64-apple-darwin.tar.gz → probe-v0.6.0-rc297-x86_64-apple-darwin.tar.gz} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc295-x86_64-pc-windows-msvc.zip → probe-v0.6.0-rc297-x86_64-pc-windows-msvc.zip} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc295-x86_64-unknown-linux-musl.tar.gz → probe-v0.6.0-rc297-x86_64-unknown-linux-musl.tar.gz} +0 -0
- package/build/agent/ProbeAgent.d.ts +40 -2
- package/build/agent/ProbeAgent.js +703 -11
- package/build/agent/mcp/client.js +115 -4
- package/build/agent/mcp/xmlBridge.js +13 -1
- package/build/agent/otelLogBridge.js +184 -0
- package/build/agent/simpleTelemetry.js +8 -0
- package/build/delegate.js +75 -6
- package/build/index.js +6 -2
- package/build/tools/common.js +84 -11
- package/build/tools/vercel.js +78 -18
- package/cjs/agent/ProbeAgent.cjs +1095 -185
- package/cjs/agent/simpleTelemetry.cjs +112 -0
- package/cjs/index.cjs +1207 -185
- package/index.d.ts +26 -0
- package/package.json +2 -2
- package/src/agent/ProbeAgent.d.ts +40 -2
- package/src/agent/ProbeAgent.js +703 -11
- package/src/agent/mcp/client.js +115 -4
- package/src/agent/mcp/xmlBridge.js +13 -1
- package/src/agent/otelLogBridge.js +184 -0
- package/src/agent/simpleTelemetry.js +8 -0
- package/src/delegate.js +75 -6
- package/src/index.js +6 -2
- package/src/tools/common.js +84 -11
- package/src/tools/vercel.js +78 -18
|
@@ -31,7 +31,7 @@ import { createAnthropic } from '@ai-sdk/anthropic';
|
|
|
31
31
|
import { createOpenAI } from '@ai-sdk/openai';
|
|
32
32
|
import { createGoogleGenerativeAI } from '@ai-sdk/google';
|
|
33
33
|
import { createAmazonBedrock } from '@ai-sdk/amazon-bedrock';
|
|
34
|
-
import { streamText, tool, stepCountIs, jsonSchema, Output } from 'ai';
|
|
34
|
+
import { streamText, generateText, tool, stepCountIs, jsonSchema, Output } from 'ai';
|
|
35
35
|
import { randomUUID } from 'crypto';
|
|
36
36
|
import { EventEmitter } from 'events';
|
|
37
37
|
import { existsSync } from 'fs';
|
|
@@ -214,6 +214,7 @@ export class ProbeAgent {
|
|
|
214
214
|
this.debug = options.debug || process.env.DEBUG === '1';
|
|
215
215
|
this.cancelled = false;
|
|
216
216
|
this._abortController = new AbortController();
|
|
217
|
+
this._activeSubagents = new Map(); // sessionId → subagent ProbeAgent instance
|
|
217
218
|
this.tracer = options.tracer || null;
|
|
218
219
|
this.outline = !!options.outline;
|
|
219
220
|
this.searchDelegate = options.searchDelegate !== undefined ? !!options.searchDelegate : true;
|
|
@@ -391,10 +392,12 @@ export class ProbeAgent {
|
|
|
391
392
|
console.log(`[DEBUG] Max operation timeout: ${this.maxOperationTimeout}ms`);
|
|
392
393
|
}
|
|
393
394
|
|
|
394
|
-
// Timeout behavior: 'graceful' (default) winds down with bonus steps, 'hard' aborts immediately
|
|
395
|
+
// Timeout behavior: 'graceful' (default) winds down with bonus steps, 'hard' aborts immediately,
|
|
396
|
+
// 'negotiated' lets the AI request more time via request_more_time tool
|
|
395
397
|
this.timeoutBehavior = options.timeoutBehavior ?? (() => {
|
|
396
398
|
const val = process.env.TIMEOUT_BEHAVIOR;
|
|
397
399
|
if (val === 'hard') return 'hard';
|
|
400
|
+
if (val === 'negotiated') return 'negotiated';
|
|
398
401
|
return 'graceful';
|
|
399
402
|
})();
|
|
400
403
|
|
|
@@ -404,8 +407,32 @@ export class ProbeAgent {
|
|
|
404
407
|
return (isNaN(parsed) || parsed < 1 || parsed > 20) ? 4 : parsed;
|
|
405
408
|
})();
|
|
406
409
|
|
|
410
|
+
// Negotiated timeout: total extra time budget in ms (default 30 min)
|
|
411
|
+
this.negotiatedTimeoutBudget = options.negotiatedTimeoutBudget ?? (() => {
|
|
412
|
+
const parsed = parseInt(process.env.NEGOTIATED_TIMEOUT_BUDGET, 10);
|
|
413
|
+
return (isNaN(parsed) || parsed < 60000 || parsed > 7200000) ? 1800000 : parsed;
|
|
414
|
+
})();
|
|
415
|
+
|
|
416
|
+
// Negotiated timeout: max extension requests (default 3)
|
|
417
|
+
this.negotiatedTimeoutMaxRequests = options.negotiatedTimeoutMaxRequests ?? (() => {
|
|
418
|
+
const parsed = parseInt(process.env.NEGOTIATED_TIMEOUT_MAX_REQUESTS, 10);
|
|
419
|
+
return (isNaN(parsed) || parsed < 1 || parsed > 10) ? 3 : parsed;
|
|
420
|
+
})();
|
|
421
|
+
|
|
422
|
+
// Negotiated timeout: max ms per extension request (default 10 min)
|
|
423
|
+
this.negotiatedTimeoutMaxPerRequest = options.negotiatedTimeoutMaxPerRequest ?? (() => {
|
|
424
|
+
const parsed = parseInt(process.env.NEGOTIATED_TIMEOUT_MAX_PER_REQUEST, 10);
|
|
425
|
+
return (isNaN(parsed) || parsed < 60000 || parsed > 3600000) ? 600000 : parsed;
|
|
426
|
+
})();
|
|
427
|
+
|
|
428
|
+
// Graceful stop deadline: how long to wait for subagents/MCP after observer declines (default 45s)
|
|
429
|
+
this.gracefulStopDeadline = options.gracefulStopDeadline ?? (() => {
|
|
430
|
+
const parsed = parseInt(process.env.GRACEFUL_STOP_DEADLINE, 10);
|
|
431
|
+
return (isNaN(parsed) || parsed < 5000 || parsed > 300000) ? 45000 : parsed;
|
|
432
|
+
})();
|
|
433
|
+
|
|
407
434
|
if (this.debug) {
|
|
408
|
-
console.log(`[DEBUG] Timeout behavior: ${this.timeoutBehavior}, bonus steps: ${this.gracefulTimeoutBonusSteps}`);
|
|
435
|
+
console.log(`[DEBUG] Timeout behavior: ${this.timeoutBehavior}, bonus steps: ${this.gracefulTimeoutBonusSteps}, graceful stop deadline: ${this.gracefulStopDeadline}ms`);
|
|
409
436
|
}
|
|
410
437
|
|
|
411
438
|
// Retry configuration
|
|
@@ -843,6 +870,17 @@ export class ProbeAgent {
|
|
|
843
870
|
searchDelegateModel: this.searchDelegateModel,
|
|
844
871
|
delegationManager: this.delegationManager, // Per-instance delegation limits
|
|
845
872
|
parentAbortSignal: this._abortController.signal, // Propagate cancellation to delegations
|
|
873
|
+
// Timeout settings for delegate subagents to inherit
|
|
874
|
+
timeoutBehavior: this.timeoutBehavior,
|
|
875
|
+
maxOperationTimeout: this.maxOperationTimeout,
|
|
876
|
+
requestTimeout: this.requestTimeout,
|
|
877
|
+
gracefulTimeoutBonusSteps: this.gracefulTimeoutBonusSteps,
|
|
878
|
+
negotiatedTimeoutBudget: this.negotiatedTimeoutBudget,
|
|
879
|
+
negotiatedTimeoutMaxRequests: this.negotiatedTimeoutMaxRequests,
|
|
880
|
+
negotiatedTimeoutMaxPerRequest: this.negotiatedTimeoutMaxPerRequest,
|
|
881
|
+
parentOperationStartTime: this._operationStartTime, // For remaining budget calculation
|
|
882
|
+
onSubagentCreated: (sid, subagent) => this._registerSubagent(sid, subagent),
|
|
883
|
+
onSubagentCompleted: (sid) => this._unregisterSubagent(sid),
|
|
846
884
|
outputBuffer: this._outputBuffer,
|
|
847
885
|
concurrencyLimiter: this.concurrencyLimiter, // Global AI concurrency limiter
|
|
848
886
|
isToolAllowed,
|
|
@@ -1577,8 +1615,8 @@ export class ProbeAgent {
|
|
|
1577
1615
|
// This timer only handles the hard abort for non-graceful mode and engine paths.
|
|
1578
1616
|
if (this.maxOperationTimeout && this.maxOperationTimeout > 0) {
|
|
1579
1617
|
const gts = this._gracefulTimeoutState;
|
|
1580
|
-
if (this.timeoutBehavior === 'graceful' && gts) {
|
|
1581
|
-
// Graceful mode: timer is managed in run() method.
|
|
1618
|
+
if ((this.timeoutBehavior === 'graceful' || this.timeoutBehavior === 'negotiated') && gts) {
|
|
1619
|
+
// Graceful/negotiated mode: timer is managed in run() method.
|
|
1582
1620
|
// Only set up the AbortController link (no timer here).
|
|
1583
1621
|
} else {
|
|
1584
1622
|
// Hard mode: immediate abort (legacy behavior)
|
|
@@ -2721,7 +2759,7 @@ export class ProbeAgent {
|
|
|
2721
2759
|
}
|
|
2722
2760
|
|
|
2723
2761
|
// Initialize the MCP XML bridge
|
|
2724
|
-
this.mcpBridge = new MCPXmlBridge({ debug: this.debug });
|
|
2762
|
+
this.mcpBridge = new MCPXmlBridge({ debug: this.debug, agentEvents: this.events });
|
|
2725
2763
|
await this.mcpBridge.initialize(mcpConfig);
|
|
2726
2764
|
|
|
2727
2765
|
const mcpToolNames = this.mcpBridge.getToolNames();
|
|
@@ -3269,6 +3307,9 @@ Follow these instructions carefully:
|
|
|
3269
3307
|
options = schemaOrOptions || {};
|
|
3270
3308
|
}
|
|
3271
3309
|
|
|
3310
|
+
// Track operation start time for delegate budget calculation
|
|
3311
|
+
this._operationStartTime = Date.now();
|
|
3312
|
+
|
|
3272
3313
|
try {
|
|
3273
3314
|
// Track initial history length for storage
|
|
3274
3315
|
const oldHistoryLength = this.history.length;
|
|
@@ -3392,7 +3433,11 @@ Follow these instructions carefully:
|
|
|
3392
3433
|
}
|
|
3393
3434
|
|
|
3394
3435
|
let currentIteration = 0;
|
|
3395
|
-
let finalResult =
|
|
3436
|
+
let finalResult = null; // Will be set to a descriptive failure message if max iterations reached
|
|
3437
|
+
const DEFAULT_MAX_ITER_MSG = 'I was unable to complete your request due to reaching the maximum number of tool iterations.';
|
|
3438
|
+
// Track all tool calls across iterations for failure diagnostics
|
|
3439
|
+
const _toolCallLog = []; // { name, args (truncated) }
|
|
3440
|
+
let abortSummaryTaken = false; // Set when negotiated timeout abort summary runs — skip completionPrompt
|
|
3396
3441
|
|
|
3397
3442
|
// Adjust max iterations if schema is provided
|
|
3398
3443
|
// +1 for schema formatting
|
|
@@ -3589,6 +3634,293 @@ Follow these instructions carefully:
|
|
|
3589
3634
|
};
|
|
3590
3635
|
this._gracefulTimeoutState = gracefulTimeoutState;
|
|
3591
3636
|
|
|
3637
|
+
// Negotiated timeout state — used when timeoutBehavior === 'negotiated'
|
|
3638
|
+
// The "timeout observer" pattern: when timeout fires, a separate LLM call
|
|
3639
|
+
// decides whether to extend — this works even when the main loop is blocked
|
|
3640
|
+
// by a long-running delegate or MCP tool.
|
|
3641
|
+
const negotiatedTimeoutState = {
|
|
3642
|
+
extensionsUsed: 0,
|
|
3643
|
+
totalExtraTimeMs: 0,
|
|
3644
|
+
softTimeoutId: null,
|
|
3645
|
+
hardAbortTimeoutId: null,
|
|
3646
|
+
maxRequests: this.negotiatedTimeoutMaxRequests,
|
|
3647
|
+
maxPerRequestMs: this.negotiatedTimeoutMaxPerRequest,
|
|
3648
|
+
budgetMs: this.negotiatedTimeoutBudget,
|
|
3649
|
+
observerRunning: false, // true while observer LLM call is in flight
|
|
3650
|
+
extensionMessage: null, // message to show in prepareStep after extension granted
|
|
3651
|
+
startTime: Date.now(),
|
|
3652
|
+
};
|
|
3653
|
+
|
|
3654
|
+
this._negotiatedTimeoutState = negotiatedTimeoutState;
|
|
3655
|
+
|
|
3656
|
+
// Track in-flight tools via event emitter
|
|
3657
|
+
const activeTools = new Map(); // toolCallId → { name, args, startedAt }
|
|
3658
|
+
this._activeTools = activeTools;
|
|
3659
|
+
|
|
3660
|
+
const onToolCall = (event) => {
|
|
3661
|
+
// Use a composite key: name + truncated args for dedup
|
|
3662
|
+
const key = event.toolCallId || `${event.name}:${JSON.stringify(event.args || {}).slice(0, 100)}`;
|
|
3663
|
+
if (event.status === 'started') {
|
|
3664
|
+
activeTools.set(key, {
|
|
3665
|
+
name: event.name,
|
|
3666
|
+
args: event.args,
|
|
3667
|
+
startedAt: event.timestamp || new Date().toISOString(),
|
|
3668
|
+
});
|
|
3669
|
+
} else if (event.status === 'completed' || event.status === 'error') {
|
|
3670
|
+
activeTools.delete(key);
|
|
3671
|
+
}
|
|
3672
|
+
};
|
|
3673
|
+
this.events.on('toolCall', onToolCall);
|
|
3674
|
+
|
|
3675
|
+
// Timeout observer: separate LLM call that decides whether to extend.
|
|
3676
|
+
// Runs independently of the main agent loop — works even when blocked by delegates.
|
|
3677
|
+
const runTimeoutObserver = async () => {
|
|
3678
|
+
if (negotiatedTimeoutState.observerRunning) return;
|
|
3679
|
+
negotiatedTimeoutState.observerRunning = true;
|
|
3680
|
+
|
|
3681
|
+
const remainingRequests = negotiatedTimeoutState.maxRequests - negotiatedTimeoutState.extensionsUsed;
|
|
3682
|
+
const remainingBudgetMs = negotiatedTimeoutState.budgetMs - negotiatedTimeoutState.totalExtraTimeMs;
|
|
3683
|
+
const maxPerReqMin = Math.round(negotiatedTimeoutState.maxPerRequestMs / 60000);
|
|
3684
|
+
const elapsedMin = Math.round((Date.now() - negotiatedTimeoutState.startTime) / 60000);
|
|
3685
|
+
|
|
3686
|
+
// Check if extensions/budget exhausted — go straight to graceful wind-down
|
|
3687
|
+
if (remainingRequests <= 0 || remainingBudgetMs <= 0) {
|
|
3688
|
+
if (this.debug) {
|
|
3689
|
+
console.log(`[DEBUG] Timeout observer: no extensions/budget remaining — aborting in-flight tools and triggering graceful wind-down`);
|
|
3690
|
+
}
|
|
3691
|
+
if (this.tracer) {
|
|
3692
|
+
this.tracer.addEvent('negotiated_timeout.observer_exhausted', {
|
|
3693
|
+
extensions_used: negotiatedTimeoutState.extensionsUsed,
|
|
3694
|
+
max_requests: negotiatedTimeoutState.maxRequests,
|
|
3695
|
+
total_extra_time_ms: negotiatedTimeoutState.totalExtraTimeMs,
|
|
3696
|
+
budget_ms: negotiatedTimeoutState.budgetMs,
|
|
3697
|
+
elapsed_min: elapsedMin,
|
|
3698
|
+
active_tools: Array.from(activeTools.values()).map(t => t.name),
|
|
3699
|
+
});
|
|
3700
|
+
}
|
|
3701
|
+
// Two-phase graceful stop: signal subagents/MCP to wind down, hard abort after deadline
|
|
3702
|
+
await this._initiateGracefulStop(gracefulTimeoutState, 'budget/extensions exhausted');
|
|
3703
|
+
negotiatedTimeoutState.observerRunning = false;
|
|
3704
|
+
return;
|
|
3705
|
+
}
|
|
3706
|
+
|
|
3707
|
+
// Build context for the observer
|
|
3708
|
+
const activeToolsList = Array.from(activeTools.values());
|
|
3709
|
+
const now = Date.now();
|
|
3710
|
+
const formatDuration = (ms) => {
|
|
3711
|
+
const totalSec = Math.round(ms / 1000);
|
|
3712
|
+
if (totalSec < 60) return `${totalSec}s`;
|
|
3713
|
+
const min = Math.floor(totalSec / 60);
|
|
3714
|
+
const sec = totalSec % 60;
|
|
3715
|
+
if (min < 60) return `${min}m ${sec}s`;
|
|
3716
|
+
const hr = Math.floor(min / 60);
|
|
3717
|
+
const remainMin = min % 60;
|
|
3718
|
+
return `${hr}h ${remainMin}m`;
|
|
3719
|
+
};
|
|
3720
|
+
const activeToolsDesc = activeToolsList.length > 0
|
|
3721
|
+
? activeToolsList.map(t => {
|
|
3722
|
+
const runningForMs = now - new Date(t.startedAt).getTime();
|
|
3723
|
+
return `- ${t.name}(${JSON.stringify(t.args || {}).slice(0, 200)}) — running for ${formatDuration(runningForMs)}`;
|
|
3724
|
+
}).join('\n')
|
|
3725
|
+
: '(none currently running)';
|
|
3726
|
+
|
|
3727
|
+
// Summarize recent history (last few exchanges, capped)
|
|
3728
|
+
const recentHistory = this.history.slice(-6).map(msg => {
|
|
3729
|
+
const content = typeof msg.content === 'string'
|
|
3730
|
+
? msg.content.slice(0, 300)
|
|
3731
|
+
: JSON.stringify(msg.content).slice(0, 300);
|
|
3732
|
+
return `[${msg.role}]: ${content}`;
|
|
3733
|
+
}).join('\n');
|
|
3734
|
+
|
|
3735
|
+
const observerPrompt = `You are a timeout observer for an AI coding agent. The agent has been working for ${elapsedMin} minute(s) and has reached its time limit.
|
|
3736
|
+
|
|
3737
|
+
## Recent Conversation
|
|
3738
|
+
${recentHistory || '(no history yet)'}
|
|
3739
|
+
|
|
3740
|
+
## Currently Running Tools
|
|
3741
|
+
${activeToolsDesc}
|
|
3742
|
+
|
|
3743
|
+
## Budget
|
|
3744
|
+
- Extensions used: ${negotiatedTimeoutState.extensionsUsed}/${negotiatedTimeoutState.maxRequests}
|
|
3745
|
+
- Time budget remaining: ${Math.round(remainingBudgetMs / 60000)} minutes
|
|
3746
|
+
- Max per extension: ${maxPerReqMin} minutes
|
|
3747
|
+
|
|
3748
|
+
Decide whether the agent should get more time. EXTEND if:
|
|
3749
|
+
- Tools are actively running (especially delegates or complex analysis) — they need time to finish
|
|
3750
|
+
- The agent is making clear progress on a complex task
|
|
3751
|
+
- New information is being gathered that will improve the final answer
|
|
3752
|
+
|
|
3753
|
+
DO NOT EXTEND if:
|
|
3754
|
+
- The agent appears stuck in a loop (repeating the same tool calls or getting the same errors)
|
|
3755
|
+
- The conversation shows the agent retrying failed operations without changing approach
|
|
3756
|
+
- The agent has enough information to answer but keeps searching for more
|
|
3757
|
+
- Tool calls are returning empty or error results repeatedly
|
|
3758
|
+
- The agent is doing redundant work (searching for things it already found)
|
|
3759
|
+
|
|
3760
|
+
A stuck agent will not recover with more time — it will just burn the budget. Better to force it to answer with what it has.
|
|
3761
|
+
|
|
3762
|
+
Respond with ONLY valid JSON (no markdown, no explanation):
|
|
3763
|
+
{"extend": true, "minutes": <1-${maxPerReqMin}>, "reason": "your reason here"}
|
|
3764
|
+
or
|
|
3765
|
+
{"extend": false, "reason": "your reason here"}`;
|
|
3766
|
+
|
|
3767
|
+
const observerFn = async () => {
|
|
3768
|
+
const modelInstance = this.provider ? this.provider(this.model) : this.model;
|
|
3769
|
+
|
|
3770
|
+
if (this.debug) {
|
|
3771
|
+
console.log(`[DEBUG] Timeout observer: making LLM call (${activeToolsList.length} active tools, ${elapsedMin} min elapsed)`);
|
|
3772
|
+
}
|
|
3773
|
+
|
|
3774
|
+
if (this.tracer) {
|
|
3775
|
+
this.tracer.addEvent('negotiated_timeout.observer_invoked', {
|
|
3776
|
+
elapsed_min: elapsedMin,
|
|
3777
|
+
active_tools: activeToolsList.map(t => t.name),
|
|
3778
|
+
active_tools_detail: activeToolsList.map(t => ({
|
|
3779
|
+
name: t.name,
|
|
3780
|
+
running_for_ms: now - new Date(t.startedAt).getTime(),
|
|
3781
|
+
args_preview: JSON.stringify(t.args || {}).slice(0, 100),
|
|
3782
|
+
})),
|
|
3783
|
+
active_tools_count: activeToolsList.length,
|
|
3784
|
+
extensions_used: negotiatedTimeoutState.extensionsUsed,
|
|
3785
|
+
remaining_requests: remainingRequests,
|
|
3786
|
+
remaining_budget_ms: remainingBudgetMs,
|
|
3787
|
+
history_length: this.history.length,
|
|
3788
|
+
});
|
|
3789
|
+
}
|
|
3790
|
+
|
|
3791
|
+
const observerResult = await generateText({
|
|
3792
|
+
model: modelInstance,
|
|
3793
|
+
messages: [{ role: 'user', content: observerPrompt }],
|
|
3794
|
+
maxTokens: 500,
|
|
3795
|
+
});
|
|
3796
|
+
|
|
3797
|
+
const responseText = observerResult.text.trim();
|
|
3798
|
+
|
|
3799
|
+
if (this.tracer) {
|
|
3800
|
+
this.tracer.addEvent('negotiated_timeout.observer_response', {
|
|
3801
|
+
response_text: responseText,
|
|
3802
|
+
usage_prompt_tokens: observerResult.usage?.promptTokens,
|
|
3803
|
+
usage_completion_tokens: observerResult.usage?.completionTokens,
|
|
3804
|
+
});
|
|
3805
|
+
}
|
|
3806
|
+
|
|
3807
|
+
// Parse JSON response — handle potential markdown wrapping
|
|
3808
|
+
const jsonStr = responseText.replace(/^```(?:json)?\s*/, '').replace(/\s*```$/, '');
|
|
3809
|
+
const decision = JSON.parse(jsonStr);
|
|
3810
|
+
|
|
3811
|
+
if (decision.extend && decision.minutes > 0) {
|
|
3812
|
+
const requestedMs = Math.min(decision.minutes, maxPerReqMin) * 60000;
|
|
3813
|
+
const grantedMs = Math.min(requestedMs, remainingBudgetMs, negotiatedTimeoutState.maxPerRequestMs);
|
|
3814
|
+
const grantedMin = Math.round(grantedMs / 60000 * 10) / 10;
|
|
3815
|
+
|
|
3816
|
+
// Update state
|
|
3817
|
+
negotiatedTimeoutState.extensionsUsed++;
|
|
3818
|
+
negotiatedTimeoutState.totalExtraTimeMs += grantedMs;
|
|
3819
|
+
|
|
3820
|
+
// Set message for prepareStep to show when main loop unblocks
|
|
3821
|
+
negotiatedTimeoutState.extensionMessage =
|
|
3822
|
+
`⏰ Time limit was reached. The timeout observer granted ${grantedMin} more minute(s) ` +
|
|
3823
|
+
`(reason: ${decision.reason || 'work in progress'}). ` +
|
|
3824
|
+
`Extensions remaining: ${negotiatedTimeoutState.maxRequests - negotiatedTimeoutState.extensionsUsed}. ` +
|
|
3825
|
+
`Continue your work efficiently.`;
|
|
3826
|
+
|
|
3827
|
+
// Schedule next observer call
|
|
3828
|
+
negotiatedTimeoutState.softTimeoutId = setTimeout(() => {
|
|
3829
|
+
runTimeoutObserver();
|
|
3830
|
+
}, grantedMs);
|
|
3831
|
+
|
|
3832
|
+
if (this.debug) {
|
|
3833
|
+
console.log(`[DEBUG] Timeout observer: granted ${grantedMin} min (reason: ${decision.reason}). Extensions: ${negotiatedTimeoutState.extensionsUsed}/${negotiatedTimeoutState.maxRequests}`);
|
|
3834
|
+
}
|
|
3835
|
+
|
|
3836
|
+
if (this.tracer) {
|
|
3837
|
+
this.tracer.addEvent('negotiated_timeout.observer_extended', {
|
|
3838
|
+
decision_reason: decision.reason,
|
|
3839
|
+
requested_minutes: decision.minutes,
|
|
3840
|
+
granted_ms: grantedMs,
|
|
3841
|
+
granted_min: grantedMin,
|
|
3842
|
+
extensions_used: negotiatedTimeoutState.extensionsUsed,
|
|
3843
|
+
max_requests: negotiatedTimeoutState.maxRequests,
|
|
3844
|
+
total_extra_time_ms: negotiatedTimeoutState.totalExtraTimeMs,
|
|
3845
|
+
budget_remaining_ms: remainingBudgetMs - grantedMs,
|
|
3846
|
+
active_tools: activeToolsList.map(t => t.name),
|
|
3847
|
+
active_tools_count: activeToolsList.length,
|
|
3848
|
+
});
|
|
3849
|
+
}
|
|
3850
|
+
|
|
3851
|
+
// Notify the parent that the agent extended its timeout (#522).
|
|
3852
|
+
// The parent can listen to this event and extend its own deadline
|
|
3853
|
+
// (e.g., adjust Promise.race timeout) instead of killing the agent.
|
|
3854
|
+
this.events.emit('timeout.extended', {
|
|
3855
|
+
grantedMs,
|
|
3856
|
+
reason: decision.reason || 'work in progress',
|
|
3857
|
+
extensionsUsed: negotiatedTimeoutState.extensionsUsed,
|
|
3858
|
+
extensionsRemaining: negotiatedTimeoutState.maxRequests - negotiatedTimeoutState.extensionsUsed,
|
|
3859
|
+
totalExtraTimeMs: negotiatedTimeoutState.totalExtraTimeMs,
|
|
3860
|
+
budgetRemainingMs: remainingBudgetMs - grantedMs,
|
|
3861
|
+
});
|
|
3862
|
+
} else {
|
|
3863
|
+
// Observer decided not to extend — two-phase graceful stop
|
|
3864
|
+
if (this.debug) {
|
|
3865
|
+
console.log(`[DEBUG] Timeout observer: declined extension (reason: ${decision.reason}). Initiating graceful stop.`);
|
|
3866
|
+
}
|
|
3867
|
+
|
|
3868
|
+
if (this.tracer) {
|
|
3869
|
+
this.tracer.addEvent('negotiated_timeout.observer_declined', {
|
|
3870
|
+
decision_reason: decision.reason,
|
|
3871
|
+
extensions_used: negotiatedTimeoutState.extensionsUsed,
|
|
3872
|
+
total_extra_time_ms: negotiatedTimeoutState.totalExtraTimeMs,
|
|
3873
|
+
elapsed_min: elapsedMin,
|
|
3874
|
+
active_tools: activeToolsList.map(t => t.name),
|
|
3875
|
+
});
|
|
3876
|
+
}
|
|
3877
|
+
|
|
3878
|
+
// Notify the parent that the agent is winding down — no more extensions (#522)
|
|
3879
|
+
this.events.emit('timeout.windingDown', {
|
|
3880
|
+
reason: decision.reason || 'observer declined',
|
|
3881
|
+
extensionsUsed: negotiatedTimeoutState.extensionsUsed,
|
|
3882
|
+
totalExtraTimeMs: negotiatedTimeoutState.totalExtraTimeMs,
|
|
3883
|
+
});
|
|
3884
|
+
|
|
3885
|
+
await this._initiateGracefulStop(gracefulTimeoutState, `observer declined: ${decision.reason}`);
|
|
3886
|
+
}
|
|
3887
|
+
};
|
|
3888
|
+
|
|
3889
|
+
try {
|
|
3890
|
+
if (this.tracer) {
|
|
3891
|
+
await this.tracer.withSpan('negotiated_timeout.observer', observerFn, {
|
|
3892
|
+
'timeout.elapsed_min': elapsedMin,
|
|
3893
|
+
'timeout.extensions_used': negotiatedTimeoutState.extensionsUsed,
|
|
3894
|
+
'timeout.active_tools_count': activeToolsList.length,
|
|
3895
|
+
'timeout.remaining_budget_ms': remainingBudgetMs,
|
|
3896
|
+
});
|
|
3897
|
+
} else {
|
|
3898
|
+
await observerFn();
|
|
3899
|
+
}
|
|
3900
|
+
} catch (err) {
|
|
3901
|
+
// Observer call failed — fall back to graceful stop
|
|
3902
|
+
if (this.debug) {
|
|
3903
|
+
console.log(`[DEBUG] Timeout observer: LLM call failed (${err.message}). Initiating graceful stop.`);
|
|
3904
|
+
}
|
|
3905
|
+
|
|
3906
|
+
if (this.tracer) {
|
|
3907
|
+
this.tracer.addEvent('negotiated_timeout.observer_error', {
|
|
3908
|
+
error_message: err.message,
|
|
3909
|
+
error_name: err.name,
|
|
3910
|
+
extensions_used: negotiatedTimeoutState.extensionsUsed,
|
|
3911
|
+
elapsed_min: elapsedMin,
|
|
3912
|
+
});
|
|
3913
|
+
}
|
|
3914
|
+
|
|
3915
|
+
await this._initiateGracefulStop(gracefulTimeoutState, `observer error: ${err.message}`);
|
|
3916
|
+
} finally {
|
|
3917
|
+
negotiatedTimeoutState.observerRunning = false;
|
|
3918
|
+
}
|
|
3919
|
+
};
|
|
3920
|
+
|
|
3921
|
+
// Store observer function on state for testability
|
|
3922
|
+
negotiatedTimeoutState.runObserver = runTimeoutObserver;
|
|
3923
|
+
|
|
3592
3924
|
// Context compaction retry loop
|
|
3593
3925
|
let compactionAttempted = false;
|
|
3594
3926
|
while (true) {
|
|
@@ -3670,6 +4002,17 @@ Follow these instructions carefully:
|
|
|
3670
4002
|
return false;
|
|
3671
4003
|
},
|
|
3672
4004
|
prepareStep: ({ steps, stepNumber }) => {
|
|
4005
|
+
// Negotiated timeout: if the observer granted an extension while the main
|
|
4006
|
+
// loop was blocked (e.g. during a delegate call), inform the AI
|
|
4007
|
+
if (negotiatedTimeoutState.extensionMessage && !gracefulTimeoutState.triggered) {
|
|
4008
|
+
const msg = negotiatedTimeoutState.extensionMessage;
|
|
4009
|
+
negotiatedTimeoutState.extensionMessage = null; // show once
|
|
4010
|
+
if (this.debug) {
|
|
4011
|
+
console.log(`[DEBUG] prepareStep: delivering timeout observer extension message`);
|
|
4012
|
+
}
|
|
4013
|
+
return { userMessage: msg };
|
|
4014
|
+
}
|
|
4015
|
+
|
|
3673
4016
|
// Graceful timeout wind-down: force text-only response with wrap-up reminder
|
|
3674
4017
|
if (gracefulTimeoutState.triggered) {
|
|
3675
4018
|
gracefulTimeoutState.bonusStepsUsed++;
|
|
@@ -3699,10 +4042,19 @@ Follow these instructions carefully:
|
|
|
3699
4042
|
return { toolChoice: 'none' };
|
|
3700
4043
|
}
|
|
3701
4044
|
|
|
3702
|
-
// Last-iteration warning
|
|
4045
|
+
// Last-iteration warning — force text-only and tell the AI to summarize
|
|
3703
4046
|
if (stepNumber === maxIterations - 1) {
|
|
4047
|
+
// Build a brief summary of tools used so the model can reference them in its answer
|
|
4048
|
+
const searchesTried = _toolCallLog
|
|
4049
|
+
.filter(tc => tc.name === 'search')
|
|
4050
|
+
.map(tc => `"${tc.args.query || ''}"${tc.args.exact ? ' (exact)' : ''}`)
|
|
4051
|
+
.filter((v, i, a) => a.indexOf(v) === i); // unique
|
|
4052
|
+
const searchSummary = searchesTried.length > 0
|
|
4053
|
+
? `\nSearches attempted: ${searchesTried.join(', ')}`
|
|
4054
|
+
: '';
|
|
3704
4055
|
return {
|
|
3705
4056
|
toolChoice: 'none',
|
|
4057
|
+
userMessage: `⚠️ LAST ITERATION — you are out of tool calls. Provide your BEST answer NOW with the information gathered so far. If you could not find what was requested, explain exactly what you searched for and why it did not work, so the caller can try a different approach.${searchSummary}`
|
|
3706
4058
|
};
|
|
3707
4059
|
}
|
|
3708
4060
|
|
|
@@ -3797,6 +4149,13 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3797
4149
|
currentIteration++;
|
|
3798
4150
|
toolContext.currentIteration = currentIteration;
|
|
3799
4151
|
|
|
4152
|
+
// Track tool calls for failure diagnostics
|
|
4153
|
+
if (toolCalls?.length > 0) {
|
|
4154
|
+
for (const tc of toolCalls) {
|
|
4155
|
+
_toolCallLog.push({ name: tc.toolName, args: tc.args || {} });
|
|
4156
|
+
}
|
|
4157
|
+
}
|
|
4158
|
+
|
|
3800
4159
|
// Record telemetry — include model's reasoning and tool call details
|
|
3801
4160
|
if (this.tracer) {
|
|
3802
4161
|
const stepEvent = {
|
|
@@ -3892,7 +4251,7 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3892
4251
|
const executeAIRequest = async () => {
|
|
3893
4252
|
const result = await this.streamTextWithRetryAndFallback(streamOptions);
|
|
3894
4253
|
|
|
3895
|
-
// Set up
|
|
4254
|
+
// Set up timeout timer now that streamText is running.
|
|
3896
4255
|
// streamText() returns immediately — the actual tool loop runs asynchronously
|
|
3897
4256
|
// and completes when we await result.steps/result.text below.
|
|
3898
4257
|
let gracefulTimeoutId = null;
|
|
@@ -3915,6 +4274,16 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3915
4274
|
}, this.maxOperationTimeout);
|
|
3916
4275
|
}
|
|
3917
4276
|
|
|
4277
|
+
// Negotiated timeout: run the timeout observer (separate LLM call)
|
|
4278
|
+
if (this.timeoutBehavior === 'negotiated' && this.maxOperationTimeout > 0) {
|
|
4279
|
+
negotiatedTimeoutState.softTimeoutId = setTimeout(() => {
|
|
4280
|
+
if (this.debug) {
|
|
4281
|
+
console.log(`[DEBUG] Soft timeout after ${this.maxOperationTimeout}ms — invoking timeout observer`);
|
|
4282
|
+
}
|
|
4283
|
+
runTimeoutObserver();
|
|
4284
|
+
}, this.maxOperationTimeout);
|
|
4285
|
+
}
|
|
4286
|
+
|
|
3918
4287
|
try {
|
|
3919
4288
|
// Use only the last step's text as the final answer.
|
|
3920
4289
|
// result.text concatenates ALL steps (including intermediate planning text),
|
|
@@ -3944,6 +4313,15 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3944
4313
|
// Clean up graceful timeout timers
|
|
3945
4314
|
if (gracefulTimeoutId) clearTimeout(gracefulTimeoutId);
|
|
3946
4315
|
if (hardAbortTimeoutId) clearTimeout(hardAbortTimeoutId);
|
|
4316
|
+
// Clean up negotiated timeout timer
|
|
4317
|
+
if (negotiatedTimeoutState.softTimeoutId) clearTimeout(negotiatedTimeoutState.softTimeoutId);
|
|
4318
|
+
// Clean up graceful stop hard abort timer
|
|
4319
|
+
if (this._gracefulStopHardAbortId) {
|
|
4320
|
+
clearTimeout(this._gracefulStopHardAbortId);
|
|
4321
|
+
this._gracefulStopHardAbortId = null;
|
|
4322
|
+
}
|
|
4323
|
+
// Remove in-flight tool tracker
|
|
4324
|
+
this.events.removeListener('toolCall', onToolCall);
|
|
3947
4325
|
}
|
|
3948
4326
|
};
|
|
3949
4327
|
|
|
@@ -3994,7 +4372,7 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3994
4372
|
if (gracefulTimeoutState.triggered) {
|
|
3995
4373
|
const timeoutNotice = '**Note: This response was generated under a time constraint. The research may be incomplete, and some planned searches or analysis steps were not completed.**\n\n';
|
|
3996
4374
|
|
|
3997
|
-
if (!finalResult || finalResult === 'I was unable to complete your request
|
|
4375
|
+
if (!finalResult || finalResult === DEFAULT_MAX_ITER_MSG || finalResult.startsWith('I was unable to complete your request after')) {
|
|
3998
4376
|
// Wind-down produced empty text — try to collect useful content.
|
|
3999
4377
|
// Some models (e.g., Gemini) return finishReason:'other' with empty text
|
|
4000
4378
|
// when forced from tool-calling to text-only mode mid-task.
|
|
@@ -4056,7 +4434,7 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
4056
4434
|
// If the model answered without tool calls (or its final step had none),
|
|
4057
4435
|
// stopWhen never gets a chance to force continuation. In that case, run
|
|
4058
4436
|
// a second streamText pass with the completion prompt injected.
|
|
4059
|
-
if (this.completionPrompt && !options._completionPromptProcessed && !completionPromptInjected && finalResult) {
|
|
4437
|
+
if (this.completionPrompt && !options._completionPromptProcessed && !completionPromptInjected && !abortSummaryTaken && finalResult) {
|
|
4060
4438
|
completionPromptInjected = true;
|
|
4061
4439
|
preCompletionResult = finalResult;
|
|
4062
4440
|
|
|
@@ -4142,6 +4520,146 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
4142
4520
|
break; // Success
|
|
4143
4521
|
|
|
4144
4522
|
} catch (error) {
|
|
4523
|
+
// Negotiated timeout observer aborted in-flight tools to trigger wind-down.
|
|
4524
|
+
// Give the AI a dedicated summary call with full conversation context so it
|
|
4525
|
+
// can explain what it accomplished and what remains incomplete.
|
|
4526
|
+
if (gracefulTimeoutState.triggered && error?.name === 'AbortError') {
|
|
4527
|
+
if (this.debug) {
|
|
4528
|
+
console.log(`[DEBUG] Negotiated timeout: abort caught — making summary LLM call with conversation context`);
|
|
4529
|
+
}
|
|
4530
|
+
|
|
4531
|
+
if (this.tracer) {
|
|
4532
|
+
this.tracer.addEvent('negotiated_timeout.abort_summary_started', {
|
|
4533
|
+
conversation_messages: currentMessages.length,
|
|
4534
|
+
has_schema: !!options.schema,
|
|
4535
|
+
has_tasks: !!(this.enableTasks && this.taskManager),
|
|
4536
|
+
});
|
|
4537
|
+
}
|
|
4538
|
+
|
|
4539
|
+
try {
|
|
4540
|
+
// Build task status context if tasks are active
|
|
4541
|
+
let taskContext = '';
|
|
4542
|
+
if (this.enableTasks && this.taskManager) {
|
|
4543
|
+
const taskSummary = this.taskManager.getTaskSummary?.();
|
|
4544
|
+
if (taskSummary) {
|
|
4545
|
+
taskContext = `\n\n## Task Status\n${taskSummary}\n\nAcknowledge which tasks were completed and which were not.`;
|
|
4546
|
+
}
|
|
4547
|
+
}
|
|
4548
|
+
|
|
4549
|
+
// Build schema instructions if a schema is required
|
|
4550
|
+
let schemaContext = '';
|
|
4551
|
+
if (options.schema) {
|
|
4552
|
+
try {
|
|
4553
|
+
const parsedSchema = typeof options.schema === 'string' ? JSON.parse(options.schema) : options.schema;
|
|
4554
|
+
schemaContext = `\n\nIMPORTANT: Your response MUST be valid JSON matching this schema:\n${JSON.stringify(parsedSchema, null, 2)}\n\n` +
|
|
4555
|
+
`Respond with ONLY valid JSON — no markdown, no explanation, no text outside the JSON object. ` +
|
|
4556
|
+
`Include all findings and partial results within the JSON structure. ` +
|
|
4557
|
+
`If fields cannot be fully populated due to the interruption, use partial data or null values as appropriate.`;
|
|
4558
|
+
} catch {}
|
|
4559
|
+
}
|
|
4560
|
+
|
|
4561
|
+
const summaryPrompt = `Your operation was interrupted by a timeout observer because the time limit was reached. ` +
|
|
4562
|
+
`Some of your tool calls were cancelled mid-execution.\n\n` +
|
|
4563
|
+
`Please provide a DETAILED summary of:\n` +
|
|
4564
|
+
`1. What you were asked to do (the original task)\n` +
|
|
4565
|
+
`2. What you accomplished — include ALL findings, code snippets, data, and conclusions you gathered\n` +
|
|
4566
|
+
`3. What was still in progress or not yet started\n` +
|
|
4567
|
+
`4. Any partial results or recommendations you can offer based on what you found so far` +
|
|
4568
|
+
`${taskContext}${schemaContext}\n\n` +
|
|
4569
|
+
`Be thorough — this is the user's only response. Include all useful information you collected.`;
|
|
4570
|
+
|
|
4571
|
+
const summaryMessages = [
|
|
4572
|
+
...currentMessages,
|
|
4573
|
+
{ role: 'user', content: summaryPrompt },
|
|
4574
|
+
];
|
|
4575
|
+
|
|
4576
|
+
const modelInstance = this.provider ? this.provider(this.model) : this.model;
|
|
4577
|
+
|
|
4578
|
+
const summaryFn = async () => {
|
|
4579
|
+
const summaryResult = await generateText({
|
|
4580
|
+
model: modelInstance,
|
|
4581
|
+
messages: this.prepareMessagesWithImages(summaryMessages),
|
|
4582
|
+
maxTokens: 4000,
|
|
4583
|
+
});
|
|
4584
|
+
|
|
4585
|
+
if (this.tracer) {
|
|
4586
|
+
this.tracer.addEvent('negotiated_timeout.abort_summary_completed', {
|
|
4587
|
+
summary_length: summaryResult.text?.length || 0,
|
|
4588
|
+
usage_prompt_tokens: summaryResult.usage?.promptTokens,
|
|
4589
|
+
usage_completion_tokens: summaryResult.usage?.completionTokens,
|
|
4590
|
+
});
|
|
4591
|
+
}
|
|
4592
|
+
|
|
4593
|
+
// Record token usage for the summary call
|
|
4594
|
+
if (summaryResult.usage) {
|
|
4595
|
+
this.tokenCounter.recordUsage(summaryResult.usage);
|
|
4596
|
+
}
|
|
4597
|
+
|
|
4598
|
+
return summaryResult.text;
|
|
4599
|
+
};
|
|
4600
|
+
|
|
4601
|
+
let summaryText;
|
|
4602
|
+
if (this.tracer) {
|
|
4603
|
+
summaryText = await this.tracer.withSpan('negotiated_timeout.abort_summary', summaryFn, {
|
|
4604
|
+
'summary.conversation_messages': currentMessages.length,
|
|
4605
|
+
});
|
|
4606
|
+
} else {
|
|
4607
|
+
summaryText = await summaryFn();
|
|
4608
|
+
}
|
|
4609
|
+
|
|
4610
|
+
if (options.schema) {
|
|
4611
|
+
// Schema mode: use the summary text as-is (it should already be JSON).
|
|
4612
|
+
// Don't prepend a notice — it would break the JSON structure.
|
|
4613
|
+
// The schema validation pipeline downstream will validate/fix it.
|
|
4614
|
+
finalResult = summaryText || '{}';
|
|
4615
|
+
} else {
|
|
4616
|
+
const timeoutNotice = '**Note: This response was generated under a time constraint. The timeout observer interrupted the operation because the time budget was exhausted.**\n\n';
|
|
4617
|
+
finalResult = timeoutNotice + (summaryText || 'The operation was interrupted before a response could be generated.');
|
|
4618
|
+
}
|
|
4619
|
+
|
|
4620
|
+
// Stream the abort summary to onStream callback so callers see the output
|
|
4621
|
+
if (options.onStream && finalResult) {
|
|
4622
|
+
options.onStream(finalResult);
|
|
4623
|
+
}
|
|
4624
|
+
|
|
4625
|
+
if (this.debug) {
|
|
4626
|
+
console.log(`[DEBUG] Negotiated timeout: summary produced ${summaryText?.length || 0} chars`);
|
|
4627
|
+
}
|
|
4628
|
+
} catch (summaryErr) {
|
|
4629
|
+
if (this.debug) {
|
|
4630
|
+
console.log(`[DEBUG] Negotiated timeout: summary call failed (${summaryErr.message}), falling back to partial text`);
|
|
4631
|
+
}
|
|
4632
|
+
if (this.tracer) {
|
|
4633
|
+
this.tracer.addEvent('negotiated_timeout.abort_summary_error', {
|
|
4634
|
+
error_message: summaryErr.message,
|
|
4635
|
+
});
|
|
4636
|
+
}
|
|
4637
|
+
|
|
4638
|
+
// Fallback: collect whatever text is in conversation history
|
|
4639
|
+
const partialTexts = currentMessages
|
|
4640
|
+
.filter(m => m.role === 'assistant' && typeof m.content === 'string' && m.content.trim())
|
|
4641
|
+
.map(m => m.content);
|
|
4642
|
+
|
|
4643
|
+
if (options.schema) {
|
|
4644
|
+
// Schema mode: try to pass through the last assistant message (may contain JSON)
|
|
4645
|
+
finalResult = partialTexts.length > 0 ? partialTexts[partialTexts.length - 1] : '{}';
|
|
4646
|
+
} else {
|
|
4647
|
+
const timeoutNotice = '**Note: This response was generated under a time constraint. The operation was interrupted and some work was not completed.**\n\n';
|
|
4648
|
+
finalResult = partialTexts.length > 0
|
|
4649
|
+
? timeoutNotice + partialTexts[partialTexts.length - 1]
|
|
4650
|
+
: timeoutNotice + 'The operation was interrupted before enough information could be gathered. Please try again with a simpler query or increase the timeout.';
|
|
4651
|
+
}
|
|
4652
|
+
|
|
4653
|
+
// Stream the fallback result
|
|
4654
|
+
if (options.onStream && finalResult) {
|
|
4655
|
+
options.onStream(finalResult);
|
|
4656
|
+
}
|
|
4657
|
+
}
|
|
4658
|
+
|
|
4659
|
+
abortSummaryTaken = true;
|
|
4660
|
+
break; // Exit the compaction retry loop with the summary
|
|
4661
|
+
}
|
|
4662
|
+
|
|
4145
4663
|
// Handle context-limit error: compact messages and retry (once)
|
|
4146
4664
|
if (!compactionAttempted && handleContextLimitError) {
|
|
4147
4665
|
const compactionResult = handleContextLimitError(error, currentMessages, {
|
|
@@ -4185,6 +4703,37 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
4185
4703
|
|
|
4186
4704
|
if (currentIteration >= maxIterations) {
|
|
4187
4705
|
console.warn(`[WARN] Max tool iterations (${maxIterations}) reached for session ${this.sessionId}.`);
|
|
4706
|
+
|
|
4707
|
+
// Build a descriptive failure message with a summary of tool calls made,
|
|
4708
|
+
// so the caller (e.g. a parent agent) knows what was attempted and why it failed.
|
|
4709
|
+
if (!finalResult || finalResult === DEFAULT_MAX_ITER_MSG) {
|
|
4710
|
+
try {
|
|
4711
|
+
const searchQueries = [];
|
|
4712
|
+
const toolCounts = {};
|
|
4713
|
+
for (const tc of _toolCallLog) {
|
|
4714
|
+
toolCounts[tc.name] = (toolCounts[tc.name] || 0) + 1;
|
|
4715
|
+
if (tc.name === 'search') {
|
|
4716
|
+
const q = tc.args.query || '';
|
|
4717
|
+
const exact = tc.args.exact ? ' (exact)' : '';
|
|
4718
|
+
searchQueries.push(`"${q}"${exact}`);
|
|
4719
|
+
}
|
|
4720
|
+
}
|
|
4721
|
+
const toolBreakdown = Object.entries(toolCounts)
|
|
4722
|
+
.map(([name, count]) => `${name}: ${count}x`)
|
|
4723
|
+
.join(', ');
|
|
4724
|
+
const uniqueSearches = [...new Set(searchQueries)];
|
|
4725
|
+
|
|
4726
|
+
let summary = `I was unable to complete your request after ${currentIteration} tool iterations.\n\n`;
|
|
4727
|
+
summary += `Tool calls made: ${toolBreakdown || 'none'}\n`;
|
|
4728
|
+
if (uniqueSearches.length > 0) {
|
|
4729
|
+
summary += `Search queries tried: ${uniqueSearches.join(', ')}\n`;
|
|
4730
|
+
}
|
|
4731
|
+
summary += `\nThe search approach may be fundamentally wrong for this query. Consider: using exact=true for literal string matching, using bash/grep for pattern-based file searches, or trying a completely different strategy instead of repeating similar searches.`;
|
|
4732
|
+
finalResult = summary;
|
|
4733
|
+
} catch {
|
|
4734
|
+
finalResult = DEFAULT_MAX_ITER_MSG;
|
|
4735
|
+
}
|
|
4736
|
+
}
|
|
4188
4737
|
}
|
|
4189
4738
|
|
|
4190
4739
|
// Store final history
|
|
@@ -4888,6 +5437,149 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
4888
5437
|
}
|
|
4889
5438
|
}
|
|
4890
5439
|
|
|
5440
|
+
/**
|
|
5441
|
+
* Trigger graceful wind-down from outside (e.g., parent agent).
|
|
5442
|
+
* Unlike cancel(), this does NOT abort — it sets the graceful timeout flag
|
|
5443
|
+
* so the agent finishes its current step and then winds down naturally.
|
|
5444
|
+
*/
|
|
5445
|
+
triggerGracefulWindDown() {
|
|
5446
|
+
if (this._gracefulTimeoutState && !this._gracefulTimeoutState.triggered) {
|
|
5447
|
+
this._gracefulTimeoutState.triggered = true;
|
|
5448
|
+
if (this.debug) {
|
|
5449
|
+
console.log(`[DEBUG] Graceful wind-down triggered externally for session ${this.sessionId}`);
|
|
5450
|
+
}
|
|
5451
|
+
if (this.tracer) {
|
|
5452
|
+
this.tracer.addEvent('graceful_stop.external_trigger', {
|
|
5453
|
+
'session.id': this.sessionId,
|
|
5454
|
+
});
|
|
5455
|
+
}
|
|
5456
|
+
} else if (this.debug) {
|
|
5457
|
+
console.log(`[DEBUG] Graceful wind-down already active for session ${this.sessionId}, skipping`);
|
|
5458
|
+
}
|
|
5459
|
+
}
|
|
5460
|
+
|
|
5461
|
+
/**
|
|
5462
|
+
* Initiate two-phase graceful stop: signal subagents and MCP servers to wind down,
|
|
5463
|
+
* then hard-abort after a deadline if they haven't finished.
|
|
5464
|
+
* @param {Object} gracefulTimeoutState - The graceful timeout state object from run()
|
|
5465
|
+
* @param {string} reason - Why the graceful stop was initiated
|
|
5466
|
+
*/
|
|
5467
|
+
async _initiateGracefulStop(gracefulTimeoutState, reason) {
|
|
5468
|
+
if (gracefulTimeoutState.triggered) return; // Already initiated
|
|
5469
|
+
|
|
5470
|
+
if (this.debug) {
|
|
5471
|
+
console.log(`[DEBUG] Initiating graceful stop: ${reason} (subagents: ${this._activeSubagents.size}, hasMcpBridge: ${!!this.mcpBridge}, deadline: ${this.gracefulStopDeadline}ms)`);
|
|
5472
|
+
}
|
|
5473
|
+
|
|
5474
|
+
// Mark graceful timeout — prepareStep will pick this up for the parent's wind-down
|
|
5475
|
+
gracefulTimeoutState.triggered = true;
|
|
5476
|
+
|
|
5477
|
+
if (this.tracer) {
|
|
5478
|
+
this.tracer.addEvent('graceful_stop.initiated', {
|
|
5479
|
+
'session.id': this.sessionId,
|
|
5480
|
+
'graceful_stop.reason': reason,
|
|
5481
|
+
'graceful_stop.active_subagents': this._activeSubagents.size,
|
|
5482
|
+
'graceful_stop.has_mcp_bridge': !!this.mcpBridge,
|
|
5483
|
+
'graceful_stop.deadline_ms': this.gracefulStopDeadline,
|
|
5484
|
+
});
|
|
5485
|
+
}
|
|
5486
|
+
|
|
5487
|
+
// Signal all active subagents to wind down gracefully (not hard-cancel)
|
|
5488
|
+
let subagentsSignalled = 0;
|
|
5489
|
+
let subagentErrors = 0;
|
|
5490
|
+
for (const [sid, subagent] of this._activeSubagents) {
|
|
5491
|
+
try {
|
|
5492
|
+
subagent.triggerGracefulWindDown();
|
|
5493
|
+
subagentsSignalled++;
|
|
5494
|
+
if (this.debug) {
|
|
5495
|
+
console.log(`[DEBUG] Triggered graceful wind-down on subagent ${sid}`);
|
|
5496
|
+
}
|
|
5497
|
+
} catch (e) {
|
|
5498
|
+
subagentErrors++;
|
|
5499
|
+
if (this.debug) {
|
|
5500
|
+
console.log(`[DEBUG] Failed to trigger wind-down on subagent ${sid}: ${e.message}`);
|
|
5501
|
+
}
|
|
5502
|
+
}
|
|
5503
|
+
}
|
|
5504
|
+
|
|
5505
|
+
// Call graceful_stop on MCP servers that expose it (fire-and-forget with short timeout)
|
|
5506
|
+
let mcpResults = [];
|
|
5507
|
+
if (this.mcpBridge) {
|
|
5508
|
+
try {
|
|
5509
|
+
mcpResults = await this.mcpBridge.callGracefulStopAll();
|
|
5510
|
+
if (this.debug && mcpResults.length > 0) {
|
|
5511
|
+
console.log(`[DEBUG] MCP graceful_stop results: ${JSON.stringify(mcpResults)}`);
|
|
5512
|
+
}
|
|
5513
|
+
} catch (e) {
|
|
5514
|
+
if (this.debug) {
|
|
5515
|
+
console.log(`[DEBUG] MCP graceful_stop failed: ${e.message}`);
|
|
5516
|
+
}
|
|
5517
|
+
}
|
|
5518
|
+
}
|
|
5519
|
+
|
|
5520
|
+
if (this.tracer) {
|
|
5521
|
+
this.tracer.addEvent('graceful_stop.signals_sent', {
|
|
5522
|
+
'session.id': this.sessionId,
|
|
5523
|
+
'graceful_stop.subagents_signalled': subagentsSignalled,
|
|
5524
|
+
'graceful_stop.subagent_errors': subagentErrors,
|
|
5525
|
+
'graceful_stop.mcp_servers_called': mcpResults.filter(r => r.success).length,
|
|
5526
|
+
'graceful_stop.mcp_servers_failed': mcpResults.filter(r => !r.success).length,
|
|
5527
|
+
'graceful_stop.mcp_servers_total': mcpResults.length,
|
|
5528
|
+
});
|
|
5529
|
+
}
|
|
5530
|
+
|
|
5531
|
+
// Safety net: hard abort after deadline if tools haven't finished
|
|
5532
|
+
this._gracefulStopHardAbortId = setTimeout(() => {
|
|
5533
|
+
if (this.debug) {
|
|
5534
|
+
console.log(`[DEBUG] Graceful stop deadline (${this.gracefulStopDeadline}ms) expired — hard aborting`);
|
|
5535
|
+
}
|
|
5536
|
+
if (this.tracer) {
|
|
5537
|
+
this.tracer.addEvent('graceful_stop.deadline_expired', {
|
|
5538
|
+
'session.id': this.sessionId,
|
|
5539
|
+
'graceful_stop.deadline_ms': this.gracefulStopDeadline,
|
|
5540
|
+
});
|
|
5541
|
+
}
|
|
5542
|
+
if (this._abortController) this._abortController.abort();
|
|
5543
|
+
}, this.gracefulStopDeadline);
|
|
5544
|
+
}
|
|
5545
|
+
|
|
5546
|
+
/**
|
|
5547
|
+
* Register an active subagent for graceful stop coordination.
|
|
5548
|
+
* @param {string} sessionId
|
|
5549
|
+
* @param {ProbeAgent} subagent
|
|
5550
|
+
*/
|
|
5551
|
+
_registerSubagent(sessionId, subagent) {
|
|
5552
|
+
this._activeSubagents.set(sessionId, subagent);
|
|
5553
|
+
if (this.debug) {
|
|
5554
|
+
console.log(`[DEBUG] Registered subagent ${sessionId} (active: ${this._activeSubagents.size})`);
|
|
5555
|
+
}
|
|
5556
|
+
if (this.tracer) {
|
|
5557
|
+
this.tracer.addEvent('subagent.registered', {
|
|
5558
|
+
'session.id': this.sessionId,
|
|
5559
|
+
'subagent.session_id': sessionId,
|
|
5560
|
+
'subagent.active_count': this._activeSubagents.size,
|
|
5561
|
+
});
|
|
5562
|
+
}
|
|
5563
|
+
}
|
|
5564
|
+
|
|
5565
|
+
/**
|
|
5566
|
+
* Unregister a completed subagent.
|
|
5567
|
+
* @param {string} sessionId
|
|
5568
|
+
*/
|
|
5569
|
+
_unregisterSubagent(sessionId) {
|
|
5570
|
+
this._activeSubagents.delete(sessionId);
|
|
5571
|
+
if (this.debug) {
|
|
5572
|
+
console.log(`[DEBUG] Unregistered subagent ${sessionId} (active: ${this._activeSubagents.size})`);
|
|
5573
|
+
}
|
|
5574
|
+
if (this.tracer) {
|
|
5575
|
+
this.tracer.addEvent('subagent.unregistered', {
|
|
5576
|
+
'session.id': this.sessionId,
|
|
5577
|
+
'subagent.session_id': sessionId,
|
|
5578
|
+
'subagent.active_count': this._activeSubagents.size,
|
|
5579
|
+
});
|
|
5580
|
+
}
|
|
5581
|
+
}
|
|
5582
|
+
|
|
4891
5583
|
/**
|
|
4892
5584
|
* Get the abort signal for this agent.
|
|
4893
5585
|
* Delegations and subagents should check this signal.
|