@probelabs/probe 0.6.0-rc294 → 0.6.0-rc295
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/binaries/{probe-v0.6.0-rc294-aarch64-apple-darwin.tar.gz → probe-v0.6.0-rc295-aarch64-apple-darwin.tar.gz} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc294-aarch64-unknown-linux-musl.tar.gz → probe-v0.6.0-rc295-aarch64-unknown-linux-musl.tar.gz} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc294-x86_64-apple-darwin.tar.gz → probe-v0.6.0-rc295-x86_64-apple-darwin.tar.gz} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc294-x86_64-pc-windows-msvc.zip → probe-v0.6.0-rc295-x86_64-pc-windows-msvc.zip} +0 -0
- package/bin/binaries/{probe-v0.6.0-rc294-x86_64-unknown-linux-musl.tar.gz → probe-v0.6.0-rc295-x86_64-unknown-linux-musl.tar.gz} +0 -0
- package/build/agent/ProbeAgent.d.ts +4 -0
- package/build/agent/ProbeAgent.js +192 -26
- package/cjs/agent/ProbeAgent.cjs +150 -20
- package/cjs/index.cjs +150 -20
- package/package.json +1 -1
- package/src/agent/ProbeAgent.d.ts +4 -0
- package/src/agent/ProbeAgent.js +192 -26
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -106,6 +106,10 @@ export interface ProbeAgentOptions {
|
|
|
106
106
|
requestTimeout?: number;
|
|
107
107
|
/** Maximum timeout in ms for the entire operation including all retries and fallbacks (default: 300000 or MAX_OPERATION_TIMEOUT env var). This is the absolute maximum time for streamTextWithRetryAndFallback. */
|
|
108
108
|
maxOperationTimeout?: number;
|
|
109
|
+
/** Timeout behavior: 'graceful' winds down with bonus steps giving the agent a chance to respond, 'hard' aborts immediately (default: 'graceful'). Env var: TIMEOUT_BEHAVIOR */
|
|
110
|
+
timeoutBehavior?: 'graceful' | 'hard';
|
|
111
|
+
/** Number of bonus steps during graceful timeout wind-down (default: 4, range: 1-20). Env var: GRACEFUL_TIMEOUT_BONUS_STEPS */
|
|
112
|
+
gracefulTimeoutBonusSteps?: number;
|
|
109
113
|
}
|
|
110
114
|
|
|
111
115
|
/**
|
|
@@ -391,6 +391,23 @@ export class ProbeAgent {
|
|
|
391
391
|
console.log(`[DEBUG] Max operation timeout: ${this.maxOperationTimeout}ms`);
|
|
392
392
|
}
|
|
393
393
|
|
|
394
|
+
// Timeout behavior: 'graceful' (default) winds down with bonus steps, 'hard' aborts immediately
|
|
395
|
+
this.timeoutBehavior = options.timeoutBehavior ?? (() => {
|
|
396
|
+
const val = process.env.TIMEOUT_BEHAVIOR;
|
|
397
|
+
if (val === 'hard') return 'hard';
|
|
398
|
+
return 'graceful';
|
|
399
|
+
})();
|
|
400
|
+
|
|
401
|
+
// Number of bonus steps during graceful timeout wind-down (default 4)
|
|
402
|
+
this.gracefulTimeoutBonusSteps = options.gracefulTimeoutBonusSteps ?? (() => {
|
|
403
|
+
const parsed = parseInt(process.env.GRACEFUL_TIMEOUT_BONUS_STEPS, 10);
|
|
404
|
+
return (isNaN(parsed) || parsed < 1 || parsed > 20) ? 4 : parsed;
|
|
405
|
+
})();
|
|
406
|
+
|
|
407
|
+
if (this.debug) {
|
|
408
|
+
console.log(`[DEBUG] Timeout behavior: ${this.timeoutBehavior}, bonus steps: ${this.gracefulTimeoutBonusSteps}`);
|
|
409
|
+
}
|
|
410
|
+
|
|
394
411
|
// Retry configuration
|
|
395
412
|
this.retryConfig = options.retry || {};
|
|
396
413
|
this.retryManager = null; // Will be initialized lazily when needed
|
|
@@ -1554,13 +1571,24 @@ export class ProbeAgent {
|
|
|
1554
1571
|
}
|
|
1555
1572
|
|
|
1556
1573
|
// Set up overall operation timeout (default 5 minutes)
|
|
1574
|
+
// NOTE: For Vercel AI SDK paths, streamText() returns immediately and the
|
|
1575
|
+
// actual tool loop runs asynchronously. The graceful timeout timer is set up
|
|
1576
|
+
// in the run() method where results are actually awaited, not here.
|
|
1577
|
+
// This timer only handles the hard abort for non-graceful mode and engine paths.
|
|
1557
1578
|
if (this.maxOperationTimeout && this.maxOperationTimeout > 0) {
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1579
|
+
const gts = this._gracefulTimeoutState;
|
|
1580
|
+
if (this.timeoutBehavior === 'graceful' && gts) {
|
|
1581
|
+
// Graceful mode: timer is managed in run() method.
|
|
1582
|
+
// Only set up the AbortController link (no timer here).
|
|
1583
|
+
} else {
|
|
1584
|
+
// Hard mode: immediate abort (legacy behavior)
|
|
1585
|
+
timeoutState.timeoutId = setTimeout(() => {
|
|
1586
|
+
controller.abort();
|
|
1587
|
+
if (this.debug) {
|
|
1588
|
+
console.log(`[DEBUG] Operation timed out after ${this.maxOperationTimeout}ms (max operation timeout)`);
|
|
1589
|
+
}
|
|
1590
|
+
}, this.maxOperationTimeout);
|
|
1591
|
+
}
|
|
1564
1592
|
}
|
|
1565
1593
|
|
|
1566
1594
|
try {
|
|
@@ -3552,6 +3580,15 @@ Follow these instructions carefully:
|
|
|
3552
3580
|
let completionPromptInjected = false;
|
|
3553
3581
|
let preCompletionResult = null; // Stores the result before completionPrompt for fallback
|
|
3554
3582
|
|
|
3583
|
+
// Graceful timeout state — shared between setTimeout (in streamTextWithRetryAndFallback)
|
|
3584
|
+
// and prepareStep/stopWhen callbacks (in streamText loop)
|
|
3585
|
+
const gracefulTimeoutState = {
|
|
3586
|
+
triggered: false, // Set to true when soft timeout fires
|
|
3587
|
+
bonusStepsUsed: 0, // Steps taken after soft timeout
|
|
3588
|
+
bonusStepsMax: this.gracefulTimeoutBonusSteps
|
|
3589
|
+
};
|
|
3590
|
+
this._gracefulTimeoutState = gracefulTimeoutState;
|
|
3591
|
+
|
|
3555
3592
|
// Context compaction retry loop
|
|
3556
3593
|
let compactionAttempted = false;
|
|
3557
3594
|
while (true) {
|
|
@@ -3563,6 +3600,17 @@ Follow these instructions carefully:
|
|
|
3563
3600
|
messages: messagesForAI,
|
|
3564
3601
|
tools,
|
|
3565
3602
|
stopWhen: ({ steps }) => {
|
|
3603
|
+
// Graceful timeout wind-down: override normal limits, stop only when bonus steps exhausted
|
|
3604
|
+
if (gracefulTimeoutState.triggered) {
|
|
3605
|
+
if (gracefulTimeoutState.bonusStepsUsed >= gracefulTimeoutState.bonusStepsMax) {
|
|
3606
|
+
if (this.debug) {
|
|
3607
|
+
console.log(`[DEBUG] stopWhen: graceful timeout bonus steps exhausted (${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax}), forcing stop`);
|
|
3608
|
+
}
|
|
3609
|
+
return true;
|
|
3610
|
+
}
|
|
3611
|
+
return false; // Allow more bonus steps
|
|
3612
|
+
}
|
|
3613
|
+
|
|
3566
3614
|
// Hard limit
|
|
3567
3615
|
if (steps.length >= maxIterations) return true;
|
|
3568
3616
|
|
|
@@ -3622,6 +3670,35 @@ Follow these instructions carefully:
|
|
|
3622
3670
|
return false;
|
|
3623
3671
|
},
|
|
3624
3672
|
prepareStep: ({ steps, stepNumber }) => {
|
|
3673
|
+
// Graceful timeout wind-down: force text-only response with wrap-up reminder
|
|
3674
|
+
if (gracefulTimeoutState.triggered) {
|
|
3675
|
+
gracefulTimeoutState.bonusStepsUsed++;
|
|
3676
|
+
const remaining = gracefulTimeoutState.bonusStepsMax - gracefulTimeoutState.bonusStepsUsed;
|
|
3677
|
+
|
|
3678
|
+
if (gracefulTimeoutState.bonusStepsUsed === 1) {
|
|
3679
|
+
// First wind-down step: inject wrap-up message
|
|
3680
|
+
if (this.debug) {
|
|
3681
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step 1/${gracefulTimeoutState.bonusStepsMax}`);
|
|
3682
|
+
}
|
|
3683
|
+
if (this.tracer) {
|
|
3684
|
+
this.tracer.addEvent('graceful_timeout.wind_down_started', {
|
|
3685
|
+
bonus_steps_max: gracefulTimeoutState.bonusStepsMax,
|
|
3686
|
+
current_iteration: currentIteration,
|
|
3687
|
+
max_iterations: maxIterations
|
|
3688
|
+
});
|
|
3689
|
+
}
|
|
3690
|
+
return {
|
|
3691
|
+
toolChoice: 'none',
|
|
3692
|
+
userMessage: `⚠️ TIME LIMIT REACHED. You are running out of time. You have ${remaining} step(s) remaining. Provide your BEST answer NOW using the information you have already gathered. Do NOT call any more tools. Summarize your findings and respond completely. If something was not completed, honestly state what was not done and provide any partial results or recommendations you can offer.`
|
|
3693
|
+
};
|
|
3694
|
+
}
|
|
3695
|
+
|
|
3696
|
+
if (this.debug) {
|
|
3697
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step ${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax} (${remaining} remaining)`);
|
|
3698
|
+
}
|
|
3699
|
+
return { toolChoice: 'none' };
|
|
3700
|
+
}
|
|
3701
|
+
|
|
3625
3702
|
// Last-iteration warning
|
|
3626
3703
|
if (stepNumber === maxIterations - 1) {
|
|
3627
3704
|
return {
|
|
@@ -3746,6 +3823,14 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3746
3823
|
}));
|
|
3747
3824
|
}
|
|
3748
3825
|
this.tracer.addEvent('iteration.step', stepEvent);
|
|
3826
|
+
|
|
3827
|
+
// Track graceful timeout wind-down steps
|
|
3828
|
+
if (gracefulTimeoutState.triggered) {
|
|
3829
|
+
this.tracer.addEvent('graceful_timeout.wind_down_step', {
|
|
3830
|
+
bonus_step: gracefulTimeoutState.bonusStepsUsed,
|
|
3831
|
+
bonus_max: gracefulTimeoutState.bonusStepsMax
|
|
3832
|
+
});
|
|
3833
|
+
}
|
|
3749
3834
|
}
|
|
3750
3835
|
|
|
3751
3836
|
// Record token usage
|
|
@@ -3807,30 +3892,59 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3807
3892
|
const executeAIRequest = async () => {
|
|
3808
3893
|
const result = await this.streamTextWithRetryAndFallback(streamOptions);
|
|
3809
3894
|
|
|
3810
|
-
//
|
|
3811
|
-
//
|
|
3812
|
-
//
|
|
3813
|
-
|
|
3814
|
-
let
|
|
3815
|
-
if (
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3895
|
+
// Set up graceful timeout timer now that streamText is running.
|
|
3896
|
+
// streamText() returns immediately — the actual tool loop runs asynchronously
|
|
3897
|
+
// and completes when we await result.steps/result.text below.
|
|
3898
|
+
let gracefulTimeoutId = null;
|
|
3899
|
+
let hardAbortTimeoutId = null;
|
|
3900
|
+
if (this.timeoutBehavior === 'graceful' && gracefulTimeoutState && this.maxOperationTimeout > 0) {
|
|
3901
|
+
gracefulTimeoutId = setTimeout(() => {
|
|
3902
|
+
gracefulTimeoutState.triggered = true;
|
|
3903
|
+
if (this.debug) {
|
|
3904
|
+
console.log(`[DEBUG] Soft timeout after ${this.maxOperationTimeout}ms — entering wind-down mode (${gracefulTimeoutState.bonusStepsMax} bonus steps)`);
|
|
3905
|
+
}
|
|
3906
|
+
// Safety net: hard abort after 60s if wind-down doesn't complete
|
|
3907
|
+
hardAbortTimeoutId = setTimeout(() => {
|
|
3908
|
+
if (this._abortController) {
|
|
3909
|
+
this._abortController.abort();
|
|
3910
|
+
}
|
|
3911
|
+
if (this.debug) {
|
|
3912
|
+
console.log(`[DEBUG] Hard abort — wind-down safety net expired after 60s`);
|
|
3913
|
+
}
|
|
3914
|
+
}, 60000);
|
|
3915
|
+
}, this.maxOperationTimeout);
|
|
3821
3916
|
}
|
|
3822
3917
|
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3918
|
+
try {
|
|
3919
|
+
// Use only the last step's text as the final answer.
|
|
3920
|
+
// result.text concatenates ALL steps (including intermediate planning text),
|
|
3921
|
+
// but the user should only see the final answer from the last step.
|
|
3922
|
+
const steps = await result.steps;
|
|
3923
|
+
let finalText;
|
|
3924
|
+
if (steps && steps.length > 1) {
|
|
3925
|
+
// Multi-step: use last step's text (the actual answer after tool calls)
|
|
3926
|
+
const lastStepText = steps[steps.length - 1].text;
|
|
3927
|
+
finalText = lastStepText || await result.text;
|
|
3928
|
+
} else {
|
|
3929
|
+
finalText = await result.text;
|
|
3930
|
+
}
|
|
3826
3931
|
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
3831
|
-
}
|
|
3932
|
+
if (this.debug) {
|
|
3933
|
+
console.log(`[DEBUG] streamText completed: ${steps?.length || 0} steps, finalText=${finalText?.length || 0} chars`);
|
|
3934
|
+
}
|
|
3832
3935
|
|
|
3833
|
-
|
|
3936
|
+
// Record final token usage
|
|
3937
|
+
const usage = await result.usage;
|
|
3938
|
+
if (usage) {
|
|
3939
|
+
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
3940
|
+
}
|
|
3941
|
+
|
|
3942
|
+
return { finalText, result };
|
|
3943
|
+
} finally {
|
|
3944
|
+
// Clean up graceful timeout timers
|
|
3945
|
+
if (gracefulTimeoutId) clearTimeout(gracefulTimeoutId);
|
|
3946
|
+
if (hardAbortTimeoutId) clearTimeout(hardAbortTimeoutId);
|
|
3947
|
+
}
|
|
3834
3948
|
};
|
|
3835
3949
|
|
|
3836
3950
|
let aiResult;
|
|
@@ -3875,6 +3989,58 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3875
3989
|
finalResult = aiResult.finalText;
|
|
3876
3990
|
}
|
|
3877
3991
|
|
|
3992
|
+
// Graceful timeout handling: ensure the response clearly indicates
|
|
3993
|
+
// the research was interrupted and may be incomplete.
|
|
3994
|
+
if (gracefulTimeoutState.triggered) {
|
|
3995
|
+
const timeoutNotice = '**Note: This response was generated under a time constraint. The research may be incomplete, and some planned searches or analysis steps were not completed.**\n\n';
|
|
3996
|
+
|
|
3997
|
+
if (!finalResult || finalResult === 'I was unable to complete your request due to reaching the maximum number of tool iterations.') {
|
|
3998
|
+
// Wind-down produced empty text — try to collect useful content.
|
|
3999
|
+
// Some models (e.g., Gemini) return finishReason:'other' with empty text
|
|
4000
|
+
// when forced from tool-calling to text-only mode mid-task.
|
|
4001
|
+
try {
|
|
4002
|
+
// Try result.text (concatenation of all step texts)
|
|
4003
|
+
const allText = await aiResult.result.text;
|
|
4004
|
+
if (allText && allText.trim()) {
|
|
4005
|
+
finalResult = timeoutNotice + allText;
|
|
4006
|
+
if (this.debug) {
|
|
4007
|
+
console.log(`[DEBUG] Graceful timeout: using concatenated step text (${allText.length} chars)`);
|
|
4008
|
+
}
|
|
4009
|
+
} else {
|
|
4010
|
+
// Last resort: collect tool result summaries as partial information
|
|
4011
|
+
const steps = await aiResult.result.steps;
|
|
4012
|
+
const toolSummaries = [];
|
|
4013
|
+
for (const step of (steps || [])) {
|
|
4014
|
+
if (step.toolResults?.length > 0) {
|
|
4015
|
+
for (const tr of step.toolResults) {
|
|
4016
|
+
const resultText = typeof tr.result === 'string' ? tr.result : JSON.stringify(tr.result);
|
|
4017
|
+
if (resultText && resultText.length > 0 && resultText.length < 5000) {
|
|
4018
|
+
toolSummaries.push(resultText.substring(0, 2000));
|
|
4019
|
+
}
|
|
4020
|
+
}
|
|
4021
|
+
}
|
|
4022
|
+
}
|
|
4023
|
+
if (toolSummaries.length > 0) {
|
|
4024
|
+
finalResult = `${timeoutNotice}The operation timed out before a complete answer could be generated. Here is the partial information gathered:\n\n${toolSummaries.join('\n\n---\n\n')}`;
|
|
4025
|
+
if (this.debug) {
|
|
4026
|
+
console.log(`[DEBUG] Graceful timeout: built fallback from ${toolSummaries.length} tool results`);
|
|
4027
|
+
}
|
|
4028
|
+
} else {
|
|
4029
|
+
finalResult = 'The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.';
|
|
4030
|
+
}
|
|
4031
|
+
}
|
|
4032
|
+
} catch (e) {
|
|
4033
|
+
if (this.debug) {
|
|
4034
|
+
console.log(`[DEBUG] Graceful timeout fallback error: ${e.message}`);
|
|
4035
|
+
}
|
|
4036
|
+
finalResult = 'The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.';
|
|
4037
|
+
}
|
|
4038
|
+
} else {
|
|
4039
|
+
// Model produced text during wind-down — prepend the timeout notice
|
|
4040
|
+
finalResult = timeoutNotice + finalResult;
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
|
|
3878
4044
|
// Update currentMessages from the result for history storage
|
|
3879
4045
|
// The SDK manages the full message history internally
|
|
3880
4046
|
const resultMessages = await aiResult.result.response?.messages;
|
package/cjs/agent/ProbeAgent.cjs
CHANGED
|
@@ -99870,6 +99870,18 @@ var init_ProbeAgent = __esm({
|
|
|
99870
99870
|
if (this.debug) {
|
|
99871
99871
|
console.log(`[DEBUG] Max operation timeout: ${this.maxOperationTimeout}ms`);
|
|
99872
99872
|
}
|
|
99873
|
+
this.timeoutBehavior = options.timeoutBehavior ?? (() => {
|
|
99874
|
+
const val = process.env.TIMEOUT_BEHAVIOR;
|
|
99875
|
+
if (val === "hard") return "hard";
|
|
99876
|
+
return "graceful";
|
|
99877
|
+
})();
|
|
99878
|
+
this.gracefulTimeoutBonusSteps = options.gracefulTimeoutBonusSteps ?? (() => {
|
|
99879
|
+
const parsed = parseInt(process.env.GRACEFUL_TIMEOUT_BONUS_STEPS, 10);
|
|
99880
|
+
return isNaN(parsed) || parsed < 1 || parsed > 20 ? 4 : parsed;
|
|
99881
|
+
})();
|
|
99882
|
+
if (this.debug) {
|
|
99883
|
+
console.log(`[DEBUG] Timeout behavior: ${this.timeoutBehavior}, bonus steps: ${this.gracefulTimeoutBonusSteps}`);
|
|
99884
|
+
}
|
|
99873
99885
|
this.retryConfig = options.retry || {};
|
|
99874
99886
|
this.retryManager = null;
|
|
99875
99887
|
this.fallbackConfig = options.fallback || null;
|
|
@@ -100803,12 +100815,16 @@ var init_ProbeAgent = __esm({
|
|
|
100803
100815
|
}, { once: true });
|
|
100804
100816
|
}
|
|
100805
100817
|
if (this.maxOperationTimeout && this.maxOperationTimeout > 0) {
|
|
100806
|
-
|
|
100807
|
-
|
|
100808
|
-
|
|
100809
|
-
|
|
100810
|
-
|
|
100811
|
-
|
|
100818
|
+
const gts = this._gracefulTimeoutState;
|
|
100819
|
+
if (this.timeoutBehavior === "graceful" && gts) {
|
|
100820
|
+
} else {
|
|
100821
|
+
timeoutState.timeoutId = setTimeout(() => {
|
|
100822
|
+
controller.abort();
|
|
100823
|
+
if (this.debug) {
|
|
100824
|
+
console.log(`[DEBUG] Operation timed out after ${this.maxOperationTimeout}ms (max operation timeout)`);
|
|
100825
|
+
}
|
|
100826
|
+
}, this.maxOperationTimeout);
|
|
100827
|
+
}
|
|
100812
100828
|
}
|
|
100813
100829
|
try {
|
|
100814
100830
|
const useClaudeCode = this.clientApiProvider === "claude-code" || process.env.USE_CLAUDE_CODE === "true";
|
|
@@ -102399,6 +102415,14 @@ You are working with a workspace. Available paths: ${workspaceDesc}
|
|
|
102399
102415
|
}
|
|
102400
102416
|
let completionPromptInjected = false;
|
|
102401
102417
|
let preCompletionResult = null;
|
|
102418
|
+
const gracefulTimeoutState = {
|
|
102419
|
+
triggered: false,
|
|
102420
|
+
// Set to true when soft timeout fires
|
|
102421
|
+
bonusStepsUsed: 0,
|
|
102422
|
+
// Steps taken after soft timeout
|
|
102423
|
+
bonusStepsMax: this.gracefulTimeoutBonusSteps
|
|
102424
|
+
};
|
|
102425
|
+
this._gracefulTimeoutState = gracefulTimeoutState;
|
|
102402
102426
|
let compactionAttempted = false;
|
|
102403
102427
|
while (true) {
|
|
102404
102428
|
try {
|
|
@@ -102408,6 +102432,15 @@ You are working with a workspace. Available paths: ${workspaceDesc}
|
|
|
102408
102432
|
messages: messagesForAI,
|
|
102409
102433
|
tools: tools2,
|
|
102410
102434
|
stopWhen: ({ steps }) => {
|
|
102435
|
+
if (gracefulTimeoutState.triggered) {
|
|
102436
|
+
if (gracefulTimeoutState.bonusStepsUsed >= gracefulTimeoutState.bonusStepsMax) {
|
|
102437
|
+
if (this.debug) {
|
|
102438
|
+
console.log(`[DEBUG] stopWhen: graceful timeout bonus steps exhausted (${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax}), forcing stop`);
|
|
102439
|
+
}
|
|
102440
|
+
return true;
|
|
102441
|
+
}
|
|
102442
|
+
return false;
|
|
102443
|
+
}
|
|
102411
102444
|
if (steps.length >= maxIterations) return true;
|
|
102412
102445
|
const lastStep = steps[steps.length - 1];
|
|
102413
102446
|
const modelWantsToStop = lastStep?.finishReason === "stop" && (!lastStep?.toolCalls || lastStep.toolCalls.length === 0);
|
|
@@ -102451,6 +102484,30 @@ You are working with a workspace. Available paths: ${workspaceDesc}
|
|
|
102451
102484
|
return false;
|
|
102452
102485
|
},
|
|
102453
102486
|
prepareStep: ({ steps, stepNumber }) => {
|
|
102487
|
+
if (gracefulTimeoutState.triggered) {
|
|
102488
|
+
gracefulTimeoutState.bonusStepsUsed++;
|
|
102489
|
+
const remaining = gracefulTimeoutState.bonusStepsMax - gracefulTimeoutState.bonusStepsUsed;
|
|
102490
|
+
if (gracefulTimeoutState.bonusStepsUsed === 1) {
|
|
102491
|
+
if (this.debug) {
|
|
102492
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step 1/${gracefulTimeoutState.bonusStepsMax}`);
|
|
102493
|
+
}
|
|
102494
|
+
if (this.tracer) {
|
|
102495
|
+
this.tracer.addEvent("graceful_timeout.wind_down_started", {
|
|
102496
|
+
bonus_steps_max: gracefulTimeoutState.bonusStepsMax,
|
|
102497
|
+
current_iteration: currentIteration,
|
|
102498
|
+
max_iterations: maxIterations
|
|
102499
|
+
});
|
|
102500
|
+
}
|
|
102501
|
+
return {
|
|
102502
|
+
toolChoice: "none",
|
|
102503
|
+
userMessage: `\u26A0\uFE0F TIME LIMIT REACHED. You are running out of time. You have ${remaining} step(s) remaining. Provide your BEST answer NOW using the information you have already gathered. Do NOT call any more tools. Summarize your findings and respond completely. If something was not completed, honestly state what was not done and provide any partial results or recommendations you can offer.`
|
|
102504
|
+
};
|
|
102505
|
+
}
|
|
102506
|
+
if (this.debug) {
|
|
102507
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step ${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax} (${remaining} remaining)`);
|
|
102508
|
+
}
|
|
102509
|
+
return { toolChoice: "none" };
|
|
102510
|
+
}
|
|
102454
102511
|
if (stepNumber === maxIterations - 1) {
|
|
102455
102512
|
return {
|
|
102456
102513
|
toolChoice: "none"
|
|
@@ -102554,6 +102611,12 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
102554
102611
|
}));
|
|
102555
102612
|
}
|
|
102556
102613
|
this.tracer.addEvent("iteration.step", stepEvent);
|
|
102614
|
+
if (gracefulTimeoutState.triggered) {
|
|
102615
|
+
this.tracer.addEvent("graceful_timeout.wind_down_step", {
|
|
102616
|
+
bonus_step: gracefulTimeoutState.bonusStepsUsed,
|
|
102617
|
+
bonus_max: gracefulTimeoutState.bonusStepsMax
|
|
102618
|
+
});
|
|
102619
|
+
}
|
|
102557
102620
|
}
|
|
102558
102621
|
if (usage) {
|
|
102559
102622
|
this.tokenCounter.recordUsage(usage);
|
|
@@ -102599,22 +102662,45 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
102599
102662
|
}
|
|
102600
102663
|
const executeAIRequest = async () => {
|
|
102601
102664
|
const result = await this.streamTextWithRetryAndFallback(streamOptions);
|
|
102602
|
-
|
|
102603
|
-
let
|
|
102604
|
-
if (
|
|
102605
|
-
|
|
102606
|
-
|
|
102607
|
-
|
|
102608
|
-
|
|
102609
|
-
|
|
102610
|
-
|
|
102611
|
-
|
|
102665
|
+
let gracefulTimeoutId = null;
|
|
102666
|
+
let hardAbortTimeoutId = null;
|
|
102667
|
+
if (this.timeoutBehavior === "graceful" && gracefulTimeoutState && this.maxOperationTimeout > 0) {
|
|
102668
|
+
gracefulTimeoutId = setTimeout(() => {
|
|
102669
|
+
gracefulTimeoutState.triggered = true;
|
|
102670
|
+
if (this.debug) {
|
|
102671
|
+
console.log(`[DEBUG] Soft timeout after ${this.maxOperationTimeout}ms \u2014 entering wind-down mode (${gracefulTimeoutState.bonusStepsMax} bonus steps)`);
|
|
102672
|
+
}
|
|
102673
|
+
hardAbortTimeoutId = setTimeout(() => {
|
|
102674
|
+
if (this._abortController) {
|
|
102675
|
+
this._abortController.abort();
|
|
102676
|
+
}
|
|
102677
|
+
if (this.debug) {
|
|
102678
|
+
console.log(`[DEBUG] Hard abort \u2014 wind-down safety net expired after 60s`);
|
|
102679
|
+
}
|
|
102680
|
+
}, 6e4);
|
|
102681
|
+
}, this.maxOperationTimeout);
|
|
102612
102682
|
}
|
|
102613
|
-
|
|
102614
|
-
|
|
102615
|
-
|
|
102683
|
+
try {
|
|
102684
|
+
const steps = await result.steps;
|
|
102685
|
+
let finalText;
|
|
102686
|
+
if (steps && steps.length > 1) {
|
|
102687
|
+
const lastStepText = steps[steps.length - 1].text;
|
|
102688
|
+
finalText = lastStepText || await result.text;
|
|
102689
|
+
} else {
|
|
102690
|
+
finalText = await result.text;
|
|
102691
|
+
}
|
|
102692
|
+
if (this.debug) {
|
|
102693
|
+
console.log(`[DEBUG] streamText completed: ${steps?.length || 0} steps, finalText=${finalText?.length || 0} chars`);
|
|
102694
|
+
}
|
|
102695
|
+
const usage = await result.usage;
|
|
102696
|
+
if (usage) {
|
|
102697
|
+
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
102698
|
+
}
|
|
102699
|
+
return { finalText, result };
|
|
102700
|
+
} finally {
|
|
102701
|
+
if (gracefulTimeoutId) clearTimeout(gracefulTimeoutId);
|
|
102702
|
+
if (hardAbortTimeoutId) clearTimeout(hardAbortTimeoutId);
|
|
102616
102703
|
}
|
|
102617
|
-
return { finalText, result };
|
|
102618
102704
|
};
|
|
102619
102705
|
let aiResult;
|
|
102620
102706
|
if (this.tracer) {
|
|
@@ -102651,6 +102737,50 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
102651
102737
|
} else if (aiResult.finalText) {
|
|
102652
102738
|
finalResult = aiResult.finalText;
|
|
102653
102739
|
}
|
|
102740
|
+
if (gracefulTimeoutState.triggered) {
|
|
102741
|
+
const timeoutNotice = "**Note: This response was generated under a time constraint. The research may be incomplete, and some planned searches or analysis steps were not completed.**\n\n";
|
|
102742
|
+
if (!finalResult || finalResult === "I was unable to complete your request due to reaching the maximum number of tool iterations.") {
|
|
102743
|
+
try {
|
|
102744
|
+
const allText = await aiResult.result.text;
|
|
102745
|
+
if (allText && allText.trim()) {
|
|
102746
|
+
finalResult = timeoutNotice + allText;
|
|
102747
|
+
if (this.debug) {
|
|
102748
|
+
console.log(`[DEBUG] Graceful timeout: using concatenated step text (${allText.length} chars)`);
|
|
102749
|
+
}
|
|
102750
|
+
} else {
|
|
102751
|
+
const steps = await aiResult.result.steps;
|
|
102752
|
+
const toolSummaries = [];
|
|
102753
|
+
for (const step of steps || []) {
|
|
102754
|
+
if (step.toolResults?.length > 0) {
|
|
102755
|
+
for (const tr of step.toolResults) {
|
|
102756
|
+
const resultText = typeof tr.result === "string" ? tr.result : JSON.stringify(tr.result);
|
|
102757
|
+
if (resultText && resultText.length > 0 && resultText.length < 5e3) {
|
|
102758
|
+
toolSummaries.push(resultText.substring(0, 2e3));
|
|
102759
|
+
}
|
|
102760
|
+
}
|
|
102761
|
+
}
|
|
102762
|
+
}
|
|
102763
|
+
if (toolSummaries.length > 0) {
|
|
102764
|
+
finalResult = `${timeoutNotice}The operation timed out before a complete answer could be generated. Here is the partial information gathered:
|
|
102765
|
+
|
|
102766
|
+
${toolSummaries.join("\n\n---\n\n")}`;
|
|
102767
|
+
if (this.debug) {
|
|
102768
|
+
console.log(`[DEBUG] Graceful timeout: built fallback from ${toolSummaries.length} tool results`);
|
|
102769
|
+
}
|
|
102770
|
+
} else {
|
|
102771
|
+
finalResult = "The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.";
|
|
102772
|
+
}
|
|
102773
|
+
}
|
|
102774
|
+
} catch (e) {
|
|
102775
|
+
if (this.debug) {
|
|
102776
|
+
console.log(`[DEBUG] Graceful timeout fallback error: ${e.message}`);
|
|
102777
|
+
}
|
|
102778
|
+
finalResult = "The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.";
|
|
102779
|
+
}
|
|
102780
|
+
} else {
|
|
102781
|
+
finalResult = timeoutNotice + finalResult;
|
|
102782
|
+
}
|
|
102783
|
+
}
|
|
102654
102784
|
const resultMessages = await aiResult.result.response?.messages;
|
|
102655
102785
|
if (resultMessages) {
|
|
102656
102786
|
for (const msg of resultMessages) {
|
package/cjs/index.cjs
CHANGED
|
@@ -96781,6 +96781,18 @@ var init_ProbeAgent = __esm({
|
|
|
96781
96781
|
if (this.debug) {
|
|
96782
96782
|
console.log(`[DEBUG] Max operation timeout: ${this.maxOperationTimeout}ms`);
|
|
96783
96783
|
}
|
|
96784
|
+
this.timeoutBehavior = options.timeoutBehavior ?? (() => {
|
|
96785
|
+
const val = process.env.TIMEOUT_BEHAVIOR;
|
|
96786
|
+
if (val === "hard") return "hard";
|
|
96787
|
+
return "graceful";
|
|
96788
|
+
})();
|
|
96789
|
+
this.gracefulTimeoutBonusSteps = options.gracefulTimeoutBonusSteps ?? (() => {
|
|
96790
|
+
const parsed = parseInt(process.env.GRACEFUL_TIMEOUT_BONUS_STEPS, 10);
|
|
96791
|
+
return isNaN(parsed) || parsed < 1 || parsed > 20 ? 4 : parsed;
|
|
96792
|
+
})();
|
|
96793
|
+
if (this.debug) {
|
|
96794
|
+
console.log(`[DEBUG] Timeout behavior: ${this.timeoutBehavior}, bonus steps: ${this.gracefulTimeoutBonusSteps}`);
|
|
96795
|
+
}
|
|
96784
96796
|
this.retryConfig = options.retry || {};
|
|
96785
96797
|
this.retryManager = null;
|
|
96786
96798
|
this.fallbackConfig = options.fallback || null;
|
|
@@ -97714,12 +97726,16 @@ var init_ProbeAgent = __esm({
|
|
|
97714
97726
|
}, { once: true });
|
|
97715
97727
|
}
|
|
97716
97728
|
if (this.maxOperationTimeout && this.maxOperationTimeout > 0) {
|
|
97717
|
-
|
|
97718
|
-
|
|
97719
|
-
|
|
97720
|
-
|
|
97721
|
-
|
|
97722
|
-
|
|
97729
|
+
const gts = this._gracefulTimeoutState;
|
|
97730
|
+
if (this.timeoutBehavior === "graceful" && gts) {
|
|
97731
|
+
} else {
|
|
97732
|
+
timeoutState.timeoutId = setTimeout(() => {
|
|
97733
|
+
controller.abort();
|
|
97734
|
+
if (this.debug) {
|
|
97735
|
+
console.log(`[DEBUG] Operation timed out after ${this.maxOperationTimeout}ms (max operation timeout)`);
|
|
97736
|
+
}
|
|
97737
|
+
}, this.maxOperationTimeout);
|
|
97738
|
+
}
|
|
97723
97739
|
}
|
|
97724
97740
|
try {
|
|
97725
97741
|
const useClaudeCode = this.clientApiProvider === "claude-code" || process.env.USE_CLAUDE_CODE === "true";
|
|
@@ -99310,6 +99326,14 @@ You are working with a workspace. Available paths: ${workspaceDesc}
|
|
|
99310
99326
|
}
|
|
99311
99327
|
let completionPromptInjected = false;
|
|
99312
99328
|
let preCompletionResult = null;
|
|
99329
|
+
const gracefulTimeoutState = {
|
|
99330
|
+
triggered: false,
|
|
99331
|
+
// Set to true when soft timeout fires
|
|
99332
|
+
bonusStepsUsed: 0,
|
|
99333
|
+
// Steps taken after soft timeout
|
|
99334
|
+
bonusStepsMax: this.gracefulTimeoutBonusSteps
|
|
99335
|
+
};
|
|
99336
|
+
this._gracefulTimeoutState = gracefulTimeoutState;
|
|
99313
99337
|
let compactionAttempted = false;
|
|
99314
99338
|
while (true) {
|
|
99315
99339
|
try {
|
|
@@ -99319,6 +99343,15 @@ You are working with a workspace. Available paths: ${workspaceDesc}
|
|
|
99319
99343
|
messages: messagesForAI,
|
|
99320
99344
|
tools: tools2,
|
|
99321
99345
|
stopWhen: ({ steps }) => {
|
|
99346
|
+
if (gracefulTimeoutState.triggered) {
|
|
99347
|
+
if (gracefulTimeoutState.bonusStepsUsed >= gracefulTimeoutState.bonusStepsMax) {
|
|
99348
|
+
if (this.debug) {
|
|
99349
|
+
console.log(`[DEBUG] stopWhen: graceful timeout bonus steps exhausted (${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax}), forcing stop`);
|
|
99350
|
+
}
|
|
99351
|
+
return true;
|
|
99352
|
+
}
|
|
99353
|
+
return false;
|
|
99354
|
+
}
|
|
99322
99355
|
if (steps.length >= maxIterations) return true;
|
|
99323
99356
|
const lastStep = steps[steps.length - 1];
|
|
99324
99357
|
const modelWantsToStop = lastStep?.finishReason === "stop" && (!lastStep?.toolCalls || lastStep.toolCalls.length === 0);
|
|
@@ -99362,6 +99395,30 @@ You are working with a workspace. Available paths: ${workspaceDesc}
|
|
|
99362
99395
|
return false;
|
|
99363
99396
|
},
|
|
99364
99397
|
prepareStep: ({ steps, stepNumber }) => {
|
|
99398
|
+
if (gracefulTimeoutState.triggered) {
|
|
99399
|
+
gracefulTimeoutState.bonusStepsUsed++;
|
|
99400
|
+
const remaining = gracefulTimeoutState.bonusStepsMax - gracefulTimeoutState.bonusStepsUsed;
|
|
99401
|
+
if (gracefulTimeoutState.bonusStepsUsed === 1) {
|
|
99402
|
+
if (this.debug) {
|
|
99403
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step 1/${gracefulTimeoutState.bonusStepsMax}`);
|
|
99404
|
+
}
|
|
99405
|
+
if (this.tracer) {
|
|
99406
|
+
this.tracer.addEvent("graceful_timeout.wind_down_started", {
|
|
99407
|
+
bonus_steps_max: gracefulTimeoutState.bonusStepsMax,
|
|
99408
|
+
current_iteration: currentIteration,
|
|
99409
|
+
max_iterations: maxIterations
|
|
99410
|
+
});
|
|
99411
|
+
}
|
|
99412
|
+
return {
|
|
99413
|
+
toolChoice: "none",
|
|
99414
|
+
userMessage: `\u26A0\uFE0F TIME LIMIT REACHED. You are running out of time. You have ${remaining} step(s) remaining. Provide your BEST answer NOW using the information you have already gathered. Do NOT call any more tools. Summarize your findings and respond completely. If something was not completed, honestly state what was not done and provide any partial results or recommendations you can offer.`
|
|
99415
|
+
};
|
|
99416
|
+
}
|
|
99417
|
+
if (this.debug) {
|
|
99418
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step ${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax} (${remaining} remaining)`);
|
|
99419
|
+
}
|
|
99420
|
+
return { toolChoice: "none" };
|
|
99421
|
+
}
|
|
99365
99422
|
if (stepNumber === maxIterations - 1) {
|
|
99366
99423
|
return {
|
|
99367
99424
|
toolChoice: "none"
|
|
@@ -99465,6 +99522,12 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
99465
99522
|
}));
|
|
99466
99523
|
}
|
|
99467
99524
|
this.tracer.addEvent("iteration.step", stepEvent);
|
|
99525
|
+
if (gracefulTimeoutState.triggered) {
|
|
99526
|
+
this.tracer.addEvent("graceful_timeout.wind_down_step", {
|
|
99527
|
+
bonus_step: gracefulTimeoutState.bonusStepsUsed,
|
|
99528
|
+
bonus_max: gracefulTimeoutState.bonusStepsMax
|
|
99529
|
+
});
|
|
99530
|
+
}
|
|
99468
99531
|
}
|
|
99469
99532
|
if (usage) {
|
|
99470
99533
|
this.tokenCounter.recordUsage(usage);
|
|
@@ -99510,22 +99573,45 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
99510
99573
|
}
|
|
99511
99574
|
const executeAIRequest = async () => {
|
|
99512
99575
|
const result = await this.streamTextWithRetryAndFallback(streamOptions);
|
|
99513
|
-
|
|
99514
|
-
let
|
|
99515
|
-
if (
|
|
99516
|
-
|
|
99517
|
-
|
|
99518
|
-
|
|
99519
|
-
|
|
99520
|
-
|
|
99521
|
-
|
|
99522
|
-
|
|
99576
|
+
let gracefulTimeoutId = null;
|
|
99577
|
+
let hardAbortTimeoutId = null;
|
|
99578
|
+
if (this.timeoutBehavior === "graceful" && gracefulTimeoutState && this.maxOperationTimeout > 0) {
|
|
99579
|
+
gracefulTimeoutId = setTimeout(() => {
|
|
99580
|
+
gracefulTimeoutState.triggered = true;
|
|
99581
|
+
if (this.debug) {
|
|
99582
|
+
console.log(`[DEBUG] Soft timeout after ${this.maxOperationTimeout}ms \u2014 entering wind-down mode (${gracefulTimeoutState.bonusStepsMax} bonus steps)`);
|
|
99583
|
+
}
|
|
99584
|
+
hardAbortTimeoutId = setTimeout(() => {
|
|
99585
|
+
if (this._abortController) {
|
|
99586
|
+
this._abortController.abort();
|
|
99587
|
+
}
|
|
99588
|
+
if (this.debug) {
|
|
99589
|
+
console.log(`[DEBUG] Hard abort \u2014 wind-down safety net expired after 60s`);
|
|
99590
|
+
}
|
|
99591
|
+
}, 6e4);
|
|
99592
|
+
}, this.maxOperationTimeout);
|
|
99523
99593
|
}
|
|
99524
|
-
|
|
99525
|
-
|
|
99526
|
-
|
|
99594
|
+
try {
|
|
99595
|
+
const steps = await result.steps;
|
|
99596
|
+
let finalText;
|
|
99597
|
+
if (steps && steps.length > 1) {
|
|
99598
|
+
const lastStepText = steps[steps.length - 1].text;
|
|
99599
|
+
finalText = lastStepText || await result.text;
|
|
99600
|
+
} else {
|
|
99601
|
+
finalText = await result.text;
|
|
99602
|
+
}
|
|
99603
|
+
if (this.debug) {
|
|
99604
|
+
console.log(`[DEBUG] streamText completed: ${steps?.length || 0} steps, finalText=${finalText?.length || 0} chars`);
|
|
99605
|
+
}
|
|
99606
|
+
const usage = await result.usage;
|
|
99607
|
+
if (usage) {
|
|
99608
|
+
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
99609
|
+
}
|
|
99610
|
+
return { finalText, result };
|
|
99611
|
+
} finally {
|
|
99612
|
+
if (gracefulTimeoutId) clearTimeout(gracefulTimeoutId);
|
|
99613
|
+
if (hardAbortTimeoutId) clearTimeout(hardAbortTimeoutId);
|
|
99527
99614
|
}
|
|
99528
|
-
return { finalText, result };
|
|
99529
99615
|
};
|
|
99530
99616
|
let aiResult;
|
|
99531
99617
|
if (this.tracer) {
|
|
@@ -99562,6 +99648,50 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
99562
99648
|
} else if (aiResult.finalText) {
|
|
99563
99649
|
finalResult = aiResult.finalText;
|
|
99564
99650
|
}
|
|
99651
|
+
if (gracefulTimeoutState.triggered) {
|
|
99652
|
+
const timeoutNotice = "**Note: This response was generated under a time constraint. The research may be incomplete, and some planned searches or analysis steps were not completed.**\n\n";
|
|
99653
|
+
if (!finalResult || finalResult === "I was unable to complete your request due to reaching the maximum number of tool iterations.") {
|
|
99654
|
+
try {
|
|
99655
|
+
const allText = await aiResult.result.text;
|
|
99656
|
+
if (allText && allText.trim()) {
|
|
99657
|
+
finalResult = timeoutNotice + allText;
|
|
99658
|
+
if (this.debug) {
|
|
99659
|
+
console.log(`[DEBUG] Graceful timeout: using concatenated step text (${allText.length} chars)`);
|
|
99660
|
+
}
|
|
99661
|
+
} else {
|
|
99662
|
+
const steps = await aiResult.result.steps;
|
|
99663
|
+
const toolSummaries = [];
|
|
99664
|
+
for (const step of steps || []) {
|
|
99665
|
+
if (step.toolResults?.length > 0) {
|
|
99666
|
+
for (const tr of step.toolResults) {
|
|
99667
|
+
const resultText = typeof tr.result === "string" ? tr.result : JSON.stringify(tr.result);
|
|
99668
|
+
if (resultText && resultText.length > 0 && resultText.length < 5e3) {
|
|
99669
|
+
toolSummaries.push(resultText.substring(0, 2e3));
|
|
99670
|
+
}
|
|
99671
|
+
}
|
|
99672
|
+
}
|
|
99673
|
+
}
|
|
99674
|
+
if (toolSummaries.length > 0) {
|
|
99675
|
+
finalResult = `${timeoutNotice}The operation timed out before a complete answer could be generated. Here is the partial information gathered:
|
|
99676
|
+
|
|
99677
|
+
${toolSummaries.join("\n\n---\n\n")}`;
|
|
99678
|
+
if (this.debug) {
|
|
99679
|
+
console.log(`[DEBUG] Graceful timeout: built fallback from ${toolSummaries.length} tool results`);
|
|
99680
|
+
}
|
|
99681
|
+
} else {
|
|
99682
|
+
finalResult = "The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.";
|
|
99683
|
+
}
|
|
99684
|
+
}
|
|
99685
|
+
} catch (e) {
|
|
99686
|
+
if (this.debug) {
|
|
99687
|
+
console.log(`[DEBUG] Graceful timeout fallback error: ${e.message}`);
|
|
99688
|
+
}
|
|
99689
|
+
finalResult = "The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.";
|
|
99690
|
+
}
|
|
99691
|
+
} else {
|
|
99692
|
+
finalResult = timeoutNotice + finalResult;
|
|
99693
|
+
}
|
|
99694
|
+
}
|
|
99565
99695
|
const resultMessages = await aiResult.result.response?.messages;
|
|
99566
99696
|
if (resultMessages) {
|
|
99567
99697
|
for (const msg of resultMessages) {
|
package/package.json
CHANGED
|
@@ -106,6 +106,10 @@ export interface ProbeAgentOptions {
|
|
|
106
106
|
requestTimeout?: number;
|
|
107
107
|
/** Maximum timeout in ms for the entire operation including all retries and fallbacks (default: 300000 or MAX_OPERATION_TIMEOUT env var). This is the absolute maximum time for streamTextWithRetryAndFallback. */
|
|
108
108
|
maxOperationTimeout?: number;
|
|
109
|
+
/** Timeout behavior: 'graceful' winds down with bonus steps giving the agent a chance to respond, 'hard' aborts immediately (default: 'graceful'). Env var: TIMEOUT_BEHAVIOR */
|
|
110
|
+
timeoutBehavior?: 'graceful' | 'hard';
|
|
111
|
+
/** Number of bonus steps during graceful timeout wind-down (default: 4, range: 1-20). Env var: GRACEFUL_TIMEOUT_BONUS_STEPS */
|
|
112
|
+
gracefulTimeoutBonusSteps?: number;
|
|
109
113
|
}
|
|
110
114
|
|
|
111
115
|
/**
|
package/src/agent/ProbeAgent.js
CHANGED
|
@@ -391,6 +391,23 @@ export class ProbeAgent {
|
|
|
391
391
|
console.log(`[DEBUG] Max operation timeout: ${this.maxOperationTimeout}ms`);
|
|
392
392
|
}
|
|
393
393
|
|
|
394
|
+
// Timeout behavior: 'graceful' (default) winds down with bonus steps, 'hard' aborts immediately
|
|
395
|
+
this.timeoutBehavior = options.timeoutBehavior ?? (() => {
|
|
396
|
+
const val = process.env.TIMEOUT_BEHAVIOR;
|
|
397
|
+
if (val === 'hard') return 'hard';
|
|
398
|
+
return 'graceful';
|
|
399
|
+
})();
|
|
400
|
+
|
|
401
|
+
// Number of bonus steps during graceful timeout wind-down (default 4)
|
|
402
|
+
this.gracefulTimeoutBonusSteps = options.gracefulTimeoutBonusSteps ?? (() => {
|
|
403
|
+
const parsed = parseInt(process.env.GRACEFUL_TIMEOUT_BONUS_STEPS, 10);
|
|
404
|
+
return (isNaN(parsed) || parsed < 1 || parsed > 20) ? 4 : parsed;
|
|
405
|
+
})();
|
|
406
|
+
|
|
407
|
+
if (this.debug) {
|
|
408
|
+
console.log(`[DEBUG] Timeout behavior: ${this.timeoutBehavior}, bonus steps: ${this.gracefulTimeoutBonusSteps}`);
|
|
409
|
+
}
|
|
410
|
+
|
|
394
411
|
// Retry configuration
|
|
395
412
|
this.retryConfig = options.retry || {};
|
|
396
413
|
this.retryManager = null; // Will be initialized lazily when needed
|
|
@@ -1554,13 +1571,24 @@ export class ProbeAgent {
|
|
|
1554
1571
|
}
|
|
1555
1572
|
|
|
1556
1573
|
// Set up overall operation timeout (default 5 minutes)
|
|
1574
|
+
// NOTE: For Vercel AI SDK paths, streamText() returns immediately and the
|
|
1575
|
+
// actual tool loop runs asynchronously. The graceful timeout timer is set up
|
|
1576
|
+
// in the run() method where results are actually awaited, not here.
|
|
1577
|
+
// This timer only handles the hard abort for non-graceful mode and engine paths.
|
|
1557
1578
|
if (this.maxOperationTimeout && this.maxOperationTimeout > 0) {
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1579
|
+
const gts = this._gracefulTimeoutState;
|
|
1580
|
+
if (this.timeoutBehavior === 'graceful' && gts) {
|
|
1581
|
+
// Graceful mode: timer is managed in run() method.
|
|
1582
|
+
// Only set up the AbortController link (no timer here).
|
|
1583
|
+
} else {
|
|
1584
|
+
// Hard mode: immediate abort (legacy behavior)
|
|
1585
|
+
timeoutState.timeoutId = setTimeout(() => {
|
|
1586
|
+
controller.abort();
|
|
1587
|
+
if (this.debug) {
|
|
1588
|
+
console.log(`[DEBUG] Operation timed out after ${this.maxOperationTimeout}ms (max operation timeout)`);
|
|
1589
|
+
}
|
|
1590
|
+
}, this.maxOperationTimeout);
|
|
1591
|
+
}
|
|
1564
1592
|
}
|
|
1565
1593
|
|
|
1566
1594
|
try {
|
|
@@ -3552,6 +3580,15 @@ Follow these instructions carefully:
|
|
|
3552
3580
|
let completionPromptInjected = false;
|
|
3553
3581
|
let preCompletionResult = null; // Stores the result before completionPrompt for fallback
|
|
3554
3582
|
|
|
3583
|
+
// Graceful timeout state — shared between setTimeout (in streamTextWithRetryAndFallback)
|
|
3584
|
+
// and prepareStep/stopWhen callbacks (in streamText loop)
|
|
3585
|
+
const gracefulTimeoutState = {
|
|
3586
|
+
triggered: false, // Set to true when soft timeout fires
|
|
3587
|
+
bonusStepsUsed: 0, // Steps taken after soft timeout
|
|
3588
|
+
bonusStepsMax: this.gracefulTimeoutBonusSteps
|
|
3589
|
+
};
|
|
3590
|
+
this._gracefulTimeoutState = gracefulTimeoutState;
|
|
3591
|
+
|
|
3555
3592
|
// Context compaction retry loop
|
|
3556
3593
|
let compactionAttempted = false;
|
|
3557
3594
|
while (true) {
|
|
@@ -3563,6 +3600,17 @@ Follow these instructions carefully:
|
|
|
3563
3600
|
messages: messagesForAI,
|
|
3564
3601
|
tools,
|
|
3565
3602
|
stopWhen: ({ steps }) => {
|
|
3603
|
+
// Graceful timeout wind-down: override normal limits, stop only when bonus steps exhausted
|
|
3604
|
+
if (gracefulTimeoutState.triggered) {
|
|
3605
|
+
if (gracefulTimeoutState.bonusStepsUsed >= gracefulTimeoutState.bonusStepsMax) {
|
|
3606
|
+
if (this.debug) {
|
|
3607
|
+
console.log(`[DEBUG] stopWhen: graceful timeout bonus steps exhausted (${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax}), forcing stop`);
|
|
3608
|
+
}
|
|
3609
|
+
return true;
|
|
3610
|
+
}
|
|
3611
|
+
return false; // Allow more bonus steps
|
|
3612
|
+
}
|
|
3613
|
+
|
|
3566
3614
|
// Hard limit
|
|
3567
3615
|
if (steps.length >= maxIterations) return true;
|
|
3568
3616
|
|
|
@@ -3622,6 +3670,35 @@ Follow these instructions carefully:
|
|
|
3622
3670
|
return false;
|
|
3623
3671
|
},
|
|
3624
3672
|
prepareStep: ({ steps, stepNumber }) => {
|
|
3673
|
+
// Graceful timeout wind-down: force text-only response with wrap-up reminder
|
|
3674
|
+
if (gracefulTimeoutState.triggered) {
|
|
3675
|
+
gracefulTimeoutState.bonusStepsUsed++;
|
|
3676
|
+
const remaining = gracefulTimeoutState.bonusStepsMax - gracefulTimeoutState.bonusStepsUsed;
|
|
3677
|
+
|
|
3678
|
+
if (gracefulTimeoutState.bonusStepsUsed === 1) {
|
|
3679
|
+
// First wind-down step: inject wrap-up message
|
|
3680
|
+
if (this.debug) {
|
|
3681
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step 1/${gracefulTimeoutState.bonusStepsMax}`);
|
|
3682
|
+
}
|
|
3683
|
+
if (this.tracer) {
|
|
3684
|
+
this.tracer.addEvent('graceful_timeout.wind_down_started', {
|
|
3685
|
+
bonus_steps_max: gracefulTimeoutState.bonusStepsMax,
|
|
3686
|
+
current_iteration: currentIteration,
|
|
3687
|
+
max_iterations: maxIterations
|
|
3688
|
+
});
|
|
3689
|
+
}
|
|
3690
|
+
return {
|
|
3691
|
+
toolChoice: 'none',
|
|
3692
|
+
userMessage: `⚠️ TIME LIMIT REACHED. You are running out of time. You have ${remaining} step(s) remaining. Provide your BEST answer NOW using the information you have already gathered. Do NOT call any more tools. Summarize your findings and respond completely. If something was not completed, honestly state what was not done and provide any partial results or recommendations you can offer.`
|
|
3693
|
+
};
|
|
3694
|
+
}
|
|
3695
|
+
|
|
3696
|
+
if (this.debug) {
|
|
3697
|
+
console.log(`[DEBUG] prepareStep: graceful timeout wind-down step ${gracefulTimeoutState.bonusStepsUsed}/${gracefulTimeoutState.bonusStepsMax} (${remaining} remaining)`);
|
|
3698
|
+
}
|
|
3699
|
+
return { toolChoice: 'none' };
|
|
3700
|
+
}
|
|
3701
|
+
|
|
3625
3702
|
// Last-iteration warning
|
|
3626
3703
|
if (stepNumber === maxIterations - 1) {
|
|
3627
3704
|
return {
|
|
@@ -3746,6 +3823,14 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3746
3823
|
}));
|
|
3747
3824
|
}
|
|
3748
3825
|
this.tracer.addEvent('iteration.step', stepEvent);
|
|
3826
|
+
|
|
3827
|
+
// Track graceful timeout wind-down steps
|
|
3828
|
+
if (gracefulTimeoutState.triggered) {
|
|
3829
|
+
this.tracer.addEvent('graceful_timeout.wind_down_step', {
|
|
3830
|
+
bonus_step: gracefulTimeoutState.bonusStepsUsed,
|
|
3831
|
+
bonus_max: gracefulTimeoutState.bonusStepsMax
|
|
3832
|
+
});
|
|
3833
|
+
}
|
|
3749
3834
|
}
|
|
3750
3835
|
|
|
3751
3836
|
// Record token usage
|
|
@@ -3807,30 +3892,59 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3807
3892
|
const executeAIRequest = async () => {
|
|
3808
3893
|
const result = await this.streamTextWithRetryAndFallback(streamOptions);
|
|
3809
3894
|
|
|
3810
|
-
//
|
|
3811
|
-
//
|
|
3812
|
-
//
|
|
3813
|
-
|
|
3814
|
-
let
|
|
3815
|
-
if (
|
|
3816
|
-
|
|
3817
|
-
|
|
3818
|
-
|
|
3819
|
-
|
|
3820
|
-
|
|
3895
|
+
// Set up graceful timeout timer now that streamText is running.
|
|
3896
|
+
// streamText() returns immediately — the actual tool loop runs asynchronously
|
|
3897
|
+
// and completes when we await result.steps/result.text below.
|
|
3898
|
+
let gracefulTimeoutId = null;
|
|
3899
|
+
let hardAbortTimeoutId = null;
|
|
3900
|
+
if (this.timeoutBehavior === 'graceful' && gracefulTimeoutState && this.maxOperationTimeout > 0) {
|
|
3901
|
+
gracefulTimeoutId = setTimeout(() => {
|
|
3902
|
+
gracefulTimeoutState.triggered = true;
|
|
3903
|
+
if (this.debug) {
|
|
3904
|
+
console.log(`[DEBUG] Soft timeout after ${this.maxOperationTimeout}ms — entering wind-down mode (${gracefulTimeoutState.bonusStepsMax} bonus steps)`);
|
|
3905
|
+
}
|
|
3906
|
+
// Safety net: hard abort after 60s if wind-down doesn't complete
|
|
3907
|
+
hardAbortTimeoutId = setTimeout(() => {
|
|
3908
|
+
if (this._abortController) {
|
|
3909
|
+
this._abortController.abort();
|
|
3910
|
+
}
|
|
3911
|
+
if (this.debug) {
|
|
3912
|
+
console.log(`[DEBUG] Hard abort — wind-down safety net expired after 60s`);
|
|
3913
|
+
}
|
|
3914
|
+
}, 60000);
|
|
3915
|
+
}, this.maxOperationTimeout);
|
|
3821
3916
|
}
|
|
3822
3917
|
|
|
3823
|
-
|
|
3824
|
-
|
|
3825
|
-
|
|
3918
|
+
try {
|
|
3919
|
+
// Use only the last step's text as the final answer.
|
|
3920
|
+
// result.text concatenates ALL steps (including intermediate planning text),
|
|
3921
|
+
// but the user should only see the final answer from the last step.
|
|
3922
|
+
const steps = await result.steps;
|
|
3923
|
+
let finalText;
|
|
3924
|
+
if (steps && steps.length > 1) {
|
|
3925
|
+
// Multi-step: use last step's text (the actual answer after tool calls)
|
|
3926
|
+
const lastStepText = steps[steps.length - 1].text;
|
|
3927
|
+
finalText = lastStepText || await result.text;
|
|
3928
|
+
} else {
|
|
3929
|
+
finalText = await result.text;
|
|
3930
|
+
}
|
|
3826
3931
|
|
|
3827
|
-
|
|
3828
|
-
|
|
3829
|
-
|
|
3830
|
-
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
3831
|
-
}
|
|
3932
|
+
if (this.debug) {
|
|
3933
|
+
console.log(`[DEBUG] streamText completed: ${steps?.length || 0} steps, finalText=${finalText?.length || 0} chars`);
|
|
3934
|
+
}
|
|
3832
3935
|
|
|
3833
|
-
|
|
3936
|
+
// Record final token usage
|
|
3937
|
+
const usage = await result.usage;
|
|
3938
|
+
if (usage) {
|
|
3939
|
+
this.tokenCounter.recordUsage(usage, result.experimental_providerMetadata);
|
|
3940
|
+
}
|
|
3941
|
+
|
|
3942
|
+
return { finalText, result };
|
|
3943
|
+
} finally {
|
|
3944
|
+
// Clean up graceful timeout timers
|
|
3945
|
+
if (gracefulTimeoutId) clearTimeout(gracefulTimeoutId);
|
|
3946
|
+
if (hardAbortTimeoutId) clearTimeout(hardAbortTimeoutId);
|
|
3947
|
+
}
|
|
3834
3948
|
};
|
|
3835
3949
|
|
|
3836
3950
|
let aiResult;
|
|
@@ -3875,6 +3989,58 @@ Double-check your response based on the criteria above. If everything looks good
|
|
|
3875
3989
|
finalResult = aiResult.finalText;
|
|
3876
3990
|
}
|
|
3877
3991
|
|
|
3992
|
+
// Graceful timeout handling: ensure the response clearly indicates
|
|
3993
|
+
// the research was interrupted and may be incomplete.
|
|
3994
|
+
if (gracefulTimeoutState.triggered) {
|
|
3995
|
+
const timeoutNotice = '**Note: This response was generated under a time constraint. The research may be incomplete, and some planned searches or analysis steps were not completed.**\n\n';
|
|
3996
|
+
|
|
3997
|
+
if (!finalResult || finalResult === 'I was unable to complete your request due to reaching the maximum number of tool iterations.') {
|
|
3998
|
+
// Wind-down produced empty text — try to collect useful content.
|
|
3999
|
+
// Some models (e.g., Gemini) return finishReason:'other' with empty text
|
|
4000
|
+
// when forced from tool-calling to text-only mode mid-task.
|
|
4001
|
+
try {
|
|
4002
|
+
// Try result.text (concatenation of all step texts)
|
|
4003
|
+
const allText = await aiResult.result.text;
|
|
4004
|
+
if (allText && allText.trim()) {
|
|
4005
|
+
finalResult = timeoutNotice + allText;
|
|
4006
|
+
if (this.debug) {
|
|
4007
|
+
console.log(`[DEBUG] Graceful timeout: using concatenated step text (${allText.length} chars)`);
|
|
4008
|
+
}
|
|
4009
|
+
} else {
|
|
4010
|
+
// Last resort: collect tool result summaries as partial information
|
|
4011
|
+
const steps = await aiResult.result.steps;
|
|
4012
|
+
const toolSummaries = [];
|
|
4013
|
+
for (const step of (steps || [])) {
|
|
4014
|
+
if (step.toolResults?.length > 0) {
|
|
4015
|
+
for (const tr of step.toolResults) {
|
|
4016
|
+
const resultText = typeof tr.result === 'string' ? tr.result : JSON.stringify(tr.result);
|
|
4017
|
+
if (resultText && resultText.length > 0 && resultText.length < 5000) {
|
|
4018
|
+
toolSummaries.push(resultText.substring(0, 2000));
|
|
4019
|
+
}
|
|
4020
|
+
}
|
|
4021
|
+
}
|
|
4022
|
+
}
|
|
4023
|
+
if (toolSummaries.length > 0) {
|
|
4024
|
+
finalResult = `${timeoutNotice}The operation timed out before a complete answer could be generated. Here is the partial information gathered:\n\n${toolSummaries.join('\n\n---\n\n')}`;
|
|
4025
|
+
if (this.debug) {
|
|
4026
|
+
console.log(`[DEBUG] Graceful timeout: built fallback from ${toolSummaries.length} tool results`);
|
|
4027
|
+
}
|
|
4028
|
+
} else {
|
|
4029
|
+
finalResult = 'The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.';
|
|
4030
|
+
}
|
|
4031
|
+
}
|
|
4032
|
+
} catch (e) {
|
|
4033
|
+
if (this.debug) {
|
|
4034
|
+
console.log(`[DEBUG] Graceful timeout fallback error: ${e.message}`);
|
|
4035
|
+
}
|
|
4036
|
+
finalResult = 'The operation timed out before enough information could be gathered to provide an answer. Please try again with a simpler query or increase the timeout.';
|
|
4037
|
+
}
|
|
4038
|
+
} else {
|
|
4039
|
+
// Model produced text during wind-down — prepend the timeout notice
|
|
4040
|
+
finalResult = timeoutNotice + finalResult;
|
|
4041
|
+
}
|
|
4042
|
+
}
|
|
4043
|
+
|
|
3878
4044
|
// Update currentMessages from the result for history storage
|
|
3879
4045
|
// The SDK manages the full message history internally
|
|
3880
4046
|
const resultMessages = await aiResult.result.response?.messages;
|