opencode-auto-resume 1.1.2 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -0
- package/dist/index.js +560 -80
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -73,6 +73,81 @@ _Motivated by:_
|
|
|
73
73
|
|
|
74
74
|
---
|
|
75
75
|
|
|
76
|
+
### Streaming failure recovery
|
|
77
|
+
|
|
78
|
+
The AI provider's streaming response can fail mid-stream (connection reset, timeout, socket close). When the provider reports an error whose **name** matches `streamingFailureErrorNames` (exact, case-sensitive) or whose **message** matches `streamingFailureMessagePatterns` (regex, case-insensitive), the plugin arms a deferred recovery instead of relying only on the generic stall timeout. Invalid regex patterns fall back to substring matching.
|
|
79
|
+
|
|
80
|
+
**Default error names:**
|
|
81
|
+
- `ProviderError`, `APIError`, `StreamError`, `ConnectionError`, `TimeoutError`
|
|
82
|
+
|
|
83
|
+
**Default message patterns:**
|
|
84
|
+
- `streaming response failed`, `stream.*fail`, `connection.*reset`, `connection.*closed`
|
|
85
|
+
|
|
86
|
+
#### Recovery behavior
|
|
87
|
+
|
|
88
|
+
1. **Detection**: `session.error` is classified via `isStreamingFailure()` against the configured error names and message patterns
|
|
89
|
+
2. **State transition**: the session's `pendingRecovery` flag is armed with the error name (`pendingRecoveryReason`) and timestamp (`pendingRecoveryAt`)
|
|
90
|
+
3. **Recovery attempt**: once the session is idle and the backoff delay has elapsed, the timer loop sends a recovery prompt
|
|
91
|
+
4. **Watchdog**: if the session is still not busy 3 seconds after the prompt, the recovery is retried (up to `maxRecoveryRetries`) with exponential backoff
|
|
92
|
+
5. **Escalation**: when retries are exhausted, the plugin aborts the session and resumes it (`abort+resume`); `gaveUp` is set if that also fails
|
|
93
|
+
|
|
94
|
+
#### Configuration
|
|
95
|
+
|
|
96
|
+
Add to your plugin options (in `opencode.jsonc`):
|
|
97
|
+
|
|
98
|
+
```json
|
|
99
|
+
{
|
|
100
|
+
"streamingFailureErrorNames": ["ProviderError", "APIError", "StreamError", "ConnectionError", "TimeoutError"],
|
|
101
|
+
"streamingFailureMessagePatterns": ["streaming response failed", "stream.*fail", "connection.*reset", "connection.*closed"],
|
|
102
|
+
"maxRecoveryRetries": 2,
|
|
103
|
+
"baseBackoffMs": 1000,
|
|
104
|
+
"maxBackoffMs": 8000
|
|
105
|
+
}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
#### State machine addition
|
|
109
|
+
|
|
110
|
+
New per-session recovery fields added to the state machine:
|
|
111
|
+
- `pendingRecovery` — failure detected, recovery armed
|
|
112
|
+
- `pendingRecoveryReason` — error name that triggered the recovery
|
|
113
|
+
- `pendingRecoveryAt` — detection timestamp (backoff anchor)
|
|
114
|
+
- `recoveryAttempts` — recovery attempt counter
|
|
115
|
+
- `watchdogRetryGuard` — watchdog retry in progress (keeps the recovery armed)
|
|
116
|
+
|
|
117
|
+
Recovery chain: `pendingRecovery` → recovery attempt → `recoveryAttempts` retry → `abort+resume` → `gaveUp`.
|
|
118
|
+
|
|
119
|
+
See [Recovery Flow Documentation](docs/architecture/recovery-flow.md) for the full state machine.
|
|
120
|
+
|
|
121
|
+
#### Example scenario
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
1. AI provider starts streaming response
|
|
125
|
+
2. Network interruption causes "connection reset" error mid-stream
|
|
126
|
+
3. System detects "ConnectionError" matches streamingFailureErrorNames
|
|
127
|
+
4. Session's pendingRecovery flag is armed (reason=ConnectionError)
|
|
128
|
+
5. Session goes idle; timer loop waits until the backoff delay has elapsed
|
|
129
|
+
6. Recovery prompt sent (recoveryAttempts=1)
|
|
130
|
+
7. Success → session busy → recovery flags cleared
|
|
131
|
+
Still not busy after 3s → watchdog retry (attempt 2/2)
|
|
132
|
+
Still not busy after 3s → maxRecoveryRetries reached → abort + resume
|
|
133
|
+
Abort+continue fails → gaveUp
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
#### Configuration reference
|
|
137
|
+
|
|
138
|
+
| Option | Type | Default | Description |
|
|
139
|
+
|--------|------|---------|-------------|
|
|
140
|
+
| `streamingFailureErrorNames` | `string[]` | `["ProviderError","APIError","StreamError","ConnectionError","TimeoutError"]` | Error names that indicate a streaming failure (exact, case-sensitive) |
|
|
141
|
+
| `streamingFailureMessagePatterns` | `string[]` | `["streaming response failed","stream.*fail","connection.*reset","connection.*closed"]` | Regex patterns matching streaming failure messages (case-insensitive) |
|
|
142
|
+
| `maxRecoveryRetries` | `number` | `2` | Maximum streaming-failure recovery attempts before abort+resume escalation |
|
|
143
|
+
| `baseBackoffMs` | `number` | `1000` | Initial backoff delay in milliseconds |
|
|
144
|
+
| `maxBackoffMs` | `number` | `8000` | Maximum backoff delay cap in milliseconds |
|
|
145
|
+
|
|
146
|
+
_Motivated by:_
|
|
147
|
+
- [EPIC: Streaming Failure Recovery](docs/EPIC-Streaming-Recovery-OpenCode-Auto-Resume-v3.md) — recovery requests not consistently creating new assistant executions after mid-stream failures
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
76
151
|
### Active-tool safety guard
|
|
77
152
|
|
|
78
153
|
Before **any** abort, the plugin calls `checkSessionHasActiveTool()` to verify the session isn't mid-tool-execution. If a tool is running, the abort is skipped. This prevents the plugin from killing a long-running build, test suite, or command — even when it looks like a stall.
|
|
@@ -256,6 +331,11 @@ bun run build
|
|
|
256
331
|
| `subagentWaitMs` | `15000` | Wait before treating orphan parent as stuck |
|
|
257
332
|
| `loopMaxContinues` | `3` | Continues in window before triggering abort |
|
|
258
333
|
| `loopWindowMs` | `600000` | Hallucination loop detection window (10 min) |
|
|
334
|
+
| `streamingFailureErrorNames` | `["ProviderError","APIError","StreamError","ConnectionError","TimeoutError"]` | Error names that classify as streaming failures (exact match) |
|
|
335
|
+
| `streamingFailureMessagePatterns` | `["streaming response failed","stream.*fail","connection.*reset","connection.*closed"]` | Regex patterns (case-insensitive) in error messages indicating streaming failure |
|
|
336
|
+
| `maxRecoveryRetries` | `2` | Max streaming-failure recovery attempts before abort+resume escalation |
|
|
337
|
+
|
|
338
|
+
Message patterns are matched case-insensitively. Error names use exact match.
|
|
259
339
|
|
|
260
340
|
### Internal constants (not configurable)
|
|
261
341
|
|
package/dist/index.js
CHANGED
|
@@ -12345,8 +12345,24 @@ var DEFAULT_SUBAGENT_WAIT_MS = 15000;
|
|
|
12345
12345
|
var ABORT_CONTINUE_DELAY_MS = 2000;
|
|
12346
12346
|
var DEFAULT_LOOP_MAX_CONTINUES = 3;
|
|
12347
12347
|
var DEFAULT_LOOP_WINDOW_MS = 10 * 60000;
|
|
12348
|
-
var
|
|
12349
|
-
var
|
|
12348
|
+
var DEFAULT_TOOL_TEXT_CHECK_DELAY_MS = 3000;
|
|
12349
|
+
var DEFAULT_MAX_RECOVERY_RETRIES = 2;
|
|
12350
|
+
var DEFAULT_MIN_ACTIVITY_GAP_MS = 1000;
|
|
12351
|
+
var DEFAULT_WARMUP_MS = 15000;
|
|
12352
|
+
var DEFAULT_DEBUG = false;
|
|
12353
|
+
var DEFAULT_STREAMING_FAILURE_ERROR_NAMES = [
|
|
12354
|
+
"ProviderError",
|
|
12355
|
+
"APIError",
|
|
12356
|
+
"StreamError",
|
|
12357
|
+
"ConnectionError",
|
|
12358
|
+
"TimeoutError"
|
|
12359
|
+
];
|
|
12360
|
+
var DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS = [
|
|
12361
|
+
"streaming response failed",
|
|
12362
|
+
"stream.*fail",
|
|
12363
|
+
"connection.*reset",
|
|
12364
|
+
"connection.*closed"
|
|
12365
|
+
];
|
|
12350
12366
|
var MAX_IDLE_SESSIONS = 50;
|
|
12351
12367
|
var IDLE_CLEANUP_MS = 10 * 60000;
|
|
12352
12368
|
var SESSION_DISCOVERY_INTERVAL_MS = 60000;
|
|
@@ -12388,13 +12404,17 @@ var READY_TO_CONTINUE_PATTERNS = [
|
|
|
12388
12404
|
/will continue with task/i,
|
|
12389
12405
|
/moving on to task/i
|
|
12390
12406
|
];
|
|
12407
|
+
function stripCodeBlocks(text) {
|
|
12408
|
+
return text.replace(/```[\s\S]*?```/g, "").replace(/`[^`\n]+`/g, "");
|
|
12409
|
+
}
|
|
12391
12410
|
function containsToolCallAsText(text) {
|
|
12392
12411
|
if (text.length <= 10)
|
|
12393
12412
|
return false;
|
|
12394
|
-
|
|
12413
|
+
const stripped = stripCodeBlocks(text);
|
|
12414
|
+
if (TOOL_TEXT_PATTERNS.some((pat) => pat.test(stripped)))
|
|
12395
12415
|
return true;
|
|
12396
12416
|
for (const { open, close } of TRUNCATED_XML_PATTERNS) {
|
|
12397
|
-
if (open.test(
|
|
12417
|
+
if (open.test(stripped) && !close.test(stripped))
|
|
12398
12418
|
return true;
|
|
12399
12419
|
}
|
|
12400
12420
|
return false;
|
|
@@ -12429,17 +12449,97 @@ function containsDoneClaimPattern(text) {
|
|
|
12429
12449
|
`);
|
|
12430
12450
|
return DONE_CLAIM_PATTERNS.some((pat) => pat.test(lastLines));
|
|
12431
12451
|
}
|
|
12452
|
+
function isStreamingFailure(errorName, errorMessage, errorNames = DEFAULT_STREAMING_FAILURE_ERROR_NAMES, messagePatterns = DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS) {
|
|
12453
|
+
if (!errorName && !errorMessage)
|
|
12454
|
+
return false;
|
|
12455
|
+
if (errorName && errorNames.includes(errorName)) {
|
|
12456
|
+
return true;
|
|
12457
|
+
}
|
|
12458
|
+
if (errorMessage) {
|
|
12459
|
+
const lowerMessage = errorMessage.toLowerCase();
|
|
12460
|
+
for (const pattern of messagePatterns) {
|
|
12461
|
+
try {
|
|
12462
|
+
if (new RegExp(pattern, "i").test(lowerMessage))
|
|
12463
|
+
return true;
|
|
12464
|
+
} catch {
|
|
12465
|
+
if (lowerMessage.includes(pattern.toLowerCase()))
|
|
12466
|
+
return true;
|
|
12467
|
+
}
|
|
12468
|
+
}
|
|
12469
|
+
}
|
|
12470
|
+
return false;
|
|
12471
|
+
}
|
|
12472
|
+
function getLastAssistantError(messages) {
|
|
12473
|
+
for (let i = messages.length - 1;i >= 0; i--) {
|
|
12474
|
+
const msg = messages[i];
|
|
12475
|
+
const role = msg.role ?? msg.info?.role;
|
|
12476
|
+
if (role !== "assistant")
|
|
12477
|
+
continue;
|
|
12478
|
+
const info = msg.info;
|
|
12479
|
+
const err = msg.error ?? info?.error;
|
|
12480
|
+
if (err) {
|
|
12481
|
+
const data = err.data;
|
|
12482
|
+
const name = err.name ?? "";
|
|
12483
|
+
const message = data?.message ?? err.message ?? "";
|
|
12484
|
+
return { name, message };
|
|
12485
|
+
}
|
|
12486
|
+
const parts = msg.parts;
|
|
12487
|
+
if (parts) {
|
|
12488
|
+
for (let j = parts.length - 1;j >= 0; j--) {
|
|
12489
|
+
const part = parts[j];
|
|
12490
|
+
if (part.type !== "retry")
|
|
12491
|
+
continue;
|
|
12492
|
+
const partErr = part.error;
|
|
12493
|
+
if (!partErr)
|
|
12494
|
+
continue;
|
|
12495
|
+
const data = partErr.data;
|
|
12496
|
+
const name = partErr.name ?? "";
|
|
12497
|
+
const message = data?.message ?? partErr.message ?? "";
|
|
12498
|
+
return { name, message };
|
|
12499
|
+
}
|
|
12500
|
+
}
|
|
12501
|
+
}
|
|
12502
|
+
return null;
|
|
12503
|
+
}
|
|
12504
|
+
function backoffMs(attempt, baseBackoffMs = DEFAULT_BASE_BACKOFF_MS, maxBackoffMs = DEFAULT_MAX_BACKOFF_MS) {
|
|
12505
|
+
return Math.min(baseBackoffMs * Math.pow(2, attempt - 1), maxBackoffMs);
|
|
12506
|
+
}
|
|
12507
|
+
function containsActionIntent(text) {
|
|
12508
|
+
if (text.length <= 15)
|
|
12509
|
+
return false;
|
|
12510
|
+
const cleaned = text.replace(/<[a-zA-Z/?][^>]*>/g, "").trim();
|
|
12511
|
+
const lines = cleaned.split(`
|
|
12512
|
+
`);
|
|
12513
|
+
let lastLine = "";
|
|
12514
|
+
for (let i = lines.length - 1;i >= 0; i--) {
|
|
12515
|
+
if (lines[i].trim().length > 0) {
|
|
12516
|
+
lastLine = lines[i].trim();
|
|
12517
|
+
break;
|
|
12518
|
+
}
|
|
12519
|
+
}
|
|
12520
|
+
return lastLine.endsWith(":") && lastLine.length > 5 && lastLine.length < 500;
|
|
12521
|
+
}
|
|
12522
|
+
function isOpenTodo(t) {
|
|
12523
|
+
return t.status === "pending" || t.status === "in_progress";
|
|
12524
|
+
}
|
|
12525
|
+
function getOpenTodos(todos) {
|
|
12526
|
+
return todos.filter(isOpenTodo);
|
|
12527
|
+
}
|
|
12432
12528
|
function buildOpenTodosReminder(todos) {
|
|
12529
|
+
if (!Array.isArray(todos))
|
|
12530
|
+
return "continue";
|
|
12433
12531
|
const open = todos.filter((t) => t.status === "pending" || t.status === "in_progress");
|
|
12434
12532
|
if (open.length === 0)
|
|
12435
12533
|
return "continue";
|
|
12436
12534
|
const list = open.map((t, i) => `${i + 1}. [${t.status}] ${t.content}`).join(`
|
|
12437
12535
|
`);
|
|
12438
12536
|
const plural = open.length > 1 ? "s" : "";
|
|
12537
|
+
const taskWord = open.length > 1 ? "tasks" : "task";
|
|
12538
|
+
const thisWord = open.length > 1 ? "these" : "this";
|
|
12439
12539
|
return `You have ${open.length} unfinished task${plural}:
|
|
12440
12540
|
${list}
|
|
12441
12541
|
|
|
12442
|
-
Please continue working on
|
|
12542
|
+
Please continue working on ${thisWord} ${taskWord}.`;
|
|
12443
12543
|
}
|
|
12444
12544
|
var AutoResumePlugin = async (ctx, options) => {
|
|
12445
12545
|
const chunkTimeoutMs = options?.chunkTimeoutMs ?? DEFAULT_CHUNK_TIMEOUT_MS;
|
|
@@ -12451,6 +12551,23 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12451
12551
|
const subagentWaitMs = options?.subagentWaitMs ?? DEFAULT_SUBAGENT_WAIT_MS;
|
|
12452
12552
|
const loopMaxContinues = options?.loopMaxContinues ?? DEFAULT_LOOP_MAX_CONTINUES;
|
|
12453
12553
|
const loopWindowMs = options?.loopWindowMs ?? DEFAULT_LOOP_WINDOW_MS;
|
|
12554
|
+
const toolTextCheckDelayMs = options?.toolTextCheckDelayMs ?? DEFAULT_TOOL_TEXT_CHECK_DELAY_MS;
|
|
12555
|
+
const maxRecoveryRetries = options?.maxRecoveryRetries ?? DEFAULT_MAX_RECOVERY_RETRIES;
|
|
12556
|
+
const minActivityGapMs = options?.minActivityGapMs ?? DEFAULT_MIN_ACTIVITY_GAP_MS;
|
|
12557
|
+
const warmupMs = options?.warmupMs ?? DEFAULT_WARMUP_MS;
|
|
12558
|
+
const debug = options?.debug ?? DEFAULT_DEBUG;
|
|
12559
|
+
const streamingFailureErrorNames = options?.streamingFailureErrorNames ?? DEFAULT_STREAMING_FAILURE_ERROR_NAMES;
|
|
12560
|
+
const streamingFailureMessagePatterns = options?.streamingFailureMessagePatterns ?? DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS;
|
|
12561
|
+
const resumeOnActionIntent = options?.resumeOnActionIntent !== false;
|
|
12562
|
+
const continuePrompt = options?.continuePrompt ?? "continue";
|
|
12563
|
+
const actionIntentPrompt = options?.actionIntentPrompt ?? continuePrompt;
|
|
12564
|
+
const toolTextRecoveryPrompt = options?.toolTextRecoveryPrompt ?? TOOL_TEXT_RECOVERY_PROMPT;
|
|
12565
|
+
const thinkingToolRecoveryPrompt = options?.thinkingToolRecoveryPrompt ?? THINKING_TOOL_RECOVERY_PROMPT;
|
|
12566
|
+
const doneWithoutWorkPrompt = options?.doneWithoutWorkPrompt ?? DONE_WITHOUT_WORK_PROMPT;
|
|
12567
|
+
const dbg = (...args) => {
|
|
12568
|
+
if (debug)
|
|
12569
|
+
console.log("[debug]", ...args);
|
|
12570
|
+
};
|
|
12454
12571
|
const sessions = new Map;
|
|
12455
12572
|
let timer = null;
|
|
12456
12573
|
let discoveryTimer = null;
|
|
@@ -12482,6 +12599,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12482
12599
|
let w = sessions.get(sid);
|
|
12483
12600
|
if (!w) {
|
|
12484
12601
|
w = {
|
|
12602
|
+
createdAt: Date.now(),
|
|
12485
12603
|
lastActivityAt: Date.now(),
|
|
12486
12604
|
status: "unknown",
|
|
12487
12605
|
userCancelled: false,
|
|
@@ -12505,7 +12623,16 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12505
12623
|
toolLoopAttempts: 0,
|
|
12506
12624
|
isSubagent: false,
|
|
12507
12625
|
completionSignaled: false,
|
|
12508
|
-
todoNudgeAttempts: 0
|
|
12626
|
+
todoNudgeAttempts: 0,
|
|
12627
|
+
taskCompleteOverrides: 0,
|
|
12628
|
+
doneClaimNoTodosAttempts: 0,
|
|
12629
|
+
pendingTools: 0,
|
|
12630
|
+
pendingCommands: 0,
|
|
12631
|
+
pendingRecovery: false,
|
|
12632
|
+
pendingRecoveryReason: null,
|
|
12633
|
+
pendingRecoveryAt: 0,
|
|
12634
|
+
recoveryAttempts: 0,
|
|
12635
|
+
watchdogRetryGuard: false
|
|
12509
12636
|
};
|
|
12510
12637
|
sessions.set(sid, w);
|
|
12511
12638
|
}
|
|
@@ -12517,6 +12644,9 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12517
12644
|
w.lastActivityAt = Date.now();
|
|
12518
12645
|
}
|
|
12519
12646
|
}
|
|
12647
|
+
function hasInflightTools(w) {
|
|
12648
|
+
return w.pendingTools > 0 || w.pendingCommands > 0;
|
|
12649
|
+
}
|
|
12520
12650
|
function busyCount() {
|
|
12521
12651
|
let count = 0;
|
|
12522
12652
|
for (const [, w] of sessions) {
|
|
@@ -12563,9 +12693,6 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12563
12693
|
function short(sid) {
|
|
12564
12694
|
return sid.length > 12 ? `...${sid.slice(-8)}` : sid;
|
|
12565
12695
|
}
|
|
12566
|
-
function backoffMs(attempt) {
|
|
12567
|
-
return Math.min(baseBackoffMs * Math.pow(2, attempt - 1), maxBackoffMs);
|
|
12568
|
-
}
|
|
12569
12696
|
function cleanupIdleSessions() {
|
|
12570
12697
|
const now = Date.now();
|
|
12571
12698
|
const toDelete = [];
|
|
@@ -12600,12 +12727,52 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12600
12727
|
log("debug", `Cleaned up ${toDelete.length} idle session(s). Map size: ${sessions.size}`);
|
|
12601
12728
|
}
|
|
12602
12729
|
}
|
|
12730
|
+
function getPromptResponsePayload(result) {
|
|
12731
|
+
if (!result || typeof result !== "object")
|
|
12732
|
+
return null;
|
|
12733
|
+
const raw = result;
|
|
12734
|
+
const data = raw.data;
|
|
12735
|
+
if (data && typeof data === "object" && "parts" in data) {
|
|
12736
|
+
return data;
|
|
12737
|
+
}
|
|
12738
|
+
if ("parts" in raw) {
|
|
12739
|
+
return raw;
|
|
12740
|
+
}
|
|
12741
|
+
return null;
|
|
12742
|
+
}
|
|
12743
|
+
async function logPromptResponse(result, context) {
|
|
12744
|
+
const payload = getPromptResponsePayload(result);
|
|
12745
|
+
const isRetry = context.isRetry ?? false;
|
|
12746
|
+
if (!payload) {
|
|
12747
|
+
await log("debug", `${short(context.sessionId)} - session.prompt() response has unexpected structure: ${JSON.stringify({ sessionId: context.sessionId, isRetry, rawResponse: JSON.stringify(result) })}`);
|
|
12748
|
+
return;
|
|
12749
|
+
}
|
|
12750
|
+
const parts = Array.isArray(payload.parts) ? payload.parts : [];
|
|
12751
|
+
const partsCount = parts.length;
|
|
12752
|
+
const info = payload.info && typeof payload.info === "object" ? payload.info : undefined;
|
|
12753
|
+
const infoKeys = info ? Object.keys(info) : [];
|
|
12754
|
+
const hasParts = partsCount > 0;
|
|
12755
|
+
await log("debug", `${short(context.sessionId)} - session.prompt() response received: ${JSON.stringify({ sessionId: context.sessionId, isRetry, hasParts, partsCount, infoKeys, info })}`);
|
|
12756
|
+
if (partsCount === 0) {
|
|
12757
|
+
await log("warn", `${short(context.sessionId)} - session.prompt() returned empty parts array - possible stream initiation failure: ${JSON.stringify({ sessionId: context.sessionId, isRetry, responseInfo: info, partsCount })}`);
|
|
12758
|
+
}
|
|
12759
|
+
if (info && "error" in info) {
|
|
12760
|
+
await log("warn", `${short(context.sessionId)} - session.prompt() response.info contains error indicator: ${JSON.stringify({ sessionId: context.sessionId, isRetry, errorInfo: info.error })}`);
|
|
12761
|
+
}
|
|
12762
|
+
await log("debug", `${short(context.sessionId)} - session.prompt() raw response: ${JSON.stringify({ sessionId: context.sessionId, isRetry, rawResponse: JSON.stringify(result, null, 2) })}`);
|
|
12763
|
+
}
|
|
12603
12764
|
async function sendContinuePrompt(sid, text, w) {
|
|
12604
|
-
if (w.continuing) {
|
|
12765
|
+
if (w.continuing && !w.watchdogRetryGuard) {
|
|
12605
12766
|
await log("debug", `${short(sid)} - continue already in progress, skipping`);
|
|
12606
12767
|
return;
|
|
12607
12768
|
}
|
|
12769
|
+
if (!w.continuing)
|
|
12770
|
+
dbg(`State transition on ${short(sid)}: continuing=false -> true`);
|
|
12608
12771
|
w.continuing = true;
|
|
12772
|
+
if (w.watchdogRetryGuard) {
|
|
12773
|
+
w.pendingRecovery = true;
|
|
12774
|
+
}
|
|
12775
|
+
w.watchdogRetryGuard = false;
|
|
12609
12776
|
let agent;
|
|
12610
12777
|
let model;
|
|
12611
12778
|
try {
|
|
@@ -12636,7 +12803,8 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12636
12803
|
break;
|
|
12637
12804
|
}
|
|
12638
12805
|
}
|
|
12639
|
-
|
|
12806
|
+
dbg(`Recovery prompt sent to ${short(sid)}: prompt="${text.length > 80 ? `${text.slice(0, 80)}...` : text}", agent=${agent ?? "(default)"}, model=${model ? `${model.providerID}/${model.modelID}` : "(default)"}`);
|
|
12807
|
+
const response = await ctx.client.session.prompt({
|
|
12640
12808
|
path: { id: sid },
|
|
12641
12809
|
body: {
|
|
12642
12810
|
parts: [{ type: "text", text }],
|
|
@@ -12644,6 +12812,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12644
12812
|
model
|
|
12645
12813
|
}
|
|
12646
12814
|
});
|
|
12815
|
+
await logPromptResponse(response, { sessionId: sid });
|
|
12647
12816
|
await log("debug", `${short(sid)} - prompt sent with agent: ${agent ?? "(default)"}, model: ${model ? `${model.providerID}/${model.modelID}` : "(default)"}`);
|
|
12648
12817
|
recordContinue(sid);
|
|
12649
12818
|
w.lastRetryAt = Date.now();
|
|
@@ -12651,10 +12820,11 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12651
12820
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
12652
12821
|
await log("warn", `${short(sid)} - prompt failed: ${errMsg}`);
|
|
12653
12822
|
try {
|
|
12654
|
-
await ctx.client.session.prompt({
|
|
12823
|
+
const retryResponse = await ctx.client.session.prompt({
|
|
12655
12824
|
path: { id: sid },
|
|
12656
12825
|
body: { parts: [{ type: "text", text }], agent, model }
|
|
12657
12826
|
});
|
|
12827
|
+
await logPromptResponse(retryResponse, { sessionId: sid, isRetry: true });
|
|
12658
12828
|
recordContinue(sid);
|
|
12659
12829
|
w.lastRetryAt = Date.now();
|
|
12660
12830
|
} catch (retryErr) {
|
|
@@ -12663,6 +12833,8 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12663
12833
|
throw retryErr;
|
|
12664
12834
|
}
|
|
12665
12835
|
} finally {
|
|
12836
|
+
if (w.continuing)
|
|
12837
|
+
dbg(`State transition on ${short(sid)}: continuing=true -> false`);
|
|
12666
12838
|
w.continuing = false;
|
|
12667
12839
|
w.todoCheckAttempts = 0;
|
|
12668
12840
|
if (w.toolTextTimer) {
|
|
@@ -12672,9 +12844,53 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12672
12844
|
}
|
|
12673
12845
|
setTimeout(async () => {
|
|
12674
12846
|
if (w.status !== "busy") {
|
|
12675
|
-
|
|
12847
|
+
if (w.pendingRecovery) {
|
|
12848
|
+
if (w.recoveryAttempts < maxRecoveryRetries) {
|
|
12849
|
+
w.recoveryAttempts++;
|
|
12850
|
+
dbg(`State transition on ${short(sid)}: recoveryAttempts=${w.recoveryAttempts - 1} -> ${w.recoveryAttempts}`);
|
|
12851
|
+
w.watchdogRetryGuard = true;
|
|
12852
|
+
await log("warn", `${short(sid)} - recovery attempt ${w.recoveryAttempts}/${maxRecoveryRetries} after prompt timeout`);
|
|
12853
|
+
await log("warn", `Recovery failed on ${short(sid)} - session still ${w.status}: attempt=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, nextAction=retry`);
|
|
12854
|
+
dbg(`Watchdog check on ${short(sid)}: status=${w.status}, recoveryAttempts=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, watchdogLatencyMs=${Date.now() - w.lastRetryAt} -> RETRY`);
|
|
12855
|
+
const retryBackoffMs = backoffMs(w.recoveryAttempts, baseBackoffMs, maxBackoffMs);
|
|
12856
|
+
await log("info", `Retrying recovery on ${short(sid)}: attempt=${w.recoveryAttempts}, backoffMs=${retryBackoffMs}`);
|
|
12857
|
+
dbg(`Retrying recovery on ${short(sid)}: recoveryAttempts=${w.recoveryAttempts}, backoffMs=${retryBackoffMs}, pendingRecoveryReason=${w.pendingRecoveryReason}`);
|
|
12858
|
+
try {
|
|
12859
|
+
await sendContinuePrompt(sid, continuePrompt, w);
|
|
12860
|
+
} catch (err) {
|
|
12861
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
12862
|
+
await log("warn", `${short(sid)} - recovery retry failed: ${errMsg}`);
|
|
12863
|
+
w.recoveryAttempts = 0;
|
|
12864
|
+
}
|
|
12865
|
+
w.watchdogRetryGuard = false;
|
|
12866
|
+
} else {
|
|
12867
|
+
dbg(`Pending recovery cleared on ${short(sid)}: reason=recovery-attempt`);
|
|
12868
|
+
w.pendingRecovery = false;
|
|
12869
|
+
await log("warn", `${short(sid)} - max recovery attempts (${maxRecoveryRetries}) reached, escalating to abort+resume`);
|
|
12870
|
+
await log("warn", `Recovery failed on ${short(sid)} - session still ${w.status}: attempt=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, nextAction=abort-resume`);
|
|
12871
|
+
await log("warn", `Escalating to abort+resume on ${short(sid)}: attempt=${w.recoveryAttempts}`);
|
|
12872
|
+
dbg(`Watchdog check on ${short(sid)}: status=${w.status}, recoveryAttempts=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, watchdogLatencyMs=${Date.now() - w.lastRetryAt} -> ABORT_RESUME`);
|
|
12873
|
+
const resumed = await tryAbortAndResume(sid, w);
|
|
12874
|
+
if (!resumed && !w.aborting) {
|
|
12875
|
+
await log("warn", `Recovery exhausted on ${short(sid)}: attempts=${w.recoveryAttempts}, lastError=abort+resume failed`);
|
|
12876
|
+
dbg(`Watchdog check on ${short(sid)}: status=${w.status} -> GAVE_UP`);
|
|
12877
|
+
if (w.pendingRecoveryAt > 0) {
|
|
12878
|
+
dbg(`Total recovery cycle on ${short(sid)} (failed): totalCycleMs=${Date.now() - w.pendingRecoveryAt}`);
|
|
12879
|
+
}
|
|
12880
|
+
}
|
|
12881
|
+
}
|
|
12882
|
+
} else {
|
|
12883
|
+
await log("warn", `${short(sid)} - prompt sent >${toolTextCheckDelayMs / 1000}s ago but session is still ${w.status}`);
|
|
12884
|
+
}
|
|
12885
|
+
} else {
|
|
12886
|
+
const elapsedMs = Date.now() - w.lastRetryAt;
|
|
12887
|
+
await log("info", `Recovery successful on ${short(sid)}: elapsedMs=${elapsedMs}`);
|
|
12888
|
+
dbg(`Watchdog check on ${short(sid)}: status=busy, elapsedMs=${elapsedMs} -> SUCCESS`);
|
|
12889
|
+
if (w.pendingRecoveryAt > 0) {
|
|
12890
|
+
dbg(`Total recovery cycle on ${short(sid)}: totalCycleMs=${Date.now() - w.pendingRecoveryAt}`);
|
|
12891
|
+
}
|
|
12676
12892
|
}
|
|
12677
|
-
},
|
|
12893
|
+
}, toolTextCheckDelayMs);
|
|
12678
12894
|
}
|
|
12679
12895
|
function extractMessages(response) {
|
|
12680
12896
|
if (Array.isArray(response))
|
|
@@ -12838,13 +13054,14 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12838
13054
|
function resetSessionFlags(w) {
|
|
12839
13055
|
w.userCancelled = false;
|
|
12840
13056
|
w.resumeAttempts = 0;
|
|
13057
|
+
w.pendingTools = 0;
|
|
13058
|
+
w.pendingCommands = 0;
|
|
12841
13059
|
w.gaveUp = false;
|
|
12842
13060
|
w.orphanWatchStartAt = null;
|
|
12843
13061
|
w.aborting = false;
|
|
12844
13062
|
w.toolTextRecovered = false;
|
|
12845
13063
|
w.toolTextAttempts = 0;
|
|
12846
13064
|
w.completionSignaled = false;
|
|
12847
|
-
w.todoNudgeAttempts = 0;
|
|
12848
13065
|
w.continueTimestamps = [];
|
|
12849
13066
|
w.idleSince = null;
|
|
12850
13067
|
w.continuing = false;
|
|
@@ -12857,12 +13074,18 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12857
13074
|
clearTimeout(w.toolTextTimer);
|
|
12858
13075
|
w.toolTextTimer = null;
|
|
12859
13076
|
}
|
|
13077
|
+
w.pendingRecovery = false;
|
|
13078
|
+
w.pendingRecoveryReason = null;
|
|
13079
|
+
w.pendingRecoveryAt = 0;
|
|
13080
|
+
w.recoveryAttempts = 0;
|
|
13081
|
+
w.watchdogRetryGuard = false;
|
|
12860
13082
|
}
|
|
12861
13083
|
function resetIdleFlags(w) {
|
|
12862
|
-
w.userCancelled = false;
|
|
12863
13084
|
w.aborting = false;
|
|
12864
13085
|
w.orphanWatchStartAt = null;
|
|
12865
13086
|
w.idleSince = Date.now();
|
|
13087
|
+
w.pendingTools = 0;
|
|
13088
|
+
w.pendingCommands = 0;
|
|
12866
13089
|
}
|
|
12867
13090
|
function detectPatternLoop(recentTools) {
|
|
12868
13091
|
if (recentTools.length < 6)
|
|
@@ -12909,9 +13132,10 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12909
13132
|
if (w.checkingToolText)
|
|
12910
13133
|
return;
|
|
12911
13134
|
w.checkingToolText = true;
|
|
13135
|
+
dbg(`checkForToolCallAsText called for ${short(sid)}, userCancelled=${w.userCancelled}, toolTextRecovered=${w.toolTextRecovered}, toolTextAttempts=${w.toolTextAttempts}`);
|
|
12912
13136
|
if (w.toolTextAttempts > 0) {
|
|
12913
13137
|
const elapsed = Date.now() - w.lastRetryAt;
|
|
12914
|
-
const requiredBackoff = backoffMs(w.toolTextAttempts);
|
|
13138
|
+
const requiredBackoff = backoffMs(w.toolTextAttempts, baseBackoffMs, maxBackoffMs);
|
|
12915
13139
|
if (elapsed < requiredBackoff)
|
|
12916
13140
|
return;
|
|
12917
13141
|
}
|
|
@@ -12982,7 +13206,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12982
13206
|
}
|
|
12983
13207
|
} else {
|
|
12984
13208
|
const candidate = {
|
|
12985
|
-
prompt:
|
|
13209
|
+
prompt: continuePrompt,
|
|
12986
13210
|
source: "tool-use",
|
|
12987
13211
|
priority: 1
|
|
12988
13212
|
};
|
|
@@ -13009,7 +13233,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13009
13233
|
}
|
|
13010
13234
|
}
|
|
13011
13235
|
const candidate = {
|
|
13012
|
-
prompt: isReasoning ?
|
|
13236
|
+
prompt: isReasoning ? thinkingToolRecoveryPrompt : toolTextRecoveryPrompt,
|
|
13013
13237
|
source: isReasoning ? "reasoning" : "text",
|
|
13014
13238
|
priority: 0
|
|
13015
13239
|
};
|
|
@@ -13019,13 +13243,13 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13019
13243
|
}
|
|
13020
13244
|
if (containsReadyToContinuePattern(text)) {
|
|
13021
13245
|
const todos = w.todos || [];
|
|
13022
|
-
const hasOpenTodos = todos.some(
|
|
13246
|
+
const hasOpenTodos = todos.some(isOpenTodo);
|
|
13023
13247
|
if (!hasOpenTodos && todos.length > 0) {
|
|
13024
13248
|
w.todoCheckAttempts++;
|
|
13025
13249
|
if (w.todoCheckAttempts >= 2) {
|
|
13026
13250
|
await log("info", `${short(sid)} - todos completed but agent hasn't closed them. Sending continue...`);
|
|
13027
13251
|
const candidate2 = {
|
|
13028
|
-
prompt:
|
|
13252
|
+
prompt: continuePrompt,
|
|
13029
13253
|
source: "todo-completed-continue",
|
|
13030
13254
|
priority: 1
|
|
13031
13255
|
};
|
|
@@ -13038,7 +13262,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13038
13262
|
continue;
|
|
13039
13263
|
}
|
|
13040
13264
|
const candidate = {
|
|
13041
|
-
prompt: containsDoneClaimPattern(text) ?
|
|
13265
|
+
prompt: containsDoneClaimPattern(text) ? doneWithoutWorkPrompt : continuePrompt,
|
|
13042
13266
|
source: containsDoneClaimPattern(text) ? "done-claim" : "ready-to-continue",
|
|
13043
13267
|
priority: 1
|
|
13044
13268
|
};
|
|
@@ -13048,14 +13272,42 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13048
13272
|
}
|
|
13049
13273
|
if (!bestCandidate && containsDoneClaimPattern(text)) {
|
|
13050
13274
|
const todos = w.todos || [];
|
|
13051
|
-
const hasOpenTodos = todos.some(
|
|
13275
|
+
const hasOpenTodos = todos.some(isOpenTodo);
|
|
13052
13276
|
if (hasOpenTodos) {
|
|
13053
13277
|
await log("info", `${short(sid)} - model claims done but todos remain open. Sending recovery prompt...`);
|
|
13054
13278
|
bestCandidate = {
|
|
13055
|
-
prompt:
|
|
13279
|
+
prompt: doneWithoutWorkPrompt,
|
|
13056
13280
|
source: "done-claim-no-emoji",
|
|
13057
13281
|
priority: 1
|
|
13058
13282
|
};
|
|
13283
|
+
} else if (w.doneClaimNoTodosAttempts < maxRetries) {
|
|
13284
|
+
await log("info", `${short(sid)} - model claims done with no open todos. Sending verification prompt (attempt ${w.doneClaimNoTodosAttempts + 1}/${maxRetries})...`);
|
|
13285
|
+
bestCandidate = {
|
|
13286
|
+
prompt: doneWithoutWorkPrompt,
|
|
13287
|
+
source: "done-claim-no-todos",
|
|
13288
|
+
priority: 1
|
|
13289
|
+
};
|
|
13290
|
+
}
|
|
13291
|
+
}
|
|
13292
|
+
}
|
|
13293
|
+
}
|
|
13294
|
+
if (resumeOnActionIntent) {
|
|
13295
|
+
const lastAssistantMsg = messages.slice().reverse().find((m) => (m.role ?? m.info?.role) === "assistant");
|
|
13296
|
+
if (lastAssistantMsg) {
|
|
13297
|
+
let lastAssistantText = "";
|
|
13298
|
+
for (const part of lastAssistantMsg.parts || []) {
|
|
13299
|
+
lastAssistantText += (part.text ?? "") + `
|
|
13300
|
+
`;
|
|
13301
|
+
}
|
|
13302
|
+
if (containsActionIntent(lastAssistantText)) {
|
|
13303
|
+
dbg(`ACTION INTENT DETECTED in checkForToolCallAsText for ${short(sid)}`);
|
|
13304
|
+
const candidate = {
|
|
13305
|
+
prompt: actionIntentPrompt,
|
|
13306
|
+
source: "action-intent",
|
|
13307
|
+
priority: 2
|
|
13308
|
+
};
|
|
13309
|
+
if (!bestCandidate || candidate.priority < bestCandidate.priority) {
|
|
13310
|
+
bestCandidate = candidate;
|
|
13059
13311
|
}
|
|
13060
13312
|
}
|
|
13061
13313
|
}
|
|
@@ -13074,10 +13326,10 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13074
13326
|
}
|
|
13075
13327
|
if (!bestCandidate) {
|
|
13076
13328
|
const todos = w.todos || [];
|
|
13077
|
-
const hasOpenTodos = todos.some(
|
|
13329
|
+
const hasOpenTodos = todos.some(isOpenTodo);
|
|
13078
13330
|
if (hasOpenTodos && busyCount() === 0) {
|
|
13079
13331
|
const reminder = buildOpenTodosReminder(todos);
|
|
13080
|
-
await log("info", `${short(sid)} - no activity detected but todos remain open (${todos
|
|
13332
|
+
await log("info", `${short(sid)} - no activity detected but todos remain open (${getOpenTodos(todos).length} tasks). Sending reminder...`);
|
|
13081
13333
|
bestCandidate = {
|
|
13082
13334
|
prompt: reminder,
|
|
13083
13335
|
source: "idle-with-open-todos-reminder",
|
|
@@ -13090,23 +13342,30 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13090
13342
|
if (!bestCandidate)
|
|
13091
13343
|
return;
|
|
13092
13344
|
const isOpenTodosReminder = bestCandidate.source === "idle-with-open-todos-reminder";
|
|
13345
|
+
const isDoneClaimNoTodos = bestCandidate.source === "done-claim-no-todos";
|
|
13093
13346
|
if (isOpenTodosReminder) {
|
|
13094
13347
|
if (w.todoNudgeAttempts >= maxRetries) {
|
|
13095
|
-
await log("info", `${short(sid)} - max open-todos nudges (${maxRetries}) reached
|
|
13348
|
+
await log("info", `${short(sid)} - max open-todos nudges (${maxRetries}) reached, waiting for activity`);
|
|
13096
13349
|
return;
|
|
13097
13350
|
}
|
|
13098
|
-
|
|
13351
|
+
} else if (isDoneClaimNoTodos) {
|
|
13352
|
+
w.doneClaimNoTodosAttempts++;
|
|
13099
13353
|
} else {
|
|
13100
13354
|
w.toolTextRecovered = true;
|
|
13101
13355
|
w.toolTextAttempts++;
|
|
13102
13356
|
}
|
|
13103
|
-
|
|
13357
|
+
const attemptNum = isOpenTodosReminder ? w.todoNudgeAttempts : isDoneClaimNoTodos ? w.doneClaimNoTodosAttempts : w.toolTextAttempts;
|
|
13358
|
+
await log("info", `${bestCandidate.source} detected on ${short(sid)}! ` + `Attempt ${attemptNum}/${maxRetries}. Sending recovery prompt...`);
|
|
13104
13359
|
const timeSinceActivity = Date.now() - w.lastActivityAt;
|
|
13105
|
-
if (timeSinceActivity <
|
|
13360
|
+
if (timeSinceActivity < minActivityGapMs) {
|
|
13106
13361
|
await log("info", `${short(sid)} - skipping ${bestCandidate.source}, session was active ${Math.round(timeSinceActivity / 1000)}s ago`);
|
|
13107
13362
|
return;
|
|
13108
13363
|
}
|
|
13109
13364
|
if (isHallucinationLoop(sid)) {
|
|
13365
|
+
if (hasInflightTools(w)) {
|
|
13366
|
+
await log("debug", `Session ${short(sid)} has ${w.pendingTools} tool(s) in-flight, skipping hallucination abort`);
|
|
13367
|
+
return;
|
|
13368
|
+
}
|
|
13110
13369
|
const hasActiveTool = await checkSessionHasActiveTool(sid);
|
|
13111
13370
|
if (hasActiveTool) {
|
|
13112
13371
|
await log("debug", `Session ${short(sid)} has active tool, skipping hallucination abort`);
|
|
@@ -13117,6 +13376,8 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13117
13376
|
} else {
|
|
13118
13377
|
try {
|
|
13119
13378
|
await sendContinuePrompt(sid, bestCandidate.prompt, w);
|
|
13379
|
+
if (isOpenTodosReminder)
|
|
13380
|
+
w.todoNudgeAttempts++;
|
|
13120
13381
|
await log("info", `${short(sid)} - ${bestCandidate.source} recovery sent (attempt ${w.toolTextAttempts})`);
|
|
13121
13382
|
} catch (err) {
|
|
13122
13383
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -13143,6 +13404,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13143
13404
|
try {
|
|
13144
13405
|
await ctx.client.session.abort({ path: { id: sid } });
|
|
13145
13406
|
await log("info", `${short(sid)} - abort OK`);
|
|
13407
|
+
dbg(`Abort succeeded on ${short(sid)}, waiting ${ABORT_CONTINUE_DELAY_MS}ms before continue prompt`);
|
|
13146
13408
|
} catch (err) {
|
|
13147
13409
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
13148
13410
|
await log("warn", `${short(sid)} - abort failed: ${errMsg}`);
|
|
@@ -13153,7 +13415,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13153
13415
|
if (w.status === "busy")
|
|
13154
13416
|
w.status = "idle";
|
|
13155
13417
|
try {
|
|
13156
|
-
await sendContinuePrompt(sid,
|
|
13418
|
+
await sendContinuePrompt(sid, continuePrompt, w);
|
|
13157
13419
|
await log("info", `${short(sid)} - abort+continue done`);
|
|
13158
13420
|
w.orphanWatchStartAt = null;
|
|
13159
13421
|
w.resumeAttempts++;
|
|
@@ -13173,10 +13435,15 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13173
13435
|
}
|
|
13174
13436
|
const now = Date.now();
|
|
13175
13437
|
const elapsedSinceRetry = now - w.lastRetryAt;
|
|
13176
|
-
const requiredBackoff = backoffMs(w.resumeAttempts);
|
|
13438
|
+
const requiredBackoff = backoffMs(w.resumeAttempts, baseBackoffMs, maxBackoffMs);
|
|
13177
13439
|
if (w.lastRetryAt > 0 && elapsedSinceRetry < requiredBackoff)
|
|
13178
13440
|
return false;
|
|
13179
13441
|
if (isHallucinationLoop(sid)) {
|
|
13442
|
+
if (hasInflightTools(w)) {
|
|
13443
|
+
await log("debug", `Session ${short(sid)} has ${w.pendingTools} tool(s) in-flight, skipping hallucination abort`);
|
|
13444
|
+
w.lastRetryAt = now;
|
|
13445
|
+
return false;
|
|
13446
|
+
}
|
|
13180
13447
|
const hasActiveTool = await checkSessionHasActiveTool(sid);
|
|
13181
13448
|
if (hasActiveTool) {
|
|
13182
13449
|
await log("debug", `Session ${short(sid)} has active tool, skipping hallucination abort`);
|
|
@@ -13190,7 +13457,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13190
13457
|
const idleSec = Math.round((now - w.lastActivityAt) / 1000);
|
|
13191
13458
|
await log("info", `${reason} on ${short(sid)} (${idleSec}s, retry ${w.resumeAttempts}/${maxRetries})`);
|
|
13192
13459
|
try {
|
|
13193
|
-
await sendContinuePrompt(sid, prompt ??
|
|
13460
|
+
await sendContinuePrompt(sid, prompt ?? continuePrompt, w);
|
|
13194
13461
|
await log("info", `${short(sid)} - retry sent`);
|
|
13195
13462
|
return true;
|
|
13196
13463
|
} catch (err) {
|
|
@@ -13250,6 +13517,11 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13250
13517
|
const orphanIdle = now - w.orphanWatchStartAt;
|
|
13251
13518
|
if (orphanIdle >= subagentWaitMs + gracePeriodMs) {
|
|
13252
13519
|
if (w.resumeAttempts < maxRetries) {
|
|
13520
|
+
if (hasInflightTools(w)) {
|
|
13521
|
+
await log("debug", `Parent ${short(sid)} has ${w.pendingTools} tool(s) in-flight, skipping orphan-watch abort`);
|
|
13522
|
+
w.orphanWatchStartAt = now;
|
|
13523
|
+
continue;
|
|
13524
|
+
}
|
|
13253
13525
|
const hasActiveTool = await checkSessionHasActiveTool(sid);
|
|
13254
13526
|
if (hasActiveTool) {
|
|
13255
13527
|
await log("debug", `Parent ${short(sid)} has active tool call, skipping orphan-watch abort`);
|
|
@@ -13278,6 +13550,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13278
13550
|
}
|
|
13279
13551
|
} else if (!w.gaveUp) {
|
|
13280
13552
|
w.gaveUp = true;
|
|
13553
|
+
dbg(`State transition on ${short(sid)}: gaveUp=false -> true`);
|
|
13281
13554
|
w.orphanWatchStartAt = null;
|
|
13282
13555
|
w.aborting = false;
|
|
13283
13556
|
log("warn", `${short(sid)} - orphan retries exhausted.`);
|
|
@@ -13296,6 +13569,11 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13296
13569
|
w.lastSubagentCheckAt = now;
|
|
13297
13570
|
continue;
|
|
13298
13571
|
}
|
|
13572
|
+
if (hasInflightTools(w)) {
|
|
13573
|
+
await log("debug", `Session ${short(sid)} has ${w.pendingTools} tool(s) in-flight, skipping abort check`);
|
|
13574
|
+
w.lastSubagentCheckAt = now;
|
|
13575
|
+
continue;
|
|
13576
|
+
}
|
|
13299
13577
|
const hasActiveTool = await checkSessionHasActiveTool(sid);
|
|
13300
13578
|
if (hasActiveTool) {
|
|
13301
13579
|
await log("debug", `Session ${short(sid)} has active tool call, skipping abort check`);
|
|
@@ -13320,18 +13598,79 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13320
13598
|
}
|
|
13321
13599
|
const idle = now - w.lastActivityAt;
|
|
13322
13600
|
if (idle >= chunkTimeoutMs + gracePeriodMs) {
|
|
13323
|
-
|
|
13324
|
-
|
|
13325
|
-
await log("debug", `Session ${short(sid)} has active tool call, skipping stall recovery`);
|
|
13601
|
+
if (hasInflightTools(w)) {
|
|
13602
|
+
await log("debug", `Session ${short(sid)} has ${w.pendingTools} tool(s) in-flight, skipping stall recovery`);
|
|
13326
13603
|
w.lastSubagentCheckAt = now;
|
|
13327
|
-
} else
|
|
13328
|
-
|
|
13329
|
-
|
|
13330
|
-
|
|
13331
|
-
|
|
13604
|
+
} else {
|
|
13605
|
+
const hasActiveTool = await checkSessionHasActiveTool(sid);
|
|
13606
|
+
if (hasActiveTool) {
|
|
13607
|
+
await log("debug", `Session ${short(sid)} has active tool call, skipping stall recovery`);
|
|
13608
|
+
w.lastSubagentCheckAt = now;
|
|
13609
|
+
} else if (w.resumeAttempts < maxRetries) {
|
|
13610
|
+
tryResume(sid, w, "Stream stall");
|
|
13611
|
+
} else if (!w.gaveUp) {
|
|
13612
|
+
w.gaveUp = true;
|
|
13613
|
+
dbg(`State transition on ${short(sid)}: gaveUp=false -> true`);
|
|
13614
|
+
log("warn", `${short(sid)} - all ${maxRetries} retries exhausted.`);
|
|
13615
|
+
}
|
|
13332
13616
|
}
|
|
13333
13617
|
}
|
|
13334
13618
|
}
|
|
13619
|
+
for (const [sid, w] of sessions) {
|
|
13620
|
+
if (w.pendingRecovery && w.status === "idle" && !w.userCancelled && !w.aborting && !w.continuing && !w.gaveUp && w.recoveryAttempts === 0) {
|
|
13621
|
+
dbg(`Pending recovery check on ${short(sid)}: pendingRecovery=${w.pendingRecovery}, status=${w.status}, userCancelled=${w.userCancelled}, aborting=${w.aborting}, continuing=${w.continuing}, gaveUp=${w.gaveUp}, recoveryAttempts=${w.recoveryAttempts}, pendingRecoveryAt=${w.pendingRecoveryAt}`);
|
|
13622
|
+
const elapsed = Date.now() - w.pendingRecoveryAt;
|
|
13623
|
+
const requiredBackoff2 = backoffMs(w.recoveryAttempts, baseBackoffMs, maxBackoffMs);
|
|
13624
|
+
if (elapsed < requiredBackoff2) {
|
|
13625
|
+
dbg(`Backoff check on ${short(sid)}: elapsed=${elapsed}ms, required=${requiredBackoff2}ms, attempt=${w.recoveryAttempts}, pass=false`);
|
|
13626
|
+
dbg(`Pending recovery on ${short(sid)} waiting for backoff: ${requiredBackoff2 - elapsed}ms remaining`);
|
|
13627
|
+
continue;
|
|
13628
|
+
}
|
|
13629
|
+
dbg(`Backoff check on ${short(sid)}: elapsed=${elapsed}ms, required=${requiredBackoff2}ms, attempt=${w.recoveryAttempts}, pass=true`);
|
|
13630
|
+
await log("info", `Pending recovery triggered on ${short(sid)}: reason=${w.pendingRecoveryReason}, attempt=${w.recoveryAttempts + 1}, maxRetries=${maxRecoveryRetries}`);
|
|
13631
|
+
dbg(`Recovery timing on ${short(sid)}: detectionToAttemptMs=${Date.now() - w.pendingRecoveryAt}`);
|
|
13632
|
+
w.recoveryAttempts++;
|
|
13633
|
+
dbg(`State transition on ${short(sid)}: recoveryAttempts=${w.recoveryAttempts - 1} -> ${w.recoveryAttempts}`);
|
|
13634
|
+
try {
|
|
13635
|
+
await sendContinuePrompt(sid, continuePrompt, w);
|
|
13636
|
+
} catch (err) {
|
|
13637
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
13638
|
+
await log("warn", `${short(sid)} - pending recovery failed: ${errMsg}`);
|
|
13639
|
+
w.recoveryAttempts = 0;
|
|
13640
|
+
}
|
|
13641
|
+
}
|
|
13642
|
+
if (w.status !== "idle")
|
|
13643
|
+
continue;
|
|
13644
|
+
if (w.isSubagent)
|
|
13645
|
+
continue;
|
|
13646
|
+
if (w.userCancelled || w.completionSignaled)
|
|
13647
|
+
continue;
|
|
13648
|
+
if (w.continuing)
|
|
13649
|
+
continue;
|
|
13650
|
+
if (busyCount() !== 0)
|
|
13651
|
+
continue;
|
|
13652
|
+
const open = getOpenTodos(w.todos || []);
|
|
13653
|
+
if (open.length === 0)
|
|
13654
|
+
continue;
|
|
13655
|
+
if (w.todoNudgeAttempts >= maxRetries)
|
|
13656
|
+
continue;
|
|
13657
|
+
const elapsedSinceLastNudge = Date.now() - w.lastRetryAt;
|
|
13658
|
+
const requiredBackoff = backoffMs(w.todoNudgeAttempts, baseBackoffMs, maxBackoffMs);
|
|
13659
|
+
if (w.lastRetryAt > 0 && elapsedSinceLastNudge < requiredBackoff)
|
|
13660
|
+
continue;
|
|
13661
|
+
const isCelebration = await lastAssistantEndsWithCelebration(sid);
|
|
13662
|
+
if (isCelebration) {
|
|
13663
|
+
w.toolTextRecovered = true;
|
|
13664
|
+
w.completionSignaled = true;
|
|
13665
|
+
continue;
|
|
13666
|
+
}
|
|
13667
|
+
const reminder = buildOpenTodosReminder(w.todos || []);
|
|
13668
|
+
const sent = await tryResume(sid, w, "Idle with open todos (periodic)", reminder);
|
|
13669
|
+
if (sent) {
|
|
13670
|
+
w.todoNudgeAttempts++;
|
|
13671
|
+
await log("info", `${short(sid)} - idle periodic recheck: nudge ${w.todoNudgeAttempts}/${maxRetries}`);
|
|
13672
|
+
}
|
|
13673
|
+
}
|
|
13335
13674
|
cleanupIdleSessions();
|
|
13336
13675
|
}, checkIntervalMs);
|
|
13337
13676
|
if (timer.unref)
|
|
@@ -13359,10 +13698,23 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13359
13698
|
w.status = statusType;
|
|
13360
13699
|
if (statusType === "busy") {
|
|
13361
13700
|
w.lastActivityAt = Date.now();
|
|
13701
|
+
if (w.pendingRecovery) {
|
|
13702
|
+
dbg(`Pending recovery cleared on ${short(sid)}: reason=session-busy`);
|
|
13703
|
+
}
|
|
13362
13704
|
resetSessionFlags(w);
|
|
13363
13705
|
prevBusyCount = busyCount();
|
|
13364
13706
|
log("debug", `${short(sid)} -> busy (${prevBusyCount})`);
|
|
13365
|
-
} else if (statusType === "
|
|
13707
|
+
} else if (statusType === "interrupted") {
|
|
13708
|
+
w.status = "idle";
|
|
13709
|
+
resetIdleFlags(w);
|
|
13710
|
+
w.userCancelled = true;
|
|
13711
|
+
if (w.toolTextTimer) {
|
|
13712
|
+
clearTimeout(w.toolTextTimer);
|
|
13713
|
+
w.toolTextTimer = null;
|
|
13714
|
+
}
|
|
13715
|
+
prevBusyCount = busyCount();
|
|
13716
|
+
log("info", `${short(sid)} -> interrupted by user, backing off`);
|
|
13717
|
+
} else if (statusType === "idle") {
|
|
13366
13718
|
w.status = "idle";
|
|
13367
13719
|
resetIdleFlags(w);
|
|
13368
13720
|
const currentBusy = busyCount();
|
|
@@ -13375,11 +13727,27 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13375
13727
|
}
|
|
13376
13728
|
}
|
|
13377
13729
|
prevBusyCount = currentBusy;
|
|
13378
|
-
log("debug", `${short(sid)} -> idle (${currentBusy})
|
|
13730
|
+
log("debug", `${short(sid)} -> idle (${currentBusy})`);
|
|
13379
13731
|
if (!w.isSubagent) {
|
|
13732
|
+
if (!w.pendingRecovery && !w.completionSignaled && !w.userCancelled && !w.aborting) {
|
|
13733
|
+
try {
|
|
13734
|
+
const errInfo = getLastAssistantError(await getSessionMessages(sid));
|
|
13735
|
+
if (errInfo && isStreamingFailure(errInfo.name, errInfo.message, streamingFailureErrorNames, streamingFailureMessagePatterns)) {
|
|
13736
|
+
w.pendingRecovery = true;
|
|
13737
|
+
w.pendingRecoveryReason = errInfo.name;
|
|
13738
|
+
w.pendingRecoveryAt = Date.now();
|
|
13739
|
+
dbg(`State transition on ${short(sid)}: pendingRecovery=false -> true, reason=${errInfo.name}`);
|
|
13740
|
+
await log("info", `${short(sid)} - streaming failure detected on idle: ${errInfo.name} - ${errInfo.message}`);
|
|
13741
|
+
await tryResume(sid, w, "Streaming failure on idle", continuePrompt);
|
|
13742
|
+
}
|
|
13743
|
+
} catch (e) {
|
|
13744
|
+
const errMsg = e instanceof Error ? e.message : String(e);
|
|
13745
|
+
dbg(`session.idle sid=${short(sid)}: streaming-failure check error: ${errMsg}`);
|
|
13746
|
+
}
|
|
13747
|
+
}
|
|
13380
13748
|
const todos = w.todos || [];
|
|
13381
|
-
const
|
|
13382
|
-
if (
|
|
13749
|
+
const open = getOpenTodos(todos);
|
|
13750
|
+
if (open.length > 0 && currentBusy === 0 && !w.completionSignaled && !w.userCancelled && w.todoNudgeAttempts < maxRetries) {
|
|
13383
13751
|
const isCelebration = await lastAssistantEndsWithCelebration(sid);
|
|
13384
13752
|
if (isCelebration) {
|
|
13385
13753
|
await log("info", `${short(sid)} - \uD83C\uDF89 detected in idle handler, skipping continue`);
|
|
@@ -13390,19 +13758,52 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13390
13758
|
w.toolTextTimer = null;
|
|
13391
13759
|
}
|
|
13392
13760
|
} else {
|
|
13761
|
+
w.todoNudgeAttempts++;
|
|
13393
13762
|
const reminder = buildOpenTodosReminder(todos);
|
|
13394
|
-
await log("info", `${short(sid)} - idle with ${
|
|
13395
|
-
tryResume(sid, w, "Idle with open todos", reminder);
|
|
13763
|
+
await log("info", `${short(sid)} - idle with ${open.length} open todos. Sending reminder (nudge ${w.todoNudgeAttempts}/${maxRetries})...`);
|
|
13764
|
+
await tryResume(sid, w, "Idle with open todos", reminder);
|
|
13396
13765
|
}
|
|
13397
13766
|
}
|
|
13398
13767
|
}
|
|
13399
|
-
if (!w.completionSignaled && w.toolTextAttempts < maxRetries) {
|
|
13768
|
+
if (!w.completionSignaled && !w.userCancelled && w.toolTextAttempts < maxRetries) {
|
|
13769
|
+
if (resumeOnActionIntent && !w.toolTextRecovered) {
|
|
13770
|
+
const idleSid = sid;
|
|
13771
|
+
const idleW = w;
|
|
13772
|
+
setTimeout(async () => {
|
|
13773
|
+
try {
|
|
13774
|
+
if (Date.now() - idleW.createdAt < warmupMs) {
|
|
13775
|
+
dbg(`session.idle sid=${short(idleSid)}: skipping action intent, session is warming up (${Date.now() - idleW.createdAt}ms < ${warmupMs}ms)`);
|
|
13776
|
+
return;
|
|
13777
|
+
}
|
|
13778
|
+
const response = await ctx.client.session.messages({ path: { id: idleSid } });
|
|
13779
|
+
const msgs = extractMessages(response);
|
|
13780
|
+
const lastAssistantMsg = msgs.slice().reverse().find((m) => (m.role ?? m.info?.role) === "assistant");
|
|
13781
|
+
if (lastAssistantMsg) {
|
|
13782
|
+
let lastText = "";
|
|
13783
|
+
for (const part of lastAssistantMsg.parts || []) {
|
|
13784
|
+
lastText += (part.text ?? "") + `
|
|
13785
|
+
`;
|
|
13786
|
+
}
|
|
13787
|
+
if (containsActionIntent(lastText)) {
|
|
13788
|
+
const w2 = sessions.get(idleSid);
|
|
13789
|
+
if (!w2 || w2.toolTextRecovered || w2.completionSignaled || w2.status !== "idle")
|
|
13790
|
+
return;
|
|
13791
|
+
w2.toolTextRecovered = true;
|
|
13792
|
+
w2.toolTextAttempts++;
|
|
13793
|
+
dbg(`session.idle sid=${short(idleSid)}: ACTION INTENT DETECTED, sending "${actionIntentPrompt.slice(0, 40)}..."`);
|
|
13794
|
+
await sendContinuePrompt(idleSid, actionIntentPrompt, w2);
|
|
13795
|
+
}
|
|
13796
|
+
}
|
|
13797
|
+
} catch (e) {
|
|
13798
|
+
dbg(`session.idle sid=${short(idleSid)}: delayed check error: ${e}`);
|
|
13799
|
+
}
|
|
13800
|
+
}, 500);
|
|
13801
|
+
}
|
|
13400
13802
|
if (w.toolTextTimer)
|
|
13401
13803
|
clearTimeout(w.toolTextTimer);
|
|
13402
|
-
const checkDelay = statusType === "interrupted" ? 500 : TOOL_TEXT_CHECK_DELAY_MS;
|
|
13403
13804
|
w.toolTextTimer = setTimeout(() => {
|
|
13404
13805
|
checkForToolCallAsText(sid, w);
|
|
13405
|
-
},
|
|
13806
|
+
}, toolTextCheckDelayMs);
|
|
13406
13807
|
}
|
|
13407
13808
|
} else if (statusType === "retry") {
|
|
13408
13809
|
touchSession(sid);
|
|
@@ -13413,7 +13814,9 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13413
13814
|
case "session.created": {
|
|
13414
13815
|
if (!sid)
|
|
13415
13816
|
break;
|
|
13416
|
-
ensureWatch(sid);
|
|
13817
|
+
const w = ensureWatch(sid);
|
|
13818
|
+
w.pendingTools = 0;
|
|
13819
|
+
w.pendingCommands = 0;
|
|
13417
13820
|
log("debug", `New session: ${short(sid)} (${sessions.size})`);
|
|
13418
13821
|
break;
|
|
13419
13822
|
}
|
|
@@ -13429,12 +13832,45 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13429
13832
|
if (w) {
|
|
13430
13833
|
w.status = "idle";
|
|
13431
13834
|
resetIdleFlags(w);
|
|
13432
|
-
|
|
13835
|
+
dbg(`session.idle sid=${short(sid)}: resetIdleFlags done, toolTextRecovered=${w.toolTextRecovered}, toolTextAttempts=${w.toolTextAttempts}, maxRetries=${maxRetries}`);
|
|
13836
|
+
if (resumeOnActionIntent && !w.toolTextRecovered && !w.completionSignaled) {
|
|
13837
|
+
setTimeout(async () => {
|
|
13838
|
+
try {
|
|
13839
|
+
if (Date.now() - w.createdAt < warmupMs) {
|
|
13840
|
+
dbg(`session.idle sid=${short(sid)}: skipping action intent, session is warming up (${Date.now() - w.createdAt}ms < ${warmupMs}ms)`);
|
|
13841
|
+
return;
|
|
13842
|
+
}
|
|
13843
|
+
const response = await ctx.client.session.messages({ path: { id: sid } });
|
|
13844
|
+
const msgs = extractMessages(response);
|
|
13845
|
+
const lastAssistantMsg = msgs.slice().reverse().find((m) => (m.role ?? m.info?.role) === "assistant");
|
|
13846
|
+
if (lastAssistantMsg) {
|
|
13847
|
+
let lastText = "";
|
|
13848
|
+
for (const part of lastAssistantMsg.parts || []) {
|
|
13849
|
+
lastText += (part.text ?? "") + `
|
|
13850
|
+
`;
|
|
13851
|
+
}
|
|
13852
|
+
if (containsActionIntent(lastText)) {
|
|
13853
|
+
const w2 = sessions.get(sid);
|
|
13854
|
+
if (!w2 || w2.toolTextRecovered || w2.completionSignaled || w2.status !== "idle")
|
|
13855
|
+
return;
|
|
13856
|
+
w2.toolTextRecovered = true;
|
|
13857
|
+
w2.toolTextAttempts++;
|
|
13858
|
+
dbg(`session.idle sid=${short(sid)}: ACTION INTENT DETECTED, sending "${actionIntentPrompt.slice(0, 40)}..."`);
|
|
13859
|
+
await sendContinuePrompt(sid, actionIntentPrompt, w2);
|
|
13860
|
+
}
|
|
13861
|
+
}
|
|
13862
|
+
} catch (e) {
|
|
13863
|
+
dbg(`session.idle sid=${short(sid)}: delayed check error: ${e}`);
|
|
13864
|
+
}
|
|
13865
|
+
}, 500);
|
|
13866
|
+
}
|
|
13867
|
+
const shouldSet = !w.toolTextRecovered && w.toolTextAttempts < maxRetries;
|
|
13868
|
+
if (shouldSet) {
|
|
13433
13869
|
if (w.toolTextTimer)
|
|
13434
13870
|
clearTimeout(w.toolTextTimer);
|
|
13435
13871
|
w.toolTextTimer = setTimeout(() => {
|
|
13436
13872
|
checkForToolCallAsText(sid, w);
|
|
13437
|
-
},
|
|
13873
|
+
}, toolTextCheckDelayMs);
|
|
13438
13874
|
}
|
|
13439
13875
|
}
|
|
13440
13876
|
break;
|
|
@@ -13444,33 +13880,14 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13444
13880
|
break;
|
|
13445
13881
|
const w = sessions.get(sid);
|
|
13446
13882
|
if (w) {
|
|
13447
|
-
const wasJustContinued = w.continuing || Date.now() - w.lastRetryAt < 2000;
|
|
13448
13883
|
w.status = "idle";
|
|
13449
13884
|
resetIdleFlags(w);
|
|
13450
|
-
|
|
13451
|
-
if (
|
|
13452
|
-
w.
|
|
13453
|
-
|
|
13454
|
-
w.continuing = false;
|
|
13455
|
-
try {
|
|
13456
|
-
await sendContinuePrompt(sid, "continue", w);
|
|
13457
|
-
await log("info", `${short(sid)} - interrupted continue retried`);
|
|
13458
|
-
} catch (err) {
|
|
13459
|
-
const errMsg = err instanceof Error ? err.message : String(err);
|
|
13460
|
-
await log("warn", `${short(sid)} - interrupted continue retry failed: ${errMsg}`);
|
|
13461
|
-
}
|
|
13462
|
-
return;
|
|
13463
|
-
} else if (wasJustContinued && w.interruptedContinueCount >= 3) {
|
|
13464
|
-
await log("warn", `${short(sid)} - too many interrupted continues (${w.interruptedContinueCount}), stopping retries`);
|
|
13465
|
-
w.interruptedContinueCount = 0;
|
|
13466
|
-
}
|
|
13467
|
-
if (!w.toolTextRecovered && w.toolTextAttempts < maxRetries) {
|
|
13468
|
-
if (w.toolTextTimer)
|
|
13469
|
-
clearTimeout(w.toolTextTimer);
|
|
13470
|
-
w.toolTextTimer = setTimeout(() => {
|
|
13471
|
-
checkForToolCallAsText(sid, w);
|
|
13472
|
-
}, 500);
|
|
13885
|
+
w.userCancelled = true;
|
|
13886
|
+
if (w.toolTextTimer) {
|
|
13887
|
+
clearTimeout(w.toolTextTimer);
|
|
13888
|
+
w.toolTextTimer = null;
|
|
13473
13889
|
}
|
|
13890
|
+
log("info", `${short(sid)} -> interrupted by user, backing off`);
|
|
13474
13891
|
}
|
|
13475
13892
|
break;
|
|
13476
13893
|
}
|
|
@@ -13490,6 +13907,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13490
13907
|
case "session.error": {
|
|
13491
13908
|
const errorObj = getError(ev);
|
|
13492
13909
|
const errorName = errorObj?.name ?? "";
|
|
13910
|
+
const errorMessage = errorObj?.data?.message ?? String(errorObj?.data ?? "");
|
|
13493
13911
|
const isMessageAborted = errorName === "MessageAbortedError";
|
|
13494
13912
|
if (isMessageAborted) {
|
|
13495
13913
|
for (const [wSid, w] of sessions) {
|
|
@@ -13502,15 +13920,47 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13502
13920
|
log("info", "User abort (ESC)");
|
|
13503
13921
|
break;
|
|
13504
13922
|
}
|
|
13923
|
+
const isStreamingFail = isStreamingFailure(errorName, errorMessage, streamingFailureErrorNames, streamingFailureMessagePatterns);
|
|
13924
|
+
if (isStreamingFail) {
|
|
13925
|
+
if (sid) {
|
|
13926
|
+
const w = sessions.get(sid);
|
|
13927
|
+
if (w && w.status === "busy") {
|
|
13928
|
+
w.pendingRecovery = true;
|
|
13929
|
+
w.pendingRecoveryReason = errorName;
|
|
13930
|
+
w.pendingRecoveryAt = Date.now();
|
|
13931
|
+
dbg(`State transition on ${short(sid)}: pendingRecovery=false -> true, reason=${errorName}`);
|
|
13932
|
+
await log("info", `Streaming failure detected on ${short(sid)}: errorName=${errorName}, errorMessage=${errorMessage}, pendingRecoveryReason=${errorName}`);
|
|
13933
|
+
}
|
|
13934
|
+
log("info", `Streaming failure detected: ${errorName} - ${errorMessage}`);
|
|
13935
|
+
} else {
|
|
13936
|
+
log("warn", `Streaming failure detected but no session ID: ${errorName} - ${errorMessage}`);
|
|
13937
|
+
}
|
|
13938
|
+
}
|
|
13505
13939
|
if (busyCount() === 0)
|
|
13506
13940
|
break;
|
|
13507
|
-
const errorMessage = errorObj?.data?.message ?? String(errorObj?.data ?? "");
|
|
13508
13941
|
log("debug", `Session error: ${errorName} - ${errorMessage}`);
|
|
13942
|
+
if (sid) {
|
|
13943
|
+
const w = sessions.get(sid);
|
|
13944
|
+
if (w) {
|
|
13945
|
+
w.pendingTools = 0;
|
|
13946
|
+
w.pendingCommands = 0;
|
|
13947
|
+
}
|
|
13948
|
+
}
|
|
13509
13949
|
break;
|
|
13510
13950
|
}
|
|
13511
13951
|
case "command.executed": {
|
|
13512
|
-
for (const [,
|
|
13513
|
-
|
|
13952
|
+
for (const [sid2, w2] of sessions) {
|
|
13953
|
+
if (w2.pendingRecovery) {
|
|
13954
|
+
dbg(`Pending recovery cleared on ${short(sid2)}: reason=user-command`);
|
|
13955
|
+
}
|
|
13956
|
+
resetSessionFlags(w2);
|
|
13957
|
+
}
|
|
13958
|
+
if (!sid)
|
|
13959
|
+
break;
|
|
13960
|
+
const w = sessions.get(sid);
|
|
13961
|
+
if (w) {
|
|
13962
|
+
w.pendingCommands = Math.max(0, w.pendingCommands - 1);
|
|
13963
|
+
w.lastActivityAt = Date.now();
|
|
13514
13964
|
}
|
|
13515
13965
|
break;
|
|
13516
13966
|
}
|
|
@@ -13523,6 +13973,12 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13523
13973
|
const w = sessions.get(ctx2.sessionID);
|
|
13524
13974
|
if (w) {
|
|
13525
13975
|
if (!w.isSubagent) {
|
|
13976
|
+
const openTodos = (w.todos || []).filter((t) => t.status === "pending" || t.status === "in_progress");
|
|
13977
|
+
if (openTodos.length > 0 && w.taskCompleteOverrides < maxRetries) {
|
|
13978
|
+
w.taskCompleteOverrides++;
|
|
13979
|
+
await log("info", `${short(ctx2.sessionID)} - task_complete blocked: ${openTodos.length} open todos remain (override ${w.taskCompleteOverrides}/${maxRetries})`);
|
|
13980
|
+
return `You have ${openTodos.length} unfinished task(s). Please complete all remaining work before signaling completion.`;
|
|
13981
|
+
}
|
|
13526
13982
|
w.toolTextRecovered = true;
|
|
13527
13983
|
w.completionSignaled = true;
|
|
13528
13984
|
if (w.toolTextTimer) {
|
|
@@ -13548,12 +14004,36 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13548
14004
|
},
|
|
13549
14005
|
tool: {
|
|
13550
14006
|
task_complete: taskCompleteTool
|
|
14007
|
+
},
|
|
14008
|
+
"tool.execute.before": async (input) => {
|
|
14009
|
+
if (!input?.sessionID)
|
|
14010
|
+
return;
|
|
14011
|
+
const w = ensureWatch(input.sessionID);
|
|
14012
|
+
w.pendingTools++;
|
|
14013
|
+
w.lastActivityAt = Date.now();
|
|
14014
|
+
},
|
|
14015
|
+
"command.execute.before": async (input) => {
|
|
14016
|
+
if (!input?.sessionID)
|
|
14017
|
+
return;
|
|
14018
|
+
const w = ensureWatch(input.sessionID);
|
|
14019
|
+
w.pendingCommands++;
|
|
14020
|
+
w.lastActivityAt = Date.now();
|
|
14021
|
+
},
|
|
14022
|
+
"tool.execute.after": async (input) => {
|
|
14023
|
+
if (!input?.sessionID)
|
|
14024
|
+
return;
|
|
14025
|
+
const w = ensureWatch(input.sessionID);
|
|
14026
|
+
w.pendingTools = Math.max(0, w.pendingTools - 1);
|
|
14027
|
+
w.lastActivityAt = Date.now();
|
|
13551
14028
|
}
|
|
13552
14029
|
};
|
|
13553
14030
|
};
|
|
13554
14031
|
var src_default = AutoResumePlugin;
|
|
13555
14032
|
export {
|
|
14033
|
+
isStreamingFailure,
|
|
14034
|
+
getLastAssistantError,
|
|
13556
14035
|
src_default as default,
|
|
13557
14036
|
buildOpenTodosReminder,
|
|
14037
|
+
backoffMs,
|
|
13558
14038
|
AutoResumePlugin
|
|
13559
14039
|
};
|
package/package.json
CHANGED