opencode-auto-resume 1.1.3 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -0
- package/dist/index.js +462 -41
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -73,6 +73,81 @@ _Motivated by:_
|
|
|
73
73
|
|
|
74
74
|
---
|
|
75
75
|
|
|
76
|
+
### Streaming failure recovery
|
|
77
|
+
|
|
78
|
+
The AI provider's streaming response can fail mid-stream (connection reset, timeout, socket close). When the provider reports an error whose **name** matches `streamingFailureErrorNames` (exact, case-sensitive) or whose **message** matches `streamingFailureMessagePatterns` (regex, case-insensitive), the plugin arms a deferred recovery instead of relying only on the generic stall timeout. Invalid regex patterns fall back to substring matching.
|
|
79
|
+
|
|
80
|
+
**Default error names:**
|
|
81
|
+
- `ProviderError`, `APIError`, `StreamError`, `ConnectionError`, `TimeoutError`
|
|
82
|
+
|
|
83
|
+
**Default message patterns:**
|
|
84
|
+
- `streaming response failed`, `stream.*fail`, `connection.*reset`, `connection.*closed`
|
|
85
|
+
|
|
86
|
+
#### Recovery behavior
|
|
87
|
+
|
|
88
|
+
1. **Detection**: `session.error` is classified via `isStreamingFailure()` against the configured error names and message patterns
|
|
89
|
+
2. **State transition**: the session's `pendingRecovery` flag is armed with the error name (`pendingRecoveryReason`) and timestamp (`pendingRecoveryAt`)
|
|
90
|
+
3. **Recovery attempt**: once the session is idle and the backoff delay has elapsed, the timer loop sends a recovery prompt
|
|
91
|
+
4. **Watchdog**: if the session is still not busy 3 seconds after the prompt, the recovery is retried (up to `maxRecoveryRetries`) with exponential backoff
|
|
92
|
+
5. **Escalation**: when retries are exhausted, the plugin aborts the session and resumes it (`abort+resume`); `gaveUp` is set if that also fails
|
|
93
|
+
|
|
94
|
+
#### Configuration
|
|
95
|
+
|
|
96
|
+
Add to your plugin options (in `opencode.jsonc`):
|
|
97
|
+
|
|
98
|
+
```json
|
|
99
|
+
{
|
|
100
|
+
"streamingFailureErrorNames": ["ProviderError", "APIError", "StreamError", "ConnectionError", "TimeoutError"],
|
|
101
|
+
"streamingFailureMessagePatterns": ["streaming response failed", "stream.*fail", "connection.*reset", "connection.*closed"],
|
|
102
|
+
"maxRecoveryRetries": 2,
|
|
103
|
+
"baseBackoffMs": 1000,
|
|
104
|
+
"maxBackoffMs": 8000
|
|
105
|
+
}
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
#### State machine addition
|
|
109
|
+
|
|
110
|
+
New per-session recovery fields added to the state machine:
|
|
111
|
+
- `pendingRecovery` — failure detected, recovery armed
|
|
112
|
+
- `pendingRecoveryReason` — error name that triggered the recovery
|
|
113
|
+
- `pendingRecoveryAt` — detection timestamp (backoff anchor)
|
|
114
|
+
- `recoveryAttempts` — recovery attempt counter
|
|
115
|
+
- `watchdogRetryGuard` — watchdog retry in progress (keeps the recovery armed)
|
|
116
|
+
|
|
117
|
+
Recovery chain: `pendingRecovery` → recovery attempt → `recoveryAttempts` retry → `abort+resume` → `gaveUp`.
|
|
118
|
+
|
|
119
|
+
See [Recovery Flow Documentation](docs/architecture/recovery-flow.md) for the full state machine.
|
|
120
|
+
|
|
121
|
+
#### Example scenario
|
|
122
|
+
|
|
123
|
+
```
|
|
124
|
+
1. AI provider starts streaming response
|
|
125
|
+
2. Network interruption causes "connection reset" error mid-stream
|
|
126
|
+
3. System detects "ConnectionError" matches streamingFailureErrorNames
|
|
127
|
+
4. Session's pendingRecovery flag is armed (reason=ConnectionError)
|
|
128
|
+
5. Session goes idle; timer loop waits until the backoff delay has elapsed
|
|
129
|
+
6. Recovery prompt sent (recoveryAttempts=1)
|
|
130
|
+
7. Success → session busy → recovery flags cleared
|
|
131
|
+
Still not busy after 3s → watchdog retry (attempt 2/2)
|
|
132
|
+
Still not busy after 3s → maxRecoveryRetries reached → abort + resume
|
|
133
|
+
Abort+continue fails → gaveUp
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
#### Configuration reference
|
|
137
|
+
|
|
138
|
+
| Option | Type | Default | Description |
|
|
139
|
+
|--------|------|---------|-------------|
|
|
140
|
+
| `streamingFailureErrorNames` | `string[]` | `["ProviderError","APIError","StreamError","ConnectionError","TimeoutError"]` | Error names that indicate a streaming failure (exact, case-sensitive) |
|
|
141
|
+
| `streamingFailureMessagePatterns` | `string[]` | `["streaming response failed","stream.*fail","connection.*reset","connection.*closed"]` | Regex patterns matching streaming failure messages (case-insensitive) |
|
|
142
|
+
| `maxRecoveryRetries` | `number` | `2` | Maximum streaming-failure recovery attempts before abort+resume escalation |
|
|
143
|
+
| `baseBackoffMs` | `number` | `1000` | Initial backoff delay in milliseconds |
|
|
144
|
+
| `maxBackoffMs` | `number` | `8000` | Maximum backoff delay cap in milliseconds |
|
|
145
|
+
|
|
146
|
+
_Motivated by:_
|
|
147
|
+
- [EPIC: Streaming Failure Recovery](docs/EPIC-Streaming-Recovery-OpenCode-Auto-Resume-v3.md) — recovery requests not consistently creating new assistant executions after mid-stream failures
|
|
148
|
+
|
|
149
|
+
---
|
|
150
|
+
|
|
76
151
|
### Active-tool safety guard
|
|
77
152
|
|
|
78
153
|
Before **any** abort, the plugin calls `checkSessionHasActiveTool()` to verify the session isn't mid-tool-execution. If a tool is running, the abort is skipped. This prevents the plugin from killing a long-running build, test suite, or command — even when it looks like a stall.
|
|
@@ -256,6 +331,11 @@ bun run build
|
|
|
256
331
|
| `subagentWaitMs` | `15000` | Wait before treating orphan parent as stuck |
|
|
257
332
|
| `loopMaxContinues` | `3` | Continues in window before triggering abort |
|
|
258
333
|
| `loopWindowMs` | `600000` | Hallucination loop detection window (10 min) |
|
|
334
|
+
| `streamingFailureErrorNames` | `["ProviderError","APIError","StreamError","ConnectionError","TimeoutError"]` | Error names that classify as streaming failures (exact match) |
|
|
335
|
+
| `streamingFailureMessagePatterns` | `["streaming response failed","stream.*fail","connection.*reset","connection.*closed"]` | Regex patterns (case-insensitive) in error messages indicating streaming failure |
|
|
336
|
+
| `maxRecoveryRetries` | `2` | Max streaming-failure recovery attempts before abort+resume escalation |
|
|
337
|
+
|
|
338
|
+
Message patterns are matched case-insensitively. Error names use exact match.
|
|
259
339
|
|
|
260
340
|
### Internal constants (not configurable)
|
|
261
341
|
|
package/dist/index.js
CHANGED
|
@@ -12345,8 +12345,24 @@ var DEFAULT_SUBAGENT_WAIT_MS = 15000;
|
|
|
12345
12345
|
var ABORT_CONTINUE_DELAY_MS = 2000;
|
|
12346
12346
|
var DEFAULT_LOOP_MAX_CONTINUES = 3;
|
|
12347
12347
|
var DEFAULT_LOOP_WINDOW_MS = 10 * 60000;
|
|
12348
|
-
var
|
|
12349
|
-
var
|
|
12348
|
+
var DEFAULT_TOOL_TEXT_CHECK_DELAY_MS = 3000;
|
|
12349
|
+
var DEFAULT_MAX_RECOVERY_RETRIES = 2;
|
|
12350
|
+
var DEFAULT_MIN_ACTIVITY_GAP_MS = 1000;
|
|
12351
|
+
var DEFAULT_WARMUP_MS = 15000;
|
|
12352
|
+
var DEFAULT_DEBUG = false;
|
|
12353
|
+
var DEFAULT_STREAMING_FAILURE_ERROR_NAMES = [
|
|
12354
|
+
"ProviderError",
|
|
12355
|
+
"APIError",
|
|
12356
|
+
"StreamError",
|
|
12357
|
+
"ConnectionError",
|
|
12358
|
+
"TimeoutError"
|
|
12359
|
+
];
|
|
12360
|
+
var DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS = [
|
|
12361
|
+
"streaming response failed",
|
|
12362
|
+
"stream.*fail",
|
|
12363
|
+
"connection.*reset",
|
|
12364
|
+
"connection.*closed"
|
|
12365
|
+
];
|
|
12350
12366
|
var MAX_IDLE_SESSIONS = 50;
|
|
12351
12367
|
var IDLE_CLEANUP_MS = 10 * 60000;
|
|
12352
12368
|
var SESSION_DISCOVERY_INTERVAL_MS = 60000;
|
|
@@ -12388,13 +12404,17 @@ var READY_TO_CONTINUE_PATTERNS = [
|
|
|
12388
12404
|
/will continue with task/i,
|
|
12389
12405
|
/moving on to task/i
|
|
12390
12406
|
];
|
|
12407
|
+
function stripCodeBlocks(text) {
|
|
12408
|
+
return text.replace(/```[\s\S]*?```/g, "").replace(/`[^`\n]+`/g, "");
|
|
12409
|
+
}
|
|
12391
12410
|
function containsToolCallAsText(text) {
|
|
12392
12411
|
if (text.length <= 10)
|
|
12393
12412
|
return false;
|
|
12394
|
-
|
|
12413
|
+
const stripped = stripCodeBlocks(text);
|
|
12414
|
+
if (TOOL_TEXT_PATTERNS.some((pat) => pat.test(stripped)))
|
|
12395
12415
|
return true;
|
|
12396
12416
|
for (const { open, close } of TRUNCATED_XML_PATTERNS) {
|
|
12397
|
-
if (open.test(
|
|
12417
|
+
if (open.test(stripped) && !close.test(stripped))
|
|
12398
12418
|
return true;
|
|
12399
12419
|
}
|
|
12400
12420
|
return false;
|
|
@@ -12429,17 +12449,97 @@ function containsDoneClaimPattern(text) {
|
|
|
12429
12449
|
`);
|
|
12430
12450
|
return DONE_CLAIM_PATTERNS.some((pat) => pat.test(lastLines));
|
|
12431
12451
|
}
|
|
12452
|
+
function isStreamingFailure(errorName, errorMessage, errorNames = DEFAULT_STREAMING_FAILURE_ERROR_NAMES, messagePatterns = DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS) {
|
|
12453
|
+
if (!errorName && !errorMessage)
|
|
12454
|
+
return false;
|
|
12455
|
+
if (errorName && errorNames.includes(errorName)) {
|
|
12456
|
+
return true;
|
|
12457
|
+
}
|
|
12458
|
+
if (errorMessage) {
|
|
12459
|
+
const lowerMessage = errorMessage.toLowerCase();
|
|
12460
|
+
for (const pattern of messagePatterns) {
|
|
12461
|
+
try {
|
|
12462
|
+
if (new RegExp(pattern, "i").test(lowerMessage))
|
|
12463
|
+
return true;
|
|
12464
|
+
} catch {
|
|
12465
|
+
if (lowerMessage.includes(pattern.toLowerCase()))
|
|
12466
|
+
return true;
|
|
12467
|
+
}
|
|
12468
|
+
}
|
|
12469
|
+
}
|
|
12470
|
+
return false;
|
|
12471
|
+
}
|
|
12472
|
+
function getLastAssistantError(messages) {
|
|
12473
|
+
for (let i = messages.length - 1;i >= 0; i--) {
|
|
12474
|
+
const msg = messages[i];
|
|
12475
|
+
const role = msg.role ?? msg.info?.role;
|
|
12476
|
+
if (role !== "assistant")
|
|
12477
|
+
continue;
|
|
12478
|
+
const info = msg.info;
|
|
12479
|
+
const err = msg.error ?? info?.error;
|
|
12480
|
+
if (err) {
|
|
12481
|
+
const data = err.data;
|
|
12482
|
+
const name = err.name ?? "";
|
|
12483
|
+
const message = data?.message ?? err.message ?? "";
|
|
12484
|
+
return { name, message };
|
|
12485
|
+
}
|
|
12486
|
+
const parts = msg.parts;
|
|
12487
|
+
if (parts) {
|
|
12488
|
+
for (let j = parts.length - 1;j >= 0; j--) {
|
|
12489
|
+
const part = parts[j];
|
|
12490
|
+
if (part.type !== "retry")
|
|
12491
|
+
continue;
|
|
12492
|
+
const partErr = part.error;
|
|
12493
|
+
if (!partErr)
|
|
12494
|
+
continue;
|
|
12495
|
+
const data = partErr.data;
|
|
12496
|
+
const name = partErr.name ?? "";
|
|
12497
|
+
const message = data?.message ?? partErr.message ?? "";
|
|
12498
|
+
return { name, message };
|
|
12499
|
+
}
|
|
12500
|
+
}
|
|
12501
|
+
}
|
|
12502
|
+
return null;
|
|
12503
|
+
}
|
|
12504
|
+
function backoffMs(attempt, baseBackoffMs = DEFAULT_BASE_BACKOFF_MS, maxBackoffMs = DEFAULT_MAX_BACKOFF_MS) {
|
|
12505
|
+
return Math.min(baseBackoffMs * Math.pow(2, attempt - 1), maxBackoffMs);
|
|
12506
|
+
}
|
|
12507
|
+
function containsActionIntent(text) {
|
|
12508
|
+
if (text.length <= 15)
|
|
12509
|
+
return false;
|
|
12510
|
+
const cleaned = text.replace(/<[a-zA-Z/?][^>]*>/g, "").trim();
|
|
12511
|
+
const lines = cleaned.split(`
|
|
12512
|
+
`);
|
|
12513
|
+
let lastLine = "";
|
|
12514
|
+
for (let i = lines.length - 1;i >= 0; i--) {
|
|
12515
|
+
if (lines[i].trim().length > 0) {
|
|
12516
|
+
lastLine = lines[i].trim();
|
|
12517
|
+
break;
|
|
12518
|
+
}
|
|
12519
|
+
}
|
|
12520
|
+
return lastLine.endsWith(":") && lastLine.length > 5 && lastLine.length < 500;
|
|
12521
|
+
}
|
|
12522
|
+
function isOpenTodo(t) {
|
|
12523
|
+
return t.status === "pending" || t.status === "in_progress";
|
|
12524
|
+
}
|
|
12525
|
+
function getOpenTodos(todos) {
|
|
12526
|
+
return todos.filter(isOpenTodo);
|
|
12527
|
+
}
|
|
12432
12528
|
function buildOpenTodosReminder(todos) {
|
|
12529
|
+
if (!Array.isArray(todos))
|
|
12530
|
+
return "continue";
|
|
12433
12531
|
const open = todos.filter((t) => t.status === "pending" || t.status === "in_progress");
|
|
12434
12532
|
if (open.length === 0)
|
|
12435
12533
|
return "continue";
|
|
12436
12534
|
const list = open.map((t, i) => `${i + 1}. [${t.status}] ${t.content}`).join(`
|
|
12437
12535
|
`);
|
|
12438
12536
|
const plural = open.length > 1 ? "s" : "";
|
|
12537
|
+
const taskWord = open.length > 1 ? "tasks" : "task";
|
|
12538
|
+
const thisWord = open.length > 1 ? "these" : "this";
|
|
12439
12539
|
return `You have ${open.length} unfinished task${plural}:
|
|
12440
12540
|
${list}
|
|
12441
12541
|
|
|
12442
|
-
Please continue working on
|
|
12542
|
+
Please continue working on ${thisWord} ${taskWord}.`;
|
|
12443
12543
|
}
|
|
12444
12544
|
var AutoResumePlugin = async (ctx, options) => {
|
|
12445
12545
|
const chunkTimeoutMs = options?.chunkTimeoutMs ?? DEFAULT_CHUNK_TIMEOUT_MS;
|
|
@@ -12451,6 +12551,23 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12451
12551
|
const subagentWaitMs = options?.subagentWaitMs ?? DEFAULT_SUBAGENT_WAIT_MS;
|
|
12452
12552
|
const loopMaxContinues = options?.loopMaxContinues ?? DEFAULT_LOOP_MAX_CONTINUES;
|
|
12453
12553
|
const loopWindowMs = options?.loopWindowMs ?? DEFAULT_LOOP_WINDOW_MS;
|
|
12554
|
+
const toolTextCheckDelayMs = options?.toolTextCheckDelayMs ?? DEFAULT_TOOL_TEXT_CHECK_DELAY_MS;
|
|
12555
|
+
const maxRecoveryRetries = options?.maxRecoveryRetries ?? DEFAULT_MAX_RECOVERY_RETRIES;
|
|
12556
|
+
const minActivityGapMs = options?.minActivityGapMs ?? DEFAULT_MIN_ACTIVITY_GAP_MS;
|
|
12557
|
+
const warmupMs = options?.warmupMs ?? DEFAULT_WARMUP_MS;
|
|
12558
|
+
const debug = options?.debug ?? DEFAULT_DEBUG;
|
|
12559
|
+
const streamingFailureErrorNames = options?.streamingFailureErrorNames ?? DEFAULT_STREAMING_FAILURE_ERROR_NAMES;
|
|
12560
|
+
const streamingFailureMessagePatterns = options?.streamingFailureMessagePatterns ?? DEFAULT_STREAMING_FAILURE_MESSAGE_PATTERNS;
|
|
12561
|
+
const resumeOnActionIntent = options?.resumeOnActionIntent !== false;
|
|
12562
|
+
const continuePrompt = options?.continuePrompt ?? "continue";
|
|
12563
|
+
const actionIntentPrompt = options?.actionIntentPrompt ?? continuePrompt;
|
|
12564
|
+
const toolTextRecoveryPrompt = options?.toolTextRecoveryPrompt ?? TOOL_TEXT_RECOVERY_PROMPT;
|
|
12565
|
+
const thinkingToolRecoveryPrompt = options?.thinkingToolRecoveryPrompt ?? THINKING_TOOL_RECOVERY_PROMPT;
|
|
12566
|
+
const doneWithoutWorkPrompt = options?.doneWithoutWorkPrompt ?? DONE_WITHOUT_WORK_PROMPT;
|
|
12567
|
+
const dbg = (...args) => {
|
|
12568
|
+
if (debug)
|
|
12569
|
+
console.log("[debug]", ...args);
|
|
12570
|
+
};
|
|
12454
12571
|
const sessions = new Map;
|
|
12455
12572
|
let timer = null;
|
|
12456
12573
|
let discoveryTimer = null;
|
|
@@ -12482,6 +12599,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12482
12599
|
let w = sessions.get(sid);
|
|
12483
12600
|
if (!w) {
|
|
12484
12601
|
w = {
|
|
12602
|
+
createdAt: Date.now(),
|
|
12485
12603
|
lastActivityAt: Date.now(),
|
|
12486
12604
|
status: "unknown",
|
|
12487
12605
|
userCancelled: false,
|
|
@@ -12506,8 +12624,15 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12506
12624
|
isSubagent: false,
|
|
12507
12625
|
completionSignaled: false,
|
|
12508
12626
|
todoNudgeAttempts: 0,
|
|
12627
|
+
taskCompleteOverrides: 0,
|
|
12628
|
+
doneClaimNoTodosAttempts: 0,
|
|
12509
12629
|
pendingTools: 0,
|
|
12510
|
-
pendingCommands: 0
|
|
12630
|
+
pendingCommands: 0,
|
|
12631
|
+
pendingRecovery: false,
|
|
12632
|
+
pendingRecoveryReason: null,
|
|
12633
|
+
pendingRecoveryAt: 0,
|
|
12634
|
+
recoveryAttempts: 0,
|
|
12635
|
+
watchdogRetryGuard: false
|
|
12511
12636
|
};
|
|
12512
12637
|
sessions.set(sid, w);
|
|
12513
12638
|
}
|
|
@@ -12568,9 +12693,6 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12568
12693
|
function short(sid) {
|
|
12569
12694
|
return sid.length > 12 ? `...${sid.slice(-8)}` : sid;
|
|
12570
12695
|
}
|
|
12571
|
-
function backoffMs(attempt) {
|
|
12572
|
-
return Math.min(baseBackoffMs * Math.pow(2, attempt - 1), maxBackoffMs);
|
|
12573
|
-
}
|
|
12574
12696
|
function cleanupIdleSessions() {
|
|
12575
12697
|
const now = Date.now();
|
|
12576
12698
|
const toDelete = [];
|
|
@@ -12605,12 +12727,52 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12605
12727
|
log("debug", `Cleaned up ${toDelete.length} idle session(s). Map size: ${sessions.size}`);
|
|
12606
12728
|
}
|
|
12607
12729
|
}
|
|
12730
|
+
function getPromptResponsePayload(result) {
|
|
12731
|
+
if (!result || typeof result !== "object")
|
|
12732
|
+
return null;
|
|
12733
|
+
const raw = result;
|
|
12734
|
+
const data = raw.data;
|
|
12735
|
+
if (data && typeof data === "object" && "parts" in data) {
|
|
12736
|
+
return data;
|
|
12737
|
+
}
|
|
12738
|
+
if ("parts" in raw) {
|
|
12739
|
+
return raw;
|
|
12740
|
+
}
|
|
12741
|
+
return null;
|
|
12742
|
+
}
|
|
12743
|
+
async function logPromptResponse(result, context) {
|
|
12744
|
+
const payload = getPromptResponsePayload(result);
|
|
12745
|
+
const isRetry = context.isRetry ?? false;
|
|
12746
|
+
if (!payload) {
|
|
12747
|
+
await log("debug", `${short(context.sessionId)} - session.prompt() response has unexpected structure: ${JSON.stringify({ sessionId: context.sessionId, isRetry, rawResponse: JSON.stringify(result) })}`);
|
|
12748
|
+
return;
|
|
12749
|
+
}
|
|
12750
|
+
const parts = Array.isArray(payload.parts) ? payload.parts : [];
|
|
12751
|
+
const partsCount = parts.length;
|
|
12752
|
+
const info = payload.info && typeof payload.info === "object" ? payload.info : undefined;
|
|
12753
|
+
const infoKeys = info ? Object.keys(info) : [];
|
|
12754
|
+
const hasParts = partsCount > 0;
|
|
12755
|
+
await log("debug", `${short(context.sessionId)} - session.prompt() response received: ${JSON.stringify({ sessionId: context.sessionId, isRetry, hasParts, partsCount, infoKeys, info })}`);
|
|
12756
|
+
if (partsCount === 0) {
|
|
12757
|
+
await log("warn", `${short(context.sessionId)} - session.prompt() returned empty parts array - possible stream initiation failure: ${JSON.stringify({ sessionId: context.sessionId, isRetry, responseInfo: info, partsCount })}`);
|
|
12758
|
+
}
|
|
12759
|
+
if (info && "error" in info) {
|
|
12760
|
+
await log("warn", `${short(context.sessionId)} - session.prompt() response.info contains error indicator: ${JSON.stringify({ sessionId: context.sessionId, isRetry, errorInfo: info.error })}`);
|
|
12761
|
+
}
|
|
12762
|
+
await log("debug", `${short(context.sessionId)} - session.prompt() raw response: ${JSON.stringify({ sessionId: context.sessionId, isRetry, rawResponse: JSON.stringify(result, null, 2) })}`);
|
|
12763
|
+
}
|
|
12608
12764
|
async function sendContinuePrompt(sid, text, w) {
|
|
12609
|
-
if (w.continuing) {
|
|
12765
|
+
if (w.continuing && !w.watchdogRetryGuard) {
|
|
12610
12766
|
await log("debug", `${short(sid)} - continue already in progress, skipping`);
|
|
12611
12767
|
return;
|
|
12612
12768
|
}
|
|
12769
|
+
if (!w.continuing)
|
|
12770
|
+
dbg(`State transition on ${short(sid)}: continuing=false -> true`);
|
|
12613
12771
|
w.continuing = true;
|
|
12772
|
+
if (w.watchdogRetryGuard) {
|
|
12773
|
+
w.pendingRecovery = true;
|
|
12774
|
+
}
|
|
12775
|
+
w.watchdogRetryGuard = false;
|
|
12614
12776
|
let agent;
|
|
12615
12777
|
let model;
|
|
12616
12778
|
try {
|
|
@@ -12641,7 +12803,8 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12641
12803
|
break;
|
|
12642
12804
|
}
|
|
12643
12805
|
}
|
|
12644
|
-
|
|
12806
|
+
dbg(`Recovery prompt sent to ${short(sid)}: prompt="${text.length > 80 ? `${text.slice(0, 80)}...` : text}", agent=${agent ?? "(default)"}, model=${model ? `${model.providerID}/${model.modelID}` : "(default)"}`);
|
|
12807
|
+
const response = await ctx.client.session.prompt({
|
|
12645
12808
|
path: { id: sid },
|
|
12646
12809
|
body: {
|
|
12647
12810
|
parts: [{ type: "text", text }],
|
|
@@ -12649,6 +12812,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12649
12812
|
model
|
|
12650
12813
|
}
|
|
12651
12814
|
});
|
|
12815
|
+
await logPromptResponse(response, { sessionId: sid });
|
|
12652
12816
|
await log("debug", `${short(sid)} - prompt sent with agent: ${agent ?? "(default)"}, model: ${model ? `${model.providerID}/${model.modelID}` : "(default)"}`);
|
|
12653
12817
|
recordContinue(sid);
|
|
12654
12818
|
w.lastRetryAt = Date.now();
|
|
@@ -12656,10 +12820,11 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12656
12820
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
12657
12821
|
await log("warn", `${short(sid)} - prompt failed: ${errMsg}`);
|
|
12658
12822
|
try {
|
|
12659
|
-
await ctx.client.session.prompt({
|
|
12823
|
+
const retryResponse = await ctx.client.session.prompt({
|
|
12660
12824
|
path: { id: sid },
|
|
12661
12825
|
body: { parts: [{ type: "text", text }], agent, model }
|
|
12662
12826
|
});
|
|
12827
|
+
await logPromptResponse(retryResponse, { sessionId: sid, isRetry: true });
|
|
12663
12828
|
recordContinue(sid);
|
|
12664
12829
|
w.lastRetryAt = Date.now();
|
|
12665
12830
|
} catch (retryErr) {
|
|
@@ -12668,6 +12833,8 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12668
12833
|
throw retryErr;
|
|
12669
12834
|
}
|
|
12670
12835
|
} finally {
|
|
12836
|
+
if (w.continuing)
|
|
12837
|
+
dbg(`State transition on ${short(sid)}: continuing=true -> false`);
|
|
12671
12838
|
w.continuing = false;
|
|
12672
12839
|
w.todoCheckAttempts = 0;
|
|
12673
12840
|
if (w.toolTextTimer) {
|
|
@@ -12677,9 +12844,53 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12677
12844
|
}
|
|
12678
12845
|
setTimeout(async () => {
|
|
12679
12846
|
if (w.status !== "busy") {
|
|
12680
|
-
|
|
12847
|
+
if (w.pendingRecovery) {
|
|
12848
|
+
if (w.recoveryAttempts < maxRecoveryRetries) {
|
|
12849
|
+
w.recoveryAttempts++;
|
|
12850
|
+
dbg(`State transition on ${short(sid)}: recoveryAttempts=${w.recoveryAttempts - 1} -> ${w.recoveryAttempts}`);
|
|
12851
|
+
w.watchdogRetryGuard = true;
|
|
12852
|
+
await log("warn", `${short(sid)} - recovery attempt ${w.recoveryAttempts}/${maxRecoveryRetries} after prompt timeout`);
|
|
12853
|
+
await log("warn", `Recovery failed on ${short(sid)} - session still ${w.status}: attempt=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, nextAction=retry`);
|
|
12854
|
+
dbg(`Watchdog check on ${short(sid)}: status=${w.status}, recoveryAttempts=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, watchdogLatencyMs=${Date.now() - w.lastRetryAt} -> RETRY`);
|
|
12855
|
+
const retryBackoffMs = backoffMs(w.recoveryAttempts, baseBackoffMs, maxBackoffMs);
|
|
12856
|
+
await log("info", `Retrying recovery on ${short(sid)}: attempt=${w.recoveryAttempts}, backoffMs=${retryBackoffMs}`);
|
|
12857
|
+
dbg(`Retrying recovery on ${short(sid)}: recoveryAttempts=${w.recoveryAttempts}, backoffMs=${retryBackoffMs}, pendingRecoveryReason=${w.pendingRecoveryReason}`);
|
|
12858
|
+
try {
|
|
12859
|
+
await sendContinuePrompt(sid, continuePrompt, w);
|
|
12860
|
+
} catch (err) {
|
|
12861
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
12862
|
+
await log("warn", `${short(sid)} - recovery retry failed: ${errMsg}`);
|
|
12863
|
+
w.recoveryAttempts = 0;
|
|
12864
|
+
}
|
|
12865
|
+
w.watchdogRetryGuard = false;
|
|
12866
|
+
} else {
|
|
12867
|
+
dbg(`Pending recovery cleared on ${short(sid)}: reason=recovery-attempt`);
|
|
12868
|
+
w.pendingRecovery = false;
|
|
12869
|
+
await log("warn", `${short(sid)} - max recovery attempts (${maxRecoveryRetries}) reached, escalating to abort+resume`);
|
|
12870
|
+
await log("warn", `Recovery failed on ${short(sid)} - session still ${w.status}: attempt=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, nextAction=abort-resume`);
|
|
12871
|
+
await log("warn", `Escalating to abort+resume on ${short(sid)}: attempt=${w.recoveryAttempts}`);
|
|
12872
|
+
dbg(`Watchdog check on ${short(sid)}: status=${w.status}, recoveryAttempts=${w.recoveryAttempts}, maxRetries=${maxRecoveryRetries}, watchdogLatencyMs=${Date.now() - w.lastRetryAt} -> ABORT_RESUME`);
|
|
12873
|
+
const resumed = await tryAbortAndResume(sid, w);
|
|
12874
|
+
if (!resumed && !w.aborting) {
|
|
12875
|
+
await log("warn", `Recovery exhausted on ${short(sid)}: attempts=${w.recoveryAttempts}, lastError=abort+resume failed`);
|
|
12876
|
+
dbg(`Watchdog check on ${short(sid)}: status=${w.status} -> GAVE_UP`);
|
|
12877
|
+
if (w.pendingRecoveryAt > 0) {
|
|
12878
|
+
dbg(`Total recovery cycle on ${short(sid)} (failed): totalCycleMs=${Date.now() - w.pendingRecoveryAt}`);
|
|
12879
|
+
}
|
|
12880
|
+
}
|
|
12881
|
+
}
|
|
12882
|
+
} else {
|
|
12883
|
+
await log("warn", `${short(sid)} - prompt sent >${toolTextCheckDelayMs / 1000}s ago but session is still ${w.status}`);
|
|
12884
|
+
}
|
|
12885
|
+
} else {
|
|
12886
|
+
const elapsedMs = Date.now() - w.lastRetryAt;
|
|
12887
|
+
await log("info", `Recovery successful on ${short(sid)}: elapsedMs=${elapsedMs}`);
|
|
12888
|
+
dbg(`Watchdog check on ${short(sid)}: status=busy, elapsedMs=${elapsedMs} -> SUCCESS`);
|
|
12889
|
+
if (w.pendingRecoveryAt > 0) {
|
|
12890
|
+
dbg(`Total recovery cycle on ${short(sid)}: totalCycleMs=${Date.now() - w.pendingRecoveryAt}`);
|
|
12891
|
+
}
|
|
12681
12892
|
}
|
|
12682
|
-
},
|
|
12893
|
+
}, toolTextCheckDelayMs);
|
|
12683
12894
|
}
|
|
12684
12895
|
function extractMessages(response) {
|
|
12685
12896
|
if (Array.isArray(response))
|
|
@@ -12851,7 +13062,6 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12851
13062
|
w.toolTextRecovered = false;
|
|
12852
13063
|
w.toolTextAttempts = 0;
|
|
12853
13064
|
w.completionSignaled = false;
|
|
12854
|
-
w.todoNudgeAttempts = 0;
|
|
12855
13065
|
w.continueTimestamps = [];
|
|
12856
13066
|
w.idleSince = null;
|
|
12857
13067
|
w.continuing = false;
|
|
@@ -12864,6 +13074,11 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12864
13074
|
clearTimeout(w.toolTextTimer);
|
|
12865
13075
|
w.toolTextTimer = null;
|
|
12866
13076
|
}
|
|
13077
|
+
w.pendingRecovery = false;
|
|
13078
|
+
w.pendingRecoveryReason = null;
|
|
13079
|
+
w.pendingRecoveryAt = 0;
|
|
13080
|
+
w.recoveryAttempts = 0;
|
|
13081
|
+
w.watchdogRetryGuard = false;
|
|
12867
13082
|
}
|
|
12868
13083
|
function resetIdleFlags(w) {
|
|
12869
13084
|
w.aborting = false;
|
|
@@ -12917,9 +13132,10 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12917
13132
|
if (w.checkingToolText)
|
|
12918
13133
|
return;
|
|
12919
13134
|
w.checkingToolText = true;
|
|
13135
|
+
dbg(`checkForToolCallAsText called for ${short(sid)}, userCancelled=${w.userCancelled}, toolTextRecovered=${w.toolTextRecovered}, toolTextAttempts=${w.toolTextAttempts}`);
|
|
12920
13136
|
if (w.toolTextAttempts > 0) {
|
|
12921
13137
|
const elapsed = Date.now() - w.lastRetryAt;
|
|
12922
|
-
const requiredBackoff = backoffMs(w.toolTextAttempts);
|
|
13138
|
+
const requiredBackoff = backoffMs(w.toolTextAttempts, baseBackoffMs, maxBackoffMs);
|
|
12923
13139
|
if (elapsed < requiredBackoff)
|
|
12924
13140
|
return;
|
|
12925
13141
|
}
|
|
@@ -12990,7 +13206,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
12990
13206
|
}
|
|
12991
13207
|
} else {
|
|
12992
13208
|
const candidate = {
|
|
12993
|
-
prompt:
|
|
13209
|
+
prompt: continuePrompt,
|
|
12994
13210
|
source: "tool-use",
|
|
12995
13211
|
priority: 1
|
|
12996
13212
|
};
|
|
@@ -13017,7 +13233,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13017
13233
|
}
|
|
13018
13234
|
}
|
|
13019
13235
|
const candidate = {
|
|
13020
|
-
prompt: isReasoning ?
|
|
13236
|
+
prompt: isReasoning ? thinkingToolRecoveryPrompt : toolTextRecoveryPrompt,
|
|
13021
13237
|
source: isReasoning ? "reasoning" : "text",
|
|
13022
13238
|
priority: 0
|
|
13023
13239
|
};
|
|
@@ -13027,13 +13243,13 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13027
13243
|
}
|
|
13028
13244
|
if (containsReadyToContinuePattern(text)) {
|
|
13029
13245
|
const todos = w.todos || [];
|
|
13030
|
-
const hasOpenTodos = todos.some(
|
|
13246
|
+
const hasOpenTodos = todos.some(isOpenTodo);
|
|
13031
13247
|
if (!hasOpenTodos && todos.length > 0) {
|
|
13032
13248
|
w.todoCheckAttempts++;
|
|
13033
13249
|
if (w.todoCheckAttempts >= 2) {
|
|
13034
13250
|
await log("info", `${short(sid)} - todos completed but agent hasn't closed them. Sending continue...`);
|
|
13035
13251
|
const candidate2 = {
|
|
13036
|
-
prompt:
|
|
13252
|
+
prompt: continuePrompt,
|
|
13037
13253
|
source: "todo-completed-continue",
|
|
13038
13254
|
priority: 1
|
|
13039
13255
|
};
|
|
@@ -13046,7 +13262,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13046
13262
|
continue;
|
|
13047
13263
|
}
|
|
13048
13264
|
const candidate = {
|
|
13049
|
-
prompt: containsDoneClaimPattern(text) ?
|
|
13265
|
+
prompt: containsDoneClaimPattern(text) ? doneWithoutWorkPrompt : continuePrompt,
|
|
13050
13266
|
source: containsDoneClaimPattern(text) ? "done-claim" : "ready-to-continue",
|
|
13051
13267
|
priority: 1
|
|
13052
13268
|
};
|
|
@@ -13056,14 +13272,42 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13056
13272
|
}
|
|
13057
13273
|
if (!bestCandidate && containsDoneClaimPattern(text)) {
|
|
13058
13274
|
const todos = w.todos || [];
|
|
13059
|
-
const hasOpenTodos = todos.some(
|
|
13275
|
+
const hasOpenTodos = todos.some(isOpenTodo);
|
|
13060
13276
|
if (hasOpenTodos) {
|
|
13061
13277
|
await log("info", `${short(sid)} - model claims done but todos remain open. Sending recovery prompt...`);
|
|
13062
13278
|
bestCandidate = {
|
|
13063
|
-
prompt:
|
|
13279
|
+
prompt: doneWithoutWorkPrompt,
|
|
13064
13280
|
source: "done-claim-no-emoji",
|
|
13065
13281
|
priority: 1
|
|
13066
13282
|
};
|
|
13283
|
+
} else if (w.doneClaimNoTodosAttempts < maxRetries) {
|
|
13284
|
+
await log("info", `${short(sid)} - model claims done with no open todos. Sending verification prompt (attempt ${w.doneClaimNoTodosAttempts + 1}/${maxRetries})...`);
|
|
13285
|
+
bestCandidate = {
|
|
13286
|
+
prompt: doneWithoutWorkPrompt,
|
|
13287
|
+
source: "done-claim-no-todos",
|
|
13288
|
+
priority: 1
|
|
13289
|
+
};
|
|
13290
|
+
}
|
|
13291
|
+
}
|
|
13292
|
+
}
|
|
13293
|
+
}
|
|
13294
|
+
if (resumeOnActionIntent) {
|
|
13295
|
+
const lastAssistantMsg = messages.slice().reverse().find((m) => (m.role ?? m.info?.role) === "assistant");
|
|
13296
|
+
if (lastAssistantMsg) {
|
|
13297
|
+
let lastAssistantText = "";
|
|
13298
|
+
for (const part of lastAssistantMsg.parts || []) {
|
|
13299
|
+
lastAssistantText += (part.text ?? "") + `
|
|
13300
|
+
`;
|
|
13301
|
+
}
|
|
13302
|
+
if (containsActionIntent(lastAssistantText)) {
|
|
13303
|
+
dbg(`ACTION INTENT DETECTED in checkForToolCallAsText for ${short(sid)}`);
|
|
13304
|
+
const candidate = {
|
|
13305
|
+
prompt: actionIntentPrompt,
|
|
13306
|
+
source: "action-intent",
|
|
13307
|
+
priority: 2
|
|
13308
|
+
};
|
|
13309
|
+
if (!bestCandidate || candidate.priority < bestCandidate.priority) {
|
|
13310
|
+
bestCandidate = candidate;
|
|
13067
13311
|
}
|
|
13068
13312
|
}
|
|
13069
13313
|
}
|
|
@@ -13082,10 +13326,10 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13082
13326
|
}
|
|
13083
13327
|
if (!bestCandidate) {
|
|
13084
13328
|
const todos = w.todos || [];
|
|
13085
|
-
const hasOpenTodos = todos.some(
|
|
13329
|
+
const hasOpenTodos = todos.some(isOpenTodo);
|
|
13086
13330
|
if (hasOpenTodos && busyCount() === 0) {
|
|
13087
13331
|
const reminder = buildOpenTodosReminder(todos);
|
|
13088
|
-
await log("info", `${short(sid)} - no activity detected but todos remain open (${todos
|
|
13332
|
+
await log("info", `${short(sid)} - no activity detected but todos remain open (${getOpenTodos(todos).length} tasks). Sending reminder...`);
|
|
13089
13333
|
bestCandidate = {
|
|
13090
13334
|
prompt: reminder,
|
|
13091
13335
|
source: "idle-with-open-todos-reminder",
|
|
@@ -13098,19 +13342,22 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13098
13342
|
if (!bestCandidate)
|
|
13099
13343
|
return;
|
|
13100
13344
|
const isOpenTodosReminder = bestCandidate.source === "idle-with-open-todos-reminder";
|
|
13345
|
+
const isDoneClaimNoTodos = bestCandidate.source === "done-claim-no-todos";
|
|
13101
13346
|
if (isOpenTodosReminder) {
|
|
13102
13347
|
if (w.todoNudgeAttempts >= maxRetries) {
|
|
13103
|
-
await log("info", `${short(sid)} - max open-todos nudges (${maxRetries}) reached
|
|
13348
|
+
await log("info", `${short(sid)} - max open-todos nudges (${maxRetries}) reached, waiting for activity`);
|
|
13104
13349
|
return;
|
|
13105
13350
|
}
|
|
13106
|
-
|
|
13351
|
+
} else if (isDoneClaimNoTodos) {
|
|
13352
|
+
w.doneClaimNoTodosAttempts++;
|
|
13107
13353
|
} else {
|
|
13108
13354
|
w.toolTextRecovered = true;
|
|
13109
13355
|
w.toolTextAttempts++;
|
|
13110
13356
|
}
|
|
13111
|
-
|
|
13357
|
+
const attemptNum = isOpenTodosReminder ? w.todoNudgeAttempts : isDoneClaimNoTodos ? w.doneClaimNoTodosAttempts : w.toolTextAttempts;
|
|
13358
|
+
await log("info", `${bestCandidate.source} detected on ${short(sid)}! ` + `Attempt ${attemptNum}/${maxRetries}. Sending recovery prompt...`);
|
|
13112
13359
|
const timeSinceActivity = Date.now() - w.lastActivityAt;
|
|
13113
|
-
if (timeSinceActivity <
|
|
13360
|
+
if (timeSinceActivity < minActivityGapMs) {
|
|
13114
13361
|
await log("info", `${short(sid)} - skipping ${bestCandidate.source}, session was active ${Math.round(timeSinceActivity / 1000)}s ago`);
|
|
13115
13362
|
return;
|
|
13116
13363
|
}
|
|
@@ -13129,6 +13376,8 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13129
13376
|
} else {
|
|
13130
13377
|
try {
|
|
13131
13378
|
await sendContinuePrompt(sid, bestCandidate.prompt, w);
|
|
13379
|
+
if (isOpenTodosReminder)
|
|
13380
|
+
w.todoNudgeAttempts++;
|
|
13132
13381
|
await log("info", `${short(sid)} - ${bestCandidate.source} recovery sent (attempt ${w.toolTextAttempts})`);
|
|
13133
13382
|
} catch (err) {
|
|
13134
13383
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
@@ -13155,6 +13404,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13155
13404
|
try {
|
|
13156
13405
|
await ctx.client.session.abort({ path: { id: sid } });
|
|
13157
13406
|
await log("info", `${short(sid)} - abort OK`);
|
|
13407
|
+
dbg(`Abort succeeded on ${short(sid)}, waiting ${ABORT_CONTINUE_DELAY_MS}ms before continue prompt`);
|
|
13158
13408
|
} catch (err) {
|
|
13159
13409
|
const errMsg = err instanceof Error ? err.message : String(err);
|
|
13160
13410
|
await log("warn", `${short(sid)} - abort failed: ${errMsg}`);
|
|
@@ -13165,7 +13415,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13165
13415
|
if (w.status === "busy")
|
|
13166
13416
|
w.status = "idle";
|
|
13167
13417
|
try {
|
|
13168
|
-
await sendContinuePrompt(sid,
|
|
13418
|
+
await sendContinuePrompt(sid, continuePrompt, w);
|
|
13169
13419
|
await log("info", `${short(sid)} - abort+continue done`);
|
|
13170
13420
|
w.orphanWatchStartAt = null;
|
|
13171
13421
|
w.resumeAttempts++;
|
|
@@ -13185,7 +13435,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13185
13435
|
}
|
|
13186
13436
|
const now = Date.now();
|
|
13187
13437
|
const elapsedSinceRetry = now - w.lastRetryAt;
|
|
13188
|
-
const requiredBackoff = backoffMs(w.resumeAttempts);
|
|
13438
|
+
const requiredBackoff = backoffMs(w.resumeAttempts, baseBackoffMs, maxBackoffMs);
|
|
13189
13439
|
if (w.lastRetryAt > 0 && elapsedSinceRetry < requiredBackoff)
|
|
13190
13440
|
return false;
|
|
13191
13441
|
if (isHallucinationLoop(sid)) {
|
|
@@ -13207,7 +13457,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13207
13457
|
const idleSec = Math.round((now - w.lastActivityAt) / 1000);
|
|
13208
13458
|
await log("info", `${reason} on ${short(sid)} (${idleSec}s, retry ${w.resumeAttempts}/${maxRetries})`);
|
|
13209
13459
|
try {
|
|
13210
|
-
await sendContinuePrompt(sid, prompt ??
|
|
13460
|
+
await sendContinuePrompt(sid, prompt ?? continuePrompt, w);
|
|
13211
13461
|
await log("info", `${short(sid)} - retry sent`);
|
|
13212
13462
|
return true;
|
|
13213
13463
|
} catch (err) {
|
|
@@ -13300,6 +13550,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13300
13550
|
}
|
|
13301
13551
|
} else if (!w.gaveUp) {
|
|
13302
13552
|
w.gaveUp = true;
|
|
13553
|
+
dbg(`State transition on ${short(sid)}: gaveUp=false -> true`);
|
|
13303
13554
|
w.orphanWatchStartAt = null;
|
|
13304
13555
|
w.aborting = false;
|
|
13305
13556
|
log("warn", `${short(sid)} - orphan retries exhausted.`);
|
|
@@ -13359,11 +13610,67 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13359
13610
|
tryResume(sid, w, "Stream stall");
|
|
13360
13611
|
} else if (!w.gaveUp) {
|
|
13361
13612
|
w.gaveUp = true;
|
|
13613
|
+
dbg(`State transition on ${short(sid)}: gaveUp=false -> true`);
|
|
13362
13614
|
log("warn", `${short(sid)} - all ${maxRetries} retries exhausted.`);
|
|
13363
13615
|
}
|
|
13364
13616
|
}
|
|
13365
13617
|
}
|
|
13366
13618
|
}
|
|
13619
|
+
for (const [sid, w] of sessions) {
|
|
13620
|
+
if (w.pendingRecovery && w.status === "idle" && !w.userCancelled && !w.aborting && !w.continuing && !w.gaveUp && w.recoveryAttempts === 0) {
|
|
13621
|
+
dbg(`Pending recovery check on ${short(sid)}: pendingRecovery=${w.pendingRecovery}, status=${w.status}, userCancelled=${w.userCancelled}, aborting=${w.aborting}, continuing=${w.continuing}, gaveUp=${w.gaveUp}, recoveryAttempts=${w.recoveryAttempts}, pendingRecoveryAt=${w.pendingRecoveryAt}`);
|
|
13622
|
+
const elapsed = Date.now() - w.pendingRecoveryAt;
|
|
13623
|
+
const requiredBackoff2 = backoffMs(w.recoveryAttempts, baseBackoffMs, maxBackoffMs);
|
|
13624
|
+
if (elapsed < requiredBackoff2) {
|
|
13625
|
+
dbg(`Backoff check on ${short(sid)}: elapsed=${elapsed}ms, required=${requiredBackoff2}ms, attempt=${w.recoveryAttempts}, pass=false`);
|
|
13626
|
+
dbg(`Pending recovery on ${short(sid)} waiting for backoff: ${requiredBackoff2 - elapsed}ms remaining`);
|
|
13627
|
+
continue;
|
|
13628
|
+
}
|
|
13629
|
+
dbg(`Backoff check on ${short(sid)}: elapsed=${elapsed}ms, required=${requiredBackoff2}ms, attempt=${w.recoveryAttempts}, pass=true`);
|
|
13630
|
+
await log("info", `Pending recovery triggered on ${short(sid)}: reason=${w.pendingRecoveryReason}, attempt=${w.recoveryAttempts + 1}, maxRetries=${maxRecoveryRetries}`);
|
|
13631
|
+
dbg(`Recovery timing on ${short(sid)}: detectionToAttemptMs=${Date.now() - w.pendingRecoveryAt}`);
|
|
13632
|
+
w.recoveryAttempts++;
|
|
13633
|
+
dbg(`State transition on ${short(sid)}: recoveryAttempts=${w.recoveryAttempts - 1} -> ${w.recoveryAttempts}`);
|
|
13634
|
+
try {
|
|
13635
|
+
await sendContinuePrompt(sid, continuePrompt, w);
|
|
13636
|
+
} catch (err) {
|
|
13637
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
13638
|
+
await log("warn", `${short(sid)} - pending recovery failed: ${errMsg}`);
|
|
13639
|
+
w.recoveryAttempts = 0;
|
|
13640
|
+
}
|
|
13641
|
+
}
|
|
13642
|
+
if (w.status !== "idle")
|
|
13643
|
+
continue;
|
|
13644
|
+
if (w.isSubagent)
|
|
13645
|
+
continue;
|
|
13646
|
+
if (w.userCancelled || w.completionSignaled)
|
|
13647
|
+
continue;
|
|
13648
|
+
if (w.continuing)
|
|
13649
|
+
continue;
|
|
13650
|
+
if (busyCount() !== 0)
|
|
13651
|
+
continue;
|
|
13652
|
+
const open = getOpenTodos(w.todos || []);
|
|
13653
|
+
if (open.length === 0)
|
|
13654
|
+
continue;
|
|
13655
|
+
if (w.todoNudgeAttempts >= maxRetries)
|
|
13656
|
+
continue;
|
|
13657
|
+
const elapsedSinceLastNudge = Date.now() - w.lastRetryAt;
|
|
13658
|
+
const requiredBackoff = backoffMs(w.todoNudgeAttempts, baseBackoffMs, maxBackoffMs);
|
|
13659
|
+
if (w.lastRetryAt > 0 && elapsedSinceLastNudge < requiredBackoff)
|
|
13660
|
+
continue;
|
|
13661
|
+
const isCelebration = await lastAssistantEndsWithCelebration(sid);
|
|
13662
|
+
if (isCelebration) {
|
|
13663
|
+
w.toolTextRecovered = true;
|
|
13664
|
+
w.completionSignaled = true;
|
|
13665
|
+
continue;
|
|
13666
|
+
}
|
|
13667
|
+
const reminder = buildOpenTodosReminder(w.todos || []);
|
|
13668
|
+
const sent = await tryResume(sid, w, "Idle with open todos (periodic)", reminder);
|
|
13669
|
+
if (sent) {
|
|
13670
|
+
w.todoNudgeAttempts++;
|
|
13671
|
+
await log("info", `${short(sid)} - idle periodic recheck: nudge ${w.todoNudgeAttempts}/${maxRetries}`);
|
|
13672
|
+
}
|
|
13673
|
+
}
|
|
13367
13674
|
cleanupIdleSessions();
|
|
13368
13675
|
}, checkIntervalMs);
|
|
13369
13676
|
if (timer.unref)
|
|
@@ -13391,6 +13698,9 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13391
13698
|
w.status = statusType;
|
|
13392
13699
|
if (statusType === "busy") {
|
|
13393
13700
|
w.lastActivityAt = Date.now();
|
|
13701
|
+
if (w.pendingRecovery) {
|
|
13702
|
+
dbg(`Pending recovery cleared on ${short(sid)}: reason=session-busy`);
|
|
13703
|
+
}
|
|
13394
13704
|
resetSessionFlags(w);
|
|
13395
13705
|
prevBusyCount = busyCount();
|
|
13396
13706
|
log("debug", `${short(sid)} -> busy (${prevBusyCount})`);
|
|
@@ -13419,9 +13729,25 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13419
13729
|
prevBusyCount = currentBusy;
|
|
13420
13730
|
log("debug", `${short(sid)} -> idle (${currentBusy})`);
|
|
13421
13731
|
if (!w.isSubagent) {
|
|
13732
|
+
if (!w.pendingRecovery && !w.completionSignaled && !w.userCancelled && !w.aborting) {
|
|
13733
|
+
try {
|
|
13734
|
+
const errInfo = getLastAssistantError(await getSessionMessages(sid));
|
|
13735
|
+
if (errInfo && isStreamingFailure(errInfo.name, errInfo.message, streamingFailureErrorNames, streamingFailureMessagePatterns)) {
|
|
13736
|
+
w.pendingRecovery = true;
|
|
13737
|
+
w.pendingRecoveryReason = errInfo.name;
|
|
13738
|
+
w.pendingRecoveryAt = Date.now();
|
|
13739
|
+
dbg(`State transition on ${short(sid)}: pendingRecovery=false -> true, reason=${errInfo.name}`);
|
|
13740
|
+
await log("info", `${short(sid)} - streaming failure detected on idle: ${errInfo.name} - ${errInfo.message}`);
|
|
13741
|
+
await tryResume(sid, w, "Streaming failure on idle", continuePrompt);
|
|
13742
|
+
}
|
|
13743
|
+
} catch (e) {
|
|
13744
|
+
const errMsg = e instanceof Error ? e.message : String(e);
|
|
13745
|
+
dbg(`session.idle sid=${short(sid)}: streaming-failure check error: ${errMsg}`);
|
|
13746
|
+
}
|
|
13747
|
+
}
|
|
13422
13748
|
const todos = w.todos || [];
|
|
13423
|
-
const
|
|
13424
|
-
if (
|
|
13749
|
+
const open = getOpenTodos(todos);
|
|
13750
|
+
if (open.length > 0 && currentBusy === 0 && !w.completionSignaled && !w.userCancelled && w.todoNudgeAttempts < maxRetries) {
|
|
13425
13751
|
const isCelebration = await lastAssistantEndsWithCelebration(sid);
|
|
13426
13752
|
if (isCelebration) {
|
|
13427
13753
|
await log("info", `${short(sid)} - \uD83C\uDF89 detected in idle handler, skipping continue`);
|
|
@@ -13432,18 +13758,52 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13432
13758
|
w.toolTextTimer = null;
|
|
13433
13759
|
}
|
|
13434
13760
|
} else {
|
|
13761
|
+
w.todoNudgeAttempts++;
|
|
13435
13762
|
const reminder = buildOpenTodosReminder(todos);
|
|
13436
|
-
await log("info", `${short(sid)} - idle with ${
|
|
13437
|
-
tryResume(sid, w, "Idle with open todos", reminder);
|
|
13763
|
+
await log("info", `${short(sid)} - idle with ${open.length} open todos. Sending reminder (nudge ${w.todoNudgeAttempts}/${maxRetries})...`);
|
|
13764
|
+
await tryResume(sid, w, "Idle with open todos", reminder);
|
|
13438
13765
|
}
|
|
13439
13766
|
}
|
|
13440
13767
|
}
|
|
13441
13768
|
if (!w.completionSignaled && !w.userCancelled && w.toolTextAttempts < maxRetries) {
|
|
13769
|
+
if (resumeOnActionIntent && !w.toolTextRecovered) {
|
|
13770
|
+
const idleSid = sid;
|
|
13771
|
+
const idleW = w;
|
|
13772
|
+
setTimeout(async () => {
|
|
13773
|
+
try {
|
|
13774
|
+
if (Date.now() - idleW.createdAt < warmupMs) {
|
|
13775
|
+
dbg(`session.idle sid=${short(idleSid)}: skipping action intent, session is warming up (${Date.now() - idleW.createdAt}ms < ${warmupMs}ms)`);
|
|
13776
|
+
return;
|
|
13777
|
+
}
|
|
13778
|
+
const response = await ctx.client.session.messages({ path: { id: idleSid } });
|
|
13779
|
+
const msgs = extractMessages(response);
|
|
13780
|
+
const lastAssistantMsg = msgs.slice().reverse().find((m) => (m.role ?? m.info?.role) === "assistant");
|
|
13781
|
+
if (lastAssistantMsg) {
|
|
13782
|
+
let lastText = "";
|
|
13783
|
+
for (const part of lastAssistantMsg.parts || []) {
|
|
13784
|
+
lastText += (part.text ?? "") + `
|
|
13785
|
+
`;
|
|
13786
|
+
}
|
|
13787
|
+
if (containsActionIntent(lastText)) {
|
|
13788
|
+
const w2 = sessions.get(idleSid);
|
|
13789
|
+
if (!w2 || w2.toolTextRecovered || w2.completionSignaled || w2.status !== "idle")
|
|
13790
|
+
return;
|
|
13791
|
+
w2.toolTextRecovered = true;
|
|
13792
|
+
w2.toolTextAttempts++;
|
|
13793
|
+
dbg(`session.idle sid=${short(idleSid)}: ACTION INTENT DETECTED, sending "${actionIntentPrompt.slice(0, 40)}..."`);
|
|
13794
|
+
await sendContinuePrompt(idleSid, actionIntentPrompt, w2);
|
|
13795
|
+
}
|
|
13796
|
+
}
|
|
13797
|
+
} catch (e) {
|
|
13798
|
+
dbg(`session.idle sid=${short(idleSid)}: delayed check error: ${e}`);
|
|
13799
|
+
}
|
|
13800
|
+
}, 500);
|
|
13801
|
+
}
|
|
13442
13802
|
if (w.toolTextTimer)
|
|
13443
13803
|
clearTimeout(w.toolTextTimer);
|
|
13444
13804
|
w.toolTextTimer = setTimeout(() => {
|
|
13445
13805
|
checkForToolCallAsText(sid, w);
|
|
13446
|
-
},
|
|
13806
|
+
}, toolTextCheckDelayMs);
|
|
13447
13807
|
}
|
|
13448
13808
|
} else if (statusType === "retry") {
|
|
13449
13809
|
touchSession(sid);
|
|
@@ -13472,12 +13832,45 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13472
13832
|
if (w) {
|
|
13473
13833
|
w.status = "idle";
|
|
13474
13834
|
resetIdleFlags(w);
|
|
13475
|
-
|
|
13835
|
+
dbg(`session.idle sid=${short(sid)}: resetIdleFlags done, toolTextRecovered=${w.toolTextRecovered}, toolTextAttempts=${w.toolTextAttempts}, maxRetries=${maxRetries}`);
|
|
13836
|
+
if (resumeOnActionIntent && !w.toolTextRecovered && !w.completionSignaled) {
|
|
13837
|
+
setTimeout(async () => {
|
|
13838
|
+
try {
|
|
13839
|
+
if (Date.now() - w.createdAt < warmupMs) {
|
|
13840
|
+
dbg(`session.idle sid=${short(sid)}: skipping action intent, session is warming up (${Date.now() - w.createdAt}ms < ${warmupMs}ms)`);
|
|
13841
|
+
return;
|
|
13842
|
+
}
|
|
13843
|
+
const response = await ctx.client.session.messages({ path: { id: sid } });
|
|
13844
|
+
const msgs = extractMessages(response);
|
|
13845
|
+
const lastAssistantMsg = msgs.slice().reverse().find((m) => (m.role ?? m.info?.role) === "assistant");
|
|
13846
|
+
if (lastAssistantMsg) {
|
|
13847
|
+
let lastText = "";
|
|
13848
|
+
for (const part of lastAssistantMsg.parts || []) {
|
|
13849
|
+
lastText += (part.text ?? "") + `
|
|
13850
|
+
`;
|
|
13851
|
+
}
|
|
13852
|
+
if (containsActionIntent(lastText)) {
|
|
13853
|
+
const w2 = sessions.get(sid);
|
|
13854
|
+
if (!w2 || w2.toolTextRecovered || w2.completionSignaled || w2.status !== "idle")
|
|
13855
|
+
return;
|
|
13856
|
+
w2.toolTextRecovered = true;
|
|
13857
|
+
w2.toolTextAttempts++;
|
|
13858
|
+
dbg(`session.idle sid=${short(sid)}: ACTION INTENT DETECTED, sending "${actionIntentPrompt.slice(0, 40)}..."`);
|
|
13859
|
+
await sendContinuePrompt(sid, actionIntentPrompt, w2);
|
|
13860
|
+
}
|
|
13861
|
+
}
|
|
13862
|
+
} catch (e) {
|
|
13863
|
+
dbg(`session.idle sid=${short(sid)}: delayed check error: ${e}`);
|
|
13864
|
+
}
|
|
13865
|
+
}, 500);
|
|
13866
|
+
}
|
|
13867
|
+
const shouldSet = !w.toolTextRecovered && w.toolTextAttempts < maxRetries;
|
|
13868
|
+
if (shouldSet) {
|
|
13476
13869
|
if (w.toolTextTimer)
|
|
13477
13870
|
clearTimeout(w.toolTextTimer);
|
|
13478
13871
|
w.toolTextTimer = setTimeout(() => {
|
|
13479
13872
|
checkForToolCallAsText(sid, w);
|
|
13480
|
-
},
|
|
13873
|
+
}, toolTextCheckDelayMs);
|
|
13481
13874
|
}
|
|
13482
13875
|
}
|
|
13483
13876
|
break;
|
|
@@ -13514,6 +13907,7 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13514
13907
|
case "session.error": {
|
|
13515
13908
|
const errorObj = getError(ev);
|
|
13516
13909
|
const errorName = errorObj?.name ?? "";
|
|
13910
|
+
const errorMessage = errorObj?.data?.message ?? String(errorObj?.data ?? "");
|
|
13517
13911
|
const isMessageAborted = errorName === "MessageAbortedError";
|
|
13518
13912
|
if (isMessageAborted) {
|
|
13519
13913
|
for (const [wSid, w] of sessions) {
|
|
@@ -13526,9 +13920,24 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13526
13920
|
log("info", "User abort (ESC)");
|
|
13527
13921
|
break;
|
|
13528
13922
|
}
|
|
13923
|
+
const isStreamingFail = isStreamingFailure(errorName, errorMessage, streamingFailureErrorNames, streamingFailureMessagePatterns);
|
|
13924
|
+
if (isStreamingFail) {
|
|
13925
|
+
if (sid) {
|
|
13926
|
+
const w = sessions.get(sid);
|
|
13927
|
+
if (w && w.status === "busy") {
|
|
13928
|
+
w.pendingRecovery = true;
|
|
13929
|
+
w.pendingRecoveryReason = errorName;
|
|
13930
|
+
w.pendingRecoveryAt = Date.now();
|
|
13931
|
+
dbg(`State transition on ${short(sid)}: pendingRecovery=false -> true, reason=${errorName}`);
|
|
13932
|
+
await log("info", `Streaming failure detected on ${short(sid)}: errorName=${errorName}, errorMessage=${errorMessage}, pendingRecoveryReason=${errorName}`);
|
|
13933
|
+
}
|
|
13934
|
+
log("info", `Streaming failure detected: ${errorName} - ${errorMessage}`);
|
|
13935
|
+
} else {
|
|
13936
|
+
log("warn", `Streaming failure detected but no session ID: ${errorName} - ${errorMessage}`);
|
|
13937
|
+
}
|
|
13938
|
+
}
|
|
13529
13939
|
if (busyCount() === 0)
|
|
13530
13940
|
break;
|
|
13531
|
-
const errorMessage = errorObj?.data?.message ?? String(errorObj?.data ?? "");
|
|
13532
13941
|
log("debug", `Session error: ${errorName} - ${errorMessage}`);
|
|
13533
13942
|
if (sid) {
|
|
13534
13943
|
const w = sessions.get(sid);
|
|
@@ -13540,7 +13949,10 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13540
13949
|
break;
|
|
13541
13950
|
}
|
|
13542
13951
|
case "command.executed": {
|
|
13543
|
-
for (const [, w2] of sessions) {
|
|
13952
|
+
for (const [sid2, w2] of sessions) {
|
|
13953
|
+
if (w2.pendingRecovery) {
|
|
13954
|
+
dbg(`Pending recovery cleared on ${short(sid2)}: reason=user-command`);
|
|
13955
|
+
}
|
|
13544
13956
|
resetSessionFlags(w2);
|
|
13545
13957
|
}
|
|
13546
13958
|
if (!sid)
|
|
@@ -13561,6 +13973,12 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13561
13973
|
const w = sessions.get(ctx2.sessionID);
|
|
13562
13974
|
if (w) {
|
|
13563
13975
|
if (!w.isSubagent) {
|
|
13976
|
+
const openTodos = (w.todos || []).filter((t) => t.status === "pending" || t.status === "in_progress");
|
|
13977
|
+
if (openTodos.length > 0 && w.taskCompleteOverrides < maxRetries) {
|
|
13978
|
+
w.taskCompleteOverrides++;
|
|
13979
|
+
await log("info", `${short(ctx2.sessionID)} - task_complete blocked: ${openTodos.length} open todos remain (override ${w.taskCompleteOverrides}/${maxRetries})`);
|
|
13980
|
+
return `You have ${openTodos.length} unfinished task(s). Please complete all remaining work before signaling completion.`;
|
|
13981
|
+
}
|
|
13564
13982
|
w.toolTextRecovered = true;
|
|
13565
13983
|
w.completionSignaled = true;
|
|
13566
13984
|
if (w.toolTextTimer) {
|
|
@@ -13612,7 +14030,10 @@ var AutoResumePlugin = async (ctx, options) => {
|
|
|
13612
14030
|
};
|
|
13613
14031
|
var src_default = AutoResumePlugin;
|
|
13614
14032
|
export {
|
|
14033
|
+
isStreamingFailure,
|
|
14034
|
+
getLastAssistantError,
|
|
13615
14035
|
src_default as default,
|
|
13616
14036
|
buildOpenTodosReminder,
|
|
14037
|
+
backoffMs,
|
|
13617
14038
|
AutoResumePlugin
|
|
13618
14039
|
};
|
package/package.json
CHANGED