@bybrawe/opencode-loop 0.5.34 → 0.5.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +1 -1
- package/package.json +3 -3
- package/scripts/network-recovery-test.mjs +218 -0
- package/scripts/session-status-test.mjs +18 -4
- package/src/index.js +319 -29
- package/src/source/core/continuation.js +53 -2
- package/src/source/runtime/executor.js +145 -16
- package/src/source/runtime/network-recovery.js +40 -0
- package/src/source/runtime/run-finalization.js +28 -0
- package/src/source/runtime/session-status.js +29 -30
- package/src/source/runtime/terminal-guard.js +56 -0
|
@@ -10,9 +10,12 @@ import { createCompactionRuntime } from "./compaction.js"
|
|
|
10
10
|
import { createActionDispatcher } from "./action-dispatch.js"
|
|
11
11
|
import { createRunFinalizationRuntime } from "./run-finalization.js"
|
|
12
12
|
import { createRunAdmissionRuntime } from "./run-admission.js"
|
|
13
|
+
import { isTransientNetworkError, networkRetryDelayMs, refundInfrastructureRun } from "./network-recovery.js"
|
|
13
14
|
|
|
14
15
|
const DEFAULT_ACTIVE_GUARD_MS = 45_000
|
|
15
16
|
const DEFAULT_BUSY_RETRY_MS = 5_000
|
|
17
|
+
const DEFAULT_PROVIDER_RETRY_WATCHDOG_MS = 2 * 60_000
|
|
18
|
+
const DEFAULT_NETWORK_RETRY_MAX_MS = 60_000
|
|
16
19
|
|
|
17
20
|
function requireFunction(value, label) {
|
|
18
21
|
if (typeof value !== "function") throw new TypeError(`createLoopExecutor requires ${label}`)
|
|
@@ -51,9 +54,16 @@ export function createLoopExecutor(options = {}) {
|
|
|
51
54
|
const busyRetryMs = Number.isFinite(Number(options.busyRetryMs)) && Number(options.busyRetryMs) > 0
|
|
52
55
|
? Number(options.busyRetryMs)
|
|
53
56
|
: DEFAULT_BUSY_RETRY_MS
|
|
57
|
+
const providerRetryWatchdogMs = Number.isFinite(Number(options.providerRetryWatchdogMs)) && Number(options.providerRetryWatchdogMs) > 0
|
|
58
|
+
? Number(options.providerRetryWatchdogMs)
|
|
59
|
+
: DEFAULT_PROVIDER_RETRY_WATCHDOG_MS
|
|
60
|
+
const networkRetryMaxMs = Number.isFinite(Number(options.networkRetryMaxMs)) && Number(options.networkRetryMaxMs) > 0
|
|
61
|
+
? Number(options.networkRetryMaxMs)
|
|
62
|
+
: DEFAULT_NETWORK_RETRY_MAX_MS
|
|
54
63
|
|
|
55
64
|
const activeRuns = new Map()
|
|
56
65
|
const runLocks = new Map()
|
|
66
|
+
const retryRuns = new Map()
|
|
57
67
|
|
|
58
68
|
const statusRuntime = createSessionStatusRuntime({
|
|
59
69
|
activeRuns,
|
|
@@ -67,6 +77,7 @@ export function createLoopExecutor(options = {}) {
|
|
|
67
77
|
updateSessionStatusFromEvent,
|
|
68
78
|
staleActiveRun,
|
|
69
79
|
canFinalizeActiveRun,
|
|
80
|
+
sessionStatusType,
|
|
70
81
|
sessionIsIdle,
|
|
71
82
|
markSessionStatus,
|
|
72
83
|
clearSessionStatus,
|
|
@@ -129,6 +140,7 @@ export function createLoopExecutor(options = {}) {
|
|
|
129
140
|
if (active?.timer) clearTimeout(active.timer)
|
|
130
141
|
compactionRuntime.clearForActiveRun(sessionID, active)
|
|
131
142
|
activeRuns.delete(sessionID)
|
|
143
|
+
retryRuns.delete(sessionID)
|
|
132
144
|
}
|
|
133
145
|
|
|
134
146
|
function disposeSession(sessionID) {
|
|
@@ -138,32 +150,116 @@ export function createLoopExecutor(options = {}) {
|
|
|
138
150
|
clearSessionStatus(sessionID)
|
|
139
151
|
}
|
|
140
152
|
|
|
153
|
+
async function persistInfrastructureRefund(directory, sessionID, active, input = {}) {
|
|
154
|
+
const state = await readState(directory, sessionID)
|
|
155
|
+
const job = (state.jobs || []).find((candidate) => candidate.id === active.jobId)
|
|
156
|
+
if (!job) return { job: undefined, delayMs: busyRetryMs }
|
|
157
|
+
refundInfrastructureRun(job, {
|
|
158
|
+
runCount: active.job?.runCount,
|
|
159
|
+
previousLastRunAt: active.previousLastRunAt,
|
|
160
|
+
disabledByMaxRuns: active.disabledByMaxRuns,
|
|
161
|
+
}, {
|
|
162
|
+
reason: input.reason,
|
|
163
|
+
error: input.error,
|
|
164
|
+
now: now(),
|
|
165
|
+
})
|
|
166
|
+
state.jobs = (state.jobs || []).map((candidate) => candidate.id === job.id ? job : candidate)
|
|
167
|
+
await writeState(directory, sessionID, state)
|
|
168
|
+
const delayMs = networkRetryDelayMs(job.infrastructureFailureCount || 1, busyRetryMs, networkRetryMaxMs)
|
|
169
|
+
return { job, delayMs }
|
|
170
|
+
}
|
|
171
|
+
|
|
141
172
|
async function recoverActiveDispatchFailure(directory, client, sessionID, jobId, runToken, error) {
|
|
142
173
|
const active = activeRuns.get(sessionID)
|
|
143
174
|
if (!active || active.jobId !== jobId || active.runToken !== runToken) return false
|
|
144
175
|
|
|
176
|
+
const transient = isTransientNetworkError(error)
|
|
177
|
+
const snapshot = active
|
|
145
178
|
clearActiveRun(sessionID)
|
|
146
179
|
clearSessionStatus(sessionID)
|
|
147
180
|
|
|
148
181
|
const message = errorMessage(error)
|
|
149
|
-
|
|
150
|
-
|
|
182
|
+
let state = await readState(directory, sessionID)
|
|
183
|
+
let job = (state.jobs || []).find((candidate) => candidate.id === jobId)
|
|
184
|
+
let retryDelay = busyRetryMs
|
|
151
185
|
if (job) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
job
|
|
158
|
-
|
|
186
|
+
if (transient) {
|
|
187
|
+
const refunded = await persistInfrastructureRefund(directory, sessionID, snapshot, {
|
|
188
|
+
reason: "dispatch_failed_transient",
|
|
189
|
+
error,
|
|
190
|
+
})
|
|
191
|
+
job = refunded.job
|
|
192
|
+
retryDelay = refunded.delayMs
|
|
193
|
+
state = await readState(directory, sessionID)
|
|
194
|
+
} else {
|
|
195
|
+
job.failureCount = (job.failureCount || 0) + 1
|
|
196
|
+
job.lastFailureReason = "dispatch_failed"
|
|
197
|
+
job.lastDispatchFailure = message.slice(0, 4000)
|
|
198
|
+
job.lastDispatchFailureAt = now()
|
|
199
|
+
if (job.maxFailures > 0 && job.failureCount >= job.maxFailures) {
|
|
200
|
+
job.paused = true
|
|
201
|
+
await notifyJob(directory, job, "dispatch_failed")
|
|
202
|
+
}
|
|
203
|
+
state.jobs = (state.jobs || []).map((candidate) => candidate.id === job.id ? job : candidate)
|
|
204
|
+
await writeState(directory, sessionID, state)
|
|
159
205
|
}
|
|
160
|
-
state.jobs = (state.jobs || []).map((candidate) => candidate.id === job.id ? job : candidate)
|
|
161
|
-
await writeState(directory, sessionID, state)
|
|
162
206
|
}
|
|
163
207
|
|
|
164
|
-
await appendLoopLog(directory, "dispatch-error"
|
|
165
|
-
|
|
166
|
-
|
|
208
|
+
await appendLoopLog(directory, transient ? "network-dispatch-error" : "dispatch-error", {
|
|
209
|
+
sessionID,
|
|
210
|
+
job: job?.name || jobId,
|
|
211
|
+
error: message,
|
|
212
|
+
...(transient ? { retryInMs: retryDelay } : {}),
|
|
213
|
+
})
|
|
214
|
+
if (transient) {
|
|
215
|
+
await toast(client, `Loop network dispatch failed; retrying when the session recovers: ${message}`, "warning")
|
|
216
|
+
} else {
|
|
217
|
+
await toast(client, `Loop dispatch failed${job?.paused ? " and paused" : ""}: ${message}`, job?.paused ? "error" : "warning")
|
|
218
|
+
}
|
|
219
|
+
await scheduleDueWork(directory, client, sessionID, retryDelay)
|
|
220
|
+
return true
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
async function recoverStuckProviderRetry(directory, client, sessionID) {
|
|
224
|
+
const active = activeRuns.get(sessionID)
|
|
225
|
+
if (!active || active.compactionOnly) {
|
|
226
|
+
retryRuns.delete(sessionID)
|
|
227
|
+
return false
|
|
228
|
+
}
|
|
229
|
+
const current = retryRuns.get(sessionID)
|
|
230
|
+
if (!current || current.runToken !== active.runToken) {
|
|
231
|
+
retryRuns.set(sessionID, { runToken: active.runToken, since: now() })
|
|
232
|
+
return false
|
|
233
|
+
}
|
|
234
|
+
if (now() - current.since < providerRetryWatchdogMs) return false
|
|
235
|
+
|
|
236
|
+
const snapshot = active
|
|
237
|
+
try {
|
|
238
|
+
if (client?.session?.abort) {
|
|
239
|
+
await fireSdk(
|
|
240
|
+
client,
|
|
241
|
+
"session.abort provider retry watchdog",
|
|
242
|
+
client.session.abort.bind(client.session),
|
|
243
|
+
{ path: { id: sessionID }, body: {} },
|
|
244
|
+
{ path: { sessionID }, body: {} },
|
|
245
|
+
{ sessionID },
|
|
246
|
+
)
|
|
247
|
+
}
|
|
248
|
+
} catch {}
|
|
249
|
+
clearActiveRun(sessionID)
|
|
250
|
+
clearSessionStatus(sessionID)
|
|
251
|
+
const refunded = await persistInfrastructureRefund(directory, sessionID, snapshot, {
|
|
252
|
+
reason: "provider_retry_watchdog",
|
|
253
|
+
error: `OpenCode session.status stayed retry for ${providerRetryWatchdogMs}ms`,
|
|
254
|
+
})
|
|
255
|
+
await appendLoopLog(directory, "provider-retry-recovery", {
|
|
256
|
+
sessionID,
|
|
257
|
+
job: refunded.job?.name || snapshot.jobId,
|
|
258
|
+
retryForMs: now() - current.since,
|
|
259
|
+
retryInMs: refunded.delayMs,
|
|
260
|
+
})
|
|
261
|
+
await toast(client, "Loop provider retry exceeded the watchdog; the Loop-owned turn was released and will retry with backoff.", "warning")
|
|
262
|
+
await scheduleDueWork(directory, client, sessionID, refunded.delayMs)
|
|
167
263
|
return true
|
|
168
264
|
}
|
|
169
265
|
|
|
@@ -217,13 +313,22 @@ export function createLoopExecutor(options = {}) {
|
|
|
217
313
|
}
|
|
218
314
|
runLocks.set(sessionID, now())
|
|
219
315
|
let job
|
|
316
|
+
let previousLastRunAt = 0
|
|
317
|
+
let disabledByMaxRuns = false
|
|
220
318
|
try {
|
|
221
319
|
await finalizeActiveRun(directory, client, sessionID, { requireIdle: true, forceStale: true })
|
|
222
|
-
|
|
320
|
+
const statusType = await sessionStatusType(client, sessionID, directory)
|
|
321
|
+
if (statusType !== "idle") {
|
|
322
|
+
if (statusType === "retry") {
|
|
323
|
+
if (await recoverStuckProviderRetry(directory, client, sessionID)) return
|
|
324
|
+
} else {
|
|
325
|
+
retryRuns.delete(sessionID)
|
|
326
|
+
}
|
|
223
327
|
if (runOptions.force) await toast(client, "Loop queued: session is busy; it will run on the next idle check.", "info")
|
|
224
328
|
await reschedule(busyRetryMs)
|
|
225
329
|
return
|
|
226
330
|
}
|
|
331
|
+
retryRuns.delete(sessionID)
|
|
227
332
|
|
|
228
333
|
const active = activeRuns.get(sessionID)
|
|
229
334
|
const activeAge = active ? now() - (active.startedAt || 0) : 0
|
|
@@ -286,9 +391,11 @@ export function createLoopExecutor(options = {}) {
|
|
|
286
391
|
|
|
287
392
|
if (runNowRequested) delete job.runNowRequestedAt
|
|
288
393
|
job.watchTriggered = false
|
|
394
|
+
previousLastRunAt = Number(job.lastRunAt || 0)
|
|
289
395
|
job.lastRunAt = now()
|
|
290
396
|
job.runCount = (job.runCount || 0) + 1
|
|
291
|
-
|
|
397
|
+
disabledByMaxRuns = job.maxRuns > 0 && job.runCount >= job.maxRuns
|
|
398
|
+
if (disabledByMaxRuns) {
|
|
292
399
|
job.enabled = false
|
|
293
400
|
await notifyJob(directory, job, "max_runs_reached")
|
|
294
401
|
}
|
|
@@ -337,6 +444,8 @@ export function createLoopExecutor(options = {}) {
|
|
|
337
444
|
timer,
|
|
338
445
|
runToken,
|
|
339
446
|
compactionAction: result.compaction === true,
|
|
447
|
+
previousLastRunAt,
|
|
448
|
+
disabledByMaxRuns,
|
|
340
449
|
})
|
|
341
450
|
if (result.compaction && compactionRuntime.isCompleted(sessionID, job.id)) {
|
|
342
451
|
await compactionRuntime.finalize(directory, client, sessionID)
|
|
@@ -352,6 +461,24 @@ export function createLoopExecutor(options = {}) {
|
|
|
352
461
|
await reschedule(busyRetryMs)
|
|
353
462
|
} catch (error) {
|
|
354
463
|
clearActiveRun(sessionID)
|
|
464
|
+
if (isTransientNetworkError(error) && job) {
|
|
465
|
+
const stateAfterFailure = await readState(directory, sessionID)
|
|
466
|
+
const persisted = (stateAfterFailure.jobs || []).find((candidate) => candidate.id === job.id)
|
|
467
|
+
if (persisted) {
|
|
468
|
+
refundInfrastructureRun(persisted, {
|
|
469
|
+
runCount: job.runCount,
|
|
470
|
+
previousLastRunAt,
|
|
471
|
+
disabledByMaxRuns,
|
|
472
|
+
}, { reason: "action_dispatch_transient", error, now: now() })
|
|
473
|
+
stateAfterFailure.jobs = (stateAfterFailure.jobs || []).map((candidate) => candidate.id === persisted.id ? persisted : candidate)
|
|
474
|
+
await writeState(directory, sessionID, stateAfterFailure)
|
|
475
|
+
const delayMs = networkRetryDelayMs(persisted.infrastructureFailureCount || 1, busyRetryMs, networkRetryMaxMs)
|
|
476
|
+
await appendLoopLog(directory, "network-action-error", { sessionID, job: persisted.name || persisted.id, error: errorMessage(error), retryInMs: delayMs })
|
|
477
|
+
await toast(client, "Loop action hit a transient network failure; the logical run was refunded and will retry.", "warning")
|
|
478
|
+
await reschedule(delayMs)
|
|
479
|
+
return
|
|
480
|
+
}
|
|
481
|
+
}
|
|
355
482
|
await toast(client, `Loop job failed: ${error instanceof Error ? error.message : String(error)}`, "error")
|
|
356
483
|
await appendLoopLog(directory, "error", {
|
|
357
484
|
sessionID,
|
|
@@ -370,9 +497,11 @@ export function createLoopExecutor(options = {}) {
|
|
|
370
497
|
clearActiveRun,
|
|
371
498
|
disposeSession,
|
|
372
499
|
recoverActiveDispatchFailure,
|
|
500
|
+
recoverStuckProviderRetry,
|
|
373
501
|
finalizeActiveRun,
|
|
374
502
|
fireAction,
|
|
375
503
|
maybeRunDueJobs,
|
|
504
|
+
sessionStatusType,
|
|
376
505
|
sessionIsIdle,
|
|
377
506
|
updateSessionStatusFromEvent,
|
|
378
507
|
markSessionStatus,
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const TRANSIENT_NETWORK_PATTERNS = [
|
|
2
|
+
/\b(?:408|425|429|500|502|503|504|524)\b/i,
|
|
3
|
+
/rate[\s_-]?limit|too many requests|overloaded|service[\s_-]?unavailable|provider[_ -]?unavailable/i,
|
|
4
|
+
/terminated|fetch failed|failed to fetch|network[\s_-]?error|network connection lost/i,
|
|
5
|
+
/connection (?:error|refused|lost)|socket (?:hang up|connection was closed)|reset before headers/i,
|
|
6
|
+
/\b(?:enotfound|eai_again|econnrefused|econnreset|etimedout|ehostunreach|enetunreach|epipe)\b/i,
|
|
7
|
+
/\b(?:request|response|connection|network|stream|read) (?:timeout|timed out|time out)\b/i,
|
|
8
|
+
/\btimeout(?:error)?\b/i,
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
function errorText(value) {
|
|
12
|
+
if (value instanceof Error) return `${value.name}: ${value.message}`
|
|
13
|
+
if (typeof value === "string") return value
|
|
14
|
+
try { return JSON.stringify(value) } catch { return String(value) }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function isTransientNetworkError(value) {
|
|
18
|
+
const text = errorText(value)
|
|
19
|
+
return TRANSIENT_NETWORK_PATTERNS.some((pattern) => pattern.test(text))
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function networkRetryDelayMs(attempt, baseMs = 5_000, maxMs = 60_000) {
|
|
23
|
+
const safeAttempt = Math.max(1, Math.floor(Number(attempt) || 1))
|
|
24
|
+
const safeBase = Math.max(1, Math.floor(Number(baseMs) || 5_000))
|
|
25
|
+
const safeMax = Math.max(safeBase, Math.floor(Number(maxMs) || 60_000))
|
|
26
|
+
return Math.min(safeMax, safeBase * (2 ** Math.min(8, safeAttempt - 1)))
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function refundInfrastructureRun(job, snapshot = {}, input = {}) {
|
|
30
|
+
const chargedCount = Number(snapshot.runCount ?? job.runCount ?? 0)
|
|
31
|
+
const currentCount = Number(job.runCount || 0)
|
|
32
|
+
if (chargedCount > 0 && currentCount >= chargedCount) job.runCount = Math.max(0, currentCount - 1)
|
|
33
|
+
if (Number.isFinite(Number(snapshot.previousLastRunAt))) job.lastRunAt = Number(snapshot.previousLastRunAt)
|
|
34
|
+
if (snapshot.disabledByMaxRuns && job.maxRuns > 0 && job.runCount < job.maxRuns) job.enabled = true
|
|
35
|
+
job.infrastructureFailureCount = (job.infrastructureFailureCount || 0) + 1
|
|
36
|
+
job.lastInfrastructureFailure = String(input.reason || "transient_network_failure").slice(0, 120)
|
|
37
|
+
job.lastInfrastructureError = errorText(input.error).slice(0, 4000)
|
|
38
|
+
job.lastInfrastructureFailureAt = Number(input.now || Date.now())
|
|
39
|
+
return job
|
|
40
|
+
}
|
|
@@ -5,6 +5,7 @@ import { appendLoopLog as defaultAppendLoopLog, runShellCommand as defaultRunShe
|
|
|
5
5
|
import { toast as defaultToast } from "../opencode/host.js"
|
|
6
6
|
import { writeGoalReport as defaultWriteGoalReport } from "./goal-runtime.js"
|
|
7
7
|
import { dangerousShell as defaultDangerousShell } from "./job-workspace.js"
|
|
8
|
+
import { applyTerminalContinuationGuard as defaultApplyTerminalContinuationGuard } from "./terminal-guard.js"
|
|
8
9
|
|
|
9
10
|
function requireFunction(value, label) {
|
|
10
11
|
if (typeof value !== "function") throw new TypeError(`createRunFinalizationRuntime requires ${label}`)
|
|
@@ -25,8 +26,22 @@ export function createRunFinalizationRuntime(options = {}) {
|
|
|
25
26
|
const toast = typeof options.toast === "function" ? options.toast : defaultToast
|
|
26
27
|
const writeGoalReport = typeof options.writeGoalReport === "function" ? options.writeGoalReport : defaultWriteGoalReport
|
|
27
28
|
const dangerousShell = typeof options.dangerousShell === "function" ? options.dangerousShell : defaultDangerousShell
|
|
29
|
+
const applyTerminalContinuationGuard = typeof options.applyTerminalContinuationGuard === "function"
|
|
30
|
+
? options.applyTerminalContinuationGuard
|
|
31
|
+
: defaultApplyTerminalContinuationGuard
|
|
28
32
|
|
|
29
33
|
async function finalizeJob(directory, client, sessionID, state, job, previousJob) {
|
|
34
|
+
// Infrastructure backoff is consecutive, not lifetime. Reaching normal
|
|
35
|
+
// finalization proves a real Loop-owned turn completed, so the next future
|
|
36
|
+
// outage should restart from the shortest backoff rather than inheriting an
|
|
37
|
+
// old 60s cap from unrelated historical network failures.
|
|
38
|
+
if (job.infrastructureFailureCount) {
|
|
39
|
+
job.infrastructureFailureCount = 0
|
|
40
|
+
job.lastInfrastructureFailure = ""
|
|
41
|
+
job.lastInfrastructureError = ""
|
|
42
|
+
job.lastInfrastructureFailureAt = 0
|
|
43
|
+
}
|
|
44
|
+
|
|
30
45
|
if (job.verifyCommand) {
|
|
31
46
|
const verify = await runShellCommand(job.verifyCommand, directory, job.timeoutMs || 300_000)
|
|
32
47
|
job.lastVerifyAt = now()
|
|
@@ -86,6 +101,19 @@ export function createRunFinalizationRuntime(options = {}) {
|
|
|
86
101
|
job = await applyGoalNoProgressGuard(directory, client, sessionID, job, previousJob)
|
|
87
102
|
}
|
|
88
103
|
|
|
104
|
+
const terminal = await applyTerminalContinuationGuard(directory, client, sessionID, job)
|
|
105
|
+
job = terminal.job
|
|
106
|
+
if (terminal.pausedNow) {
|
|
107
|
+
await appendLoopLog(directory, "terminal-no-work", {
|
|
108
|
+
sessionID,
|
|
109
|
+
job: job.name || job.id,
|
|
110
|
+
count: job.terminalNoWorkCount,
|
|
111
|
+
summary: String(terminal.text || "").slice(0, 1000),
|
|
112
|
+
})
|
|
113
|
+
await notifyJob(directory, job, "terminal_no_work")
|
|
114
|
+
await toast(client, "Loop paused: completion-bounded task reported complete with no work remaining twice.", "success")
|
|
115
|
+
}
|
|
116
|
+
|
|
89
117
|
state.jobs = (state.jobs || [])
|
|
90
118
|
.map((candidate) => candidate.id === job.id ? job : candidate)
|
|
91
119
|
.filter((candidate) => candidate.enabled !== false || isGoalJob(candidate))
|
|
@@ -78,7 +78,9 @@ export function createSessionStatusRuntime(options = {}) {
|
|
|
78
78
|
const argsList = []
|
|
79
79
|
if (directory) argsList.push({ query: { directory } }, { directory }, { workspace: directory })
|
|
80
80
|
argsList.push({})
|
|
81
|
+
let attempted = false
|
|
81
82
|
for (const args of argsList) {
|
|
83
|
+
attempted = true
|
|
82
84
|
try {
|
|
83
85
|
const result = await statusMethod.call(client.session, args)
|
|
84
86
|
const error = sdkError(result)
|
|
@@ -91,8 +93,6 @@ export function createSessionStatusRuntime(options = {}) {
|
|
|
91
93
|
if (typeof observedType !== "string") continue
|
|
92
94
|
markSessionStatus(observedSessionID, observedType, observedAt)
|
|
93
95
|
}
|
|
94
|
-
// OpenCode's status list contains active sessions; idle sessions are
|
|
95
|
-
// normally omitted. Clear a completed descendant that was previously busy.
|
|
96
96
|
for (const childID of sessionParents.keys()) {
|
|
97
97
|
if (!isDescendantSession(childID, sessionID) || data[childID]) continue
|
|
98
98
|
markSessionStatus(childID, "idle", observedAt)
|
|
@@ -104,7 +104,7 @@ export function createSessionStatusRuntime(options = {}) {
|
|
|
104
104
|
return { type: "idle", source: "sdk" }
|
|
105
105
|
} catch {}
|
|
106
106
|
}
|
|
107
|
-
return undefined
|
|
107
|
+
return attempted ? { type: "unknown", source: "sdk-error" } : undefined
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
async function canFinalizeActiveRun(directory, client, sessionID, active, options = {}) {
|
|
@@ -117,24 +117,29 @@ export function createSessionStatusRuntime(options = {}) {
|
|
|
117
117
|
if (completion === "completed") return true
|
|
118
118
|
if (!options.requireIdle) return completion === "unknown" && staleActiveRun(sessionID)
|
|
119
119
|
|
|
120
|
+
const cached = sessionStatuses.get(sessionID)
|
|
121
|
+
const seenAt = sessionStatusSeenAt.get(sessionID) || 0
|
|
122
|
+
const cachedIdleAfterRun = cached === "idle" && seenAt > (active.startedAt || 0)
|
|
120
123
|
const live = await readLiveSessionStatus(client, sessionID, directory)
|
|
121
124
|
if (live?.type === "idle") return true
|
|
125
|
+
if (live?.type === "unknown" && cachedIdleAfterRun) {
|
|
126
|
+
// A concrete idle observed after this active run is sufficient to finalize
|
|
127
|
+
// that run even if a later status read fails. This does not authorize a
|
|
128
|
+
// new prompt by itself; admission applies its own current status policy.
|
|
129
|
+
return true
|
|
130
|
+
}
|
|
122
131
|
if (live?.type) {
|
|
123
|
-
if (
|
|
132
|
+
if (live.type === "busy" && options.forceStale && completion === "unknown" && staleActiveRun(sessionID)) return true
|
|
124
133
|
return false
|
|
125
134
|
}
|
|
126
135
|
|
|
127
136
|
if (options.forceStale && completion === "unknown" && staleActiveRun(sessionID)) return true
|
|
128
|
-
|
|
129
|
-
const seenAt = sessionStatusSeenAt.get(sessionID) || 0
|
|
130
|
-
return cached === "idle" && seenAt > (active.startedAt || 0)
|
|
137
|
+
return cachedIdleAfterRun
|
|
131
138
|
}
|
|
132
139
|
|
|
133
140
|
async function recoverCompletedTailWithoutActiveRun(directory, client, sessionID, liveType, seenAt) {
|
|
134
141
|
if (activeRuns.has(sessionID)) return false
|
|
135
|
-
if (liveType !== "busy"
|
|
136
|
-
// Never override the first fresh busy observation. Waiting at least one cache
|
|
137
|
-
// window gives the host time to expose a new user/assistant turn if it is real.
|
|
142
|
+
if (liveType !== "busy") return false
|
|
138
143
|
if (!seenAt || now() - seenAt < sessionStatusCacheMs) return false
|
|
139
144
|
const completion = await activeRunCompletionFromMessages(directory, client, sessionID, { startedAt: 0 })
|
|
140
145
|
if (completion !== "completed") return false
|
|
@@ -149,9 +154,6 @@ export function createSessionStatusRuntime(options = {}) {
|
|
|
149
154
|
}
|
|
150
155
|
|
|
151
156
|
async function sessionStatusType(client, sessionID, directory, options = {}) {
|
|
152
|
-
// OpenCode can briefly report an idle session while a long-running tool or
|
|
153
|
-
// subtask is still executing. Tool lifecycle hooks are the more specific
|
|
154
|
-
// signal here, so never enqueue another turn until every active call ends.
|
|
155
157
|
if (hasActiveToolCalls(sessionID) || hasBusyDescendant(sessionID)) {
|
|
156
158
|
markSessionStatus(sessionID, "busy")
|
|
157
159
|
return "busy"
|
|
@@ -159,38 +161,35 @@ export function createSessionStatusRuntime(options = {}) {
|
|
|
159
161
|
|
|
160
162
|
const cached = sessionStatuses.get(sessionID)
|
|
161
163
|
const seenAt = sessionStatusSeenAt.get(sessionID) || 0
|
|
162
|
-
|
|
163
|
-
// Idle is safe to trust until OpenCode tells us otherwise. Busy/retry is only
|
|
164
|
-
// trusted briefly: OpenCode custom commands such as /loop-status create their
|
|
165
|
-
// own short assistant turn, and some TUI builds do not always emit the final
|
|
166
|
-
// idle event after that turn. If we cache busy forever, due loop work can get
|
|
167
|
-
// stuck at "due in every idle" until the user types another command.
|
|
168
164
|
if (cached === "idle") return cached
|
|
169
165
|
if (cached && now() - seenAt < sessionStatusCacheMs) return cached
|
|
170
166
|
|
|
171
167
|
const live = await readLiveSessionStatus(client, sessionID, directory)
|
|
168
|
+
if (live?.type === "unknown") {
|
|
169
|
+
const conservative = cached === "retry" ? "retry" : "busy"
|
|
170
|
+
markSessionStatus(sessionID, conservative)
|
|
171
|
+
return conservative
|
|
172
|
+
}
|
|
172
173
|
if (live?.type) {
|
|
173
|
-
// Some OpenCode 1.15.x/1.18.x TUI builds can leave session.status at busy
|
|
174
|
-
// after a plugin command acknowledgement. If no Loop run exists yet, a
|
|
175
|
-
// completed assistant tail plus an already-stale busy observation is a
|
|
176
|
-
// stronger signal than the unchanged live status. A genuinely running turn
|
|
177
|
-
// has an unfinished assistant tail (or latest user message) and is never
|
|
178
|
-
// force-recovered by this path.
|
|
179
174
|
if (await recoverCompletedTailWithoutActiveRun(directory, client, sessionID, live.type, seenAt)) return "idle"
|
|
180
175
|
|
|
181
|
-
// When a Loop-owned turn exists, use its exact start boundary so an older
|
|
182
|
-
// completed assistant message can never finalize a newer active run.
|
|
183
176
|
if ((live.type === "busy" || live.type === "retry") && options.recoverStaleActive !== false) {
|
|
184
177
|
const active = activeRuns.get(sessionID)
|
|
185
178
|
if (active) {
|
|
186
179
|
const completion = await activeRunCompletionFromMessages(directory, client, sessionID, active)
|
|
187
|
-
if (completion === "completed" || (completion === "unknown" && staleActiveRun(sessionID))) {
|
|
180
|
+
if (completion === "completed" || (live.type === "busy" && completion === "unknown" && staleActiveRun(sessionID))) {
|
|
188
181
|
markSessionStatus(sessionID, "idle")
|
|
189
|
-
|
|
182
|
+
const logDetails = {
|
|
190
183
|
sessionID,
|
|
191
184
|
job: active.job?.name || active.jobId,
|
|
192
185
|
startedAt: active.startedAt,
|
|
193
|
-
|
|
186
|
+
...(completion === "completed" ? {} : { staleStatus: live.type }),
|
|
187
|
+
}
|
|
188
|
+
await appendLoopLog(
|
|
189
|
+
directory,
|
|
190
|
+
completion === "completed" ? "status-message-complete-recovery" : "status-stale-recovery",
|
|
191
|
+
logDetails,
|
|
192
|
+
)
|
|
194
193
|
return "idle"
|
|
195
194
|
}
|
|
196
195
|
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { isCompletionBoundedContinuation, isTerminalNoWorkReply } from "../core/continuation.js"
|
|
2
|
+
import { orderedSessionMessages, readRecentSessionMessages } from "../opencode/host.js"
|
|
3
|
+
|
|
4
|
+
function messageText(message) {
|
|
5
|
+
const parts = Array.isArray(message?.parts) ? message.parts : []
|
|
6
|
+
const fromParts = parts
|
|
7
|
+
.filter((part) => part?.type === "text" && typeof part.text === "string")
|
|
8
|
+
.map((part) => part.text)
|
|
9
|
+
.join("\n")
|
|
10
|
+
.trim()
|
|
11
|
+
if (fromParts) return fromParts
|
|
12
|
+
const info = message?.info || message || {}
|
|
13
|
+
for (const value of [info.text, info.content, info.summary]) {
|
|
14
|
+
if (typeof value === "string" && value.trim()) return value.trim()
|
|
15
|
+
}
|
|
16
|
+
return ""
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function applyTerminalContinuationGuard(directory, client, sessionID, job, options = {}) {
|
|
20
|
+
// `/loop devam et` is intentionally infinite. The guard is only for an
|
|
21
|
+
// explicit completion-bounded request such as "bitene kadar" / "until done".
|
|
22
|
+
if (job?.scheduleMode !== "idle" || !isCompletionBoundedContinuation(job?.action)) {
|
|
23
|
+
return { job, terminal: false, pausedNow: false }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const messages = await readRecentSessionMessages(client, sessionID, directory, options.messageLimit || 8)
|
|
27
|
+
if (!messages) return { job, terminal: false, pausedNow: false }
|
|
28
|
+
const tail = orderedSessionMessages(messages).at(-1)
|
|
29
|
+
const info = tail?.info || tail || {}
|
|
30
|
+
if (info.role !== "assistant") return { job, terminal: false, pausedNow: false }
|
|
31
|
+
const completed = Number(info?.time?.completed || 0)
|
|
32
|
+
const created = Number(info?.time?.created || 0)
|
|
33
|
+
const runStarted = Number(job?.lastRunAt || 0)
|
|
34
|
+
if (!Number.isFinite(completed) || completed <= 0) return { job, terminal: false, pausedNow: false }
|
|
35
|
+
if (runStarted > 0 && completed < runStarted && (!Number.isFinite(created) || created < runStarted)) {
|
|
36
|
+
return { job, terminal: false, pausedNow: false }
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const text = messageText(tail)
|
|
40
|
+
const terminal = isTerminalNoWorkReply(text)
|
|
41
|
+
if (!terminal) {
|
|
42
|
+
if (job.terminalNoWorkCount) job.terminalNoWorkCount = 0
|
|
43
|
+
return { job, terminal: false, pausedNow: false, text }
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
job.terminalNoWorkCount = (job.terminalNoWorkCount || 0) + 1
|
|
47
|
+
job.lastTerminalNoWorkAt = Date.now()
|
|
48
|
+
job.lastTerminalNoWorkSummary = text.slice(0, 1000)
|
|
49
|
+
const threshold = Math.max(2, Number(options.threshold) || 2)
|
|
50
|
+
const pausedNow = job.terminalNoWorkCount >= threshold && !job.paused
|
|
51
|
+
if (pausedNow) {
|
|
52
|
+
job.paused = true
|
|
53
|
+
job.lastFailureReason = "terminal_no_work"
|
|
54
|
+
}
|
|
55
|
+
return { job, terminal: true, pausedNow, text }
|
|
56
|
+
}
|