@bybrawe/opencode-loop 0.5.34 → 0.5.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +1 -1
- package/package.json +3 -3
- package/scripts/empty-turn-recovery-test.mjs +150 -0
- package/scripts/host-adapter-contract-test.mjs +2 -2
- package/scripts/host-goal-coexistence-canary.mjs +419 -0
- package/scripts/network-recovery-test.mjs +218 -0
- package/scripts/session-status-test.mjs +18 -4
- package/src/index.js +422 -33
- package/src/source/core/continuation.js +53 -2
- package/src/source/opencode/host.js +23 -2
- package/src/source/runtime/empty-turn.js +46 -0
- package/src/source/runtime/executor.js +173 -16
- package/src/source/runtime/network-recovery.js +40 -0
- package/src/source/runtime/run-finalization.js +28 -0
- package/src/source/runtime/session-status.js +37 -32
- package/src/source/runtime/terminal-guard.js +56 -0
|
@@ -10,11 +10,62 @@ const CONTINUATION_SHORTHANDS = new Set([
|
|
|
10
10
|
"devam et bakalım",
|
|
11
11
|
])
|
|
12
12
|
|
|
13
|
+
const COMPLETION_BOUNDED_PATTERNS = [
|
|
14
|
+
/\bbitene kadar\b/i,
|
|
15
|
+
/\b(?:tamamen|komple) projeyi bitir\b/i,
|
|
16
|
+
/\bişi bitir\b/i,
|
|
17
|
+
/\buntil (?:it(?:'s| is) )?(?:done|complete|completed|finished)\b/i,
|
|
18
|
+
/\bfinish (?:the )?(?:project|task|work)\b/i,
|
|
19
|
+
/\bkeep going until\b/i,
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
const TERMINAL_COMPLETION_PATTERNS = [
|
|
23
|
+
// JS \b is ASCII-oriented and does not see Turkish dotless ı as a word
|
|
24
|
+
// character. Use an explicit following delimiter for Turkish terminal forms.
|
|
25
|
+
/proje tamamland[ıi](?=\s|[.,;:!—-]|$)/i,
|
|
26
|
+
/\bproject (?:is )?(?:complete|completed|finished|done)\b/i,
|
|
27
|
+
/\b(?:task|work) (?:is )?(?:complete|completed|finished|done)\b/i,
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
const TERMINAL_NO_WORK_PATTERNS = [
|
|
31
|
+
/yap[ıi]lacak (?:başka )?iş yok(?=\s|[.,;:!—-]|$)/i,
|
|
32
|
+
/başka (?:bir )?iş (?:kalmad[ıi]|yok)(?=\s|[.,;:!—-]|$)/i,
|
|
33
|
+
/\bnothing (?:else )?left to do\b/i,
|
|
34
|
+
/\bno (?:more|remaining) work\b/i,
|
|
35
|
+
/\bno known (?:bugs|issues)\b/i,
|
|
36
|
+
/\bzero known (?:bugs|issues)\b/i,
|
|
37
|
+
]
|
|
38
|
+
|
|
39
|
+
const NEXT_WORK_PATTERNS = [
|
|
40
|
+
/\bnext(?: step| task)?\b/i,
|
|
41
|
+
/s[ıi]radaki(?=\s|[.,;:!—-]|$)/i,
|
|
42
|
+
/sonraki(?=\s|[.,;:!—-]|$)/i,
|
|
43
|
+
/kalan (?:iş|işler|todo|adım)(?=\s|[.,;:!—-]|$)/i,
|
|
44
|
+
/\bremaining (?:work|task|todo|step)/i,
|
|
45
|
+
/devam (?:edeceğim|ediyorum|etmek gerek)(?=\s|[.,;:!—-]|$)/i,
|
|
46
|
+
]
|
|
47
|
+
|
|
13
48
|
export function isContinuationShorthand(value) {
|
|
14
49
|
return CONTINUATION_SHORTHANDS.has(String(value || "").trim().toLowerCase().replace(/\s+/g, " "))
|
|
15
50
|
}
|
|
16
51
|
|
|
52
|
+
export function isCompletionBoundedContinuation(value) {
|
|
53
|
+
const text = String(value || "").trim()
|
|
54
|
+
return COMPLETION_BOUNDED_PATTERNS.some((pattern) => pattern.test(text))
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function isTerminalNoWorkReply(value) {
|
|
58
|
+
const text = String(value || "").trim()
|
|
59
|
+
if (!text || NEXT_WORK_PATTERNS.some((pattern) => pattern.test(text))) return false
|
|
60
|
+
const completed = TERMINAL_COMPLETION_PATTERNS.some((pattern) => pattern.test(text))
|
|
61
|
+
const noWork = TERMINAL_NO_WORK_PATTERNS.some((pattern) => pattern.test(text))
|
|
62
|
+
return completed && noWork
|
|
63
|
+
}
|
|
64
|
+
|
|
17
65
|
export function continuationProjectInstruction(value) {
|
|
18
|
-
if (!isContinuationShorthand(value)) return ""
|
|
19
|
-
|
|
66
|
+
if (!isContinuationShorthand(value) && !isCompletionBoundedContinuation(value)) return ""
|
|
67
|
+
const finish = isCompletionBoundedContinuation(value)
|
|
68
|
+
? " If you believe the project is finished, perform a fresh verification pass before declaring completion; report both that the project is complete and that no work remains only when you have concrete current evidence."
|
|
69
|
+
: ""
|
|
70
|
+
return `Treat this as continuation of the current project and conversation, not a fresh task. Inspect the repository state, relevant files, TODO/progress notes, recent changes, and git status as needed to identify the next unfinished step. Continue from existing work, do not redo completed work, and verify meaningful changes when practical.${finish}`
|
|
20
71
|
}
|
|
@@ -54,10 +54,22 @@ export function orderedSessionMessages(messages) {
|
|
|
54
54
|
.map((entry) => entry.message)
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
export function assistantMessageHasMeaningfulActivity(message) {
|
|
58
|
+
const parts = Array.isArray(message?.parts) ? message.parts : []
|
|
59
|
+
for (const part of parts) {
|
|
60
|
+
if (!part || typeof part !== "object") continue
|
|
61
|
+
if (part.type === "text" && typeof part.text === "string" && part.text.trim()) return true
|
|
62
|
+
if (["tool", "file", "patch", "artifact"].includes(String(part.type || ""))) return true
|
|
63
|
+
}
|
|
64
|
+
const info = message?.info || message || {}
|
|
65
|
+
return [info.text, info.content, info.summary].some((value) => typeof value === "string" && value.trim())
|
|
66
|
+
}
|
|
67
|
+
|
|
57
68
|
export async function activeRunCompletionFromMessages(directory, client, sessionID, active) {
|
|
58
69
|
const messages = await readRecentSessionMessages(client, sessionID, directory)
|
|
59
70
|
if (!messages) return "unknown"
|
|
60
|
-
const
|
|
71
|
+
const ordered = orderedSessionMessages(messages)
|
|
72
|
+
const tail = ordered.at(-1)
|
|
61
73
|
const info = tail?.info || tail
|
|
62
74
|
if (!info || info.role !== "assistant") return "incomplete"
|
|
63
75
|
const completed = Number(info?.time?.completed || 0)
|
|
@@ -65,7 +77,16 @@ export async function activeRunCompletionFromMessages(directory, client, session
|
|
|
65
77
|
if (!Number.isFinite(completed) || completed <= 0) return "incomplete"
|
|
66
78
|
const startedAt = Number(active?.startedAt || 0)
|
|
67
79
|
if (startedAt > 0 && completed < startedAt && (!Number.isFinite(created) || created < startedAt)) return "incomplete"
|
|
68
|
-
|
|
80
|
+
|
|
81
|
+
const relevant = ordered.filter((message) => {
|
|
82
|
+
const candidate = message?.info || message || {}
|
|
83
|
+
if (candidate.role !== "assistant") return false
|
|
84
|
+
if (startedAt <= 0) return true
|
|
85
|
+
const candidateCreated = Number(candidate?.time?.created || 0)
|
|
86
|
+
const candidateCompleted = Number(candidate?.time?.completed || 0)
|
|
87
|
+
return candidateCreated >= startedAt || candidateCompleted >= startedAt
|
|
88
|
+
})
|
|
89
|
+
return relevant.some(assistantMessageHasMeaningfulActivity) ? "completed" : "empty"
|
|
69
90
|
}
|
|
70
91
|
|
|
71
92
|
export async function resolveCompactionModel(directory, client, sessionID, preferredModel) {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { actionKind } from "../core/jobs.js"
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_MAX_EMPTY_TURNS = 2
|
|
4
|
+
|
|
5
|
+
export function guardsEmptyAssistantTurn(job) {
|
|
6
|
+
const kind = actionKind(job?.action, job || {})
|
|
7
|
+
return kind === "prompt" || kind === "goal"
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export function emptyTurnLimit(job) {
|
|
11
|
+
const configured = Number(job?.maxEmptyTurns || 0)
|
|
12
|
+
if (Number.isFinite(configured) && configured > 0) return Math.max(1, Math.floor(configured))
|
|
13
|
+
return DEFAULT_MAX_EMPTY_TURNS
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function refundEmptyAssistantTurn(job, active = {}, timestamp = Date.now()) {
|
|
17
|
+
const chargedCount = Number(active?.job?.runCount ?? job?.runCount ?? 0)
|
|
18
|
+
const currentCount = Number(job?.runCount || 0)
|
|
19
|
+
if (chargedCount > 0 && currentCount >= chargedCount) job.runCount = Math.max(0, currentCount - 1)
|
|
20
|
+
|
|
21
|
+
if (Number.isFinite(Number(active?.previousLastRunAt))) job.lastRunAt = Number(active.previousLastRunAt)
|
|
22
|
+
if (active?.disabledByMaxRuns && Number(job?.maxRuns || 0) > 0 && Number(job?.runCount || 0) < Number(job.maxRuns)) {
|
|
23
|
+
job.enabled = true
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
job.emptyTurnCount = Number(job.emptyTurnCount || 0) + 1
|
|
27
|
+
job.lastEmptyTurnAt = Number(timestamp) || Date.now()
|
|
28
|
+
job.lastFailureReason = "empty_turn"
|
|
29
|
+
|
|
30
|
+
const limit = emptyTurnLimit(job)
|
|
31
|
+
const paused = job.emptyTurnCount >= limit
|
|
32
|
+
if (paused) {
|
|
33
|
+
job.paused = true
|
|
34
|
+
delete job.runNowRequestedAt
|
|
35
|
+
} else {
|
|
36
|
+
job.runNowRequestedAt = Math.max(1, Number(timestamp) || Date.now())
|
|
37
|
+
}
|
|
38
|
+
return { job, paused, count: job.emptyTurnCount, limit }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function clearEmptyAssistantTurnStreak(job) {
|
|
42
|
+
if (!job) return job
|
|
43
|
+
job.emptyTurnCount = 0
|
|
44
|
+
if (job.lastFailureReason === "empty_turn") delete job.lastFailureReason
|
|
45
|
+
return job
|
|
46
|
+
}
|
|
@@ -10,9 +10,13 @@ import { createCompactionRuntime } from "./compaction.js"
|
|
|
10
10
|
import { createActionDispatcher } from "./action-dispatch.js"
|
|
11
11
|
import { createRunFinalizationRuntime } from "./run-finalization.js"
|
|
12
12
|
import { createRunAdmissionRuntime } from "./run-admission.js"
|
|
13
|
+
import { isTransientNetworkError, networkRetryDelayMs, refundInfrastructureRun } from "./network-recovery.js"
|
|
14
|
+
import { guardsEmptyAssistantTurn, refundEmptyAssistantTurn, clearEmptyAssistantTurnStreak } from "./empty-turn.js"
|
|
13
15
|
|
|
14
16
|
const DEFAULT_ACTIVE_GUARD_MS = 45_000
|
|
15
17
|
const DEFAULT_BUSY_RETRY_MS = 5_000
|
|
18
|
+
const DEFAULT_PROVIDER_RETRY_WATCHDOG_MS = 2 * 60_000
|
|
19
|
+
const DEFAULT_NETWORK_RETRY_MAX_MS = 60_000
|
|
16
20
|
|
|
17
21
|
function requireFunction(value, label) {
|
|
18
22
|
if (typeof value !== "function") throw new TypeError(`createLoopExecutor requires ${label}`)
|
|
@@ -51,9 +55,16 @@ export function createLoopExecutor(options = {}) {
|
|
|
51
55
|
const busyRetryMs = Number.isFinite(Number(options.busyRetryMs)) && Number(options.busyRetryMs) > 0
|
|
52
56
|
? Number(options.busyRetryMs)
|
|
53
57
|
: DEFAULT_BUSY_RETRY_MS
|
|
58
|
+
const providerRetryWatchdogMs = Number.isFinite(Number(options.providerRetryWatchdogMs)) && Number(options.providerRetryWatchdogMs) > 0
|
|
59
|
+
? Number(options.providerRetryWatchdogMs)
|
|
60
|
+
: DEFAULT_PROVIDER_RETRY_WATCHDOG_MS
|
|
61
|
+
const networkRetryMaxMs = Number.isFinite(Number(options.networkRetryMaxMs)) && Number(options.networkRetryMaxMs) > 0
|
|
62
|
+
? Number(options.networkRetryMaxMs)
|
|
63
|
+
: DEFAULT_NETWORK_RETRY_MAX_MS
|
|
54
64
|
|
|
55
65
|
const activeRuns = new Map()
|
|
56
66
|
const runLocks = new Map()
|
|
67
|
+
const retryRuns = new Map()
|
|
57
68
|
|
|
58
69
|
const statusRuntime = createSessionStatusRuntime({
|
|
59
70
|
activeRuns,
|
|
@@ -67,6 +78,8 @@ export function createLoopExecutor(options = {}) {
|
|
|
67
78
|
updateSessionStatusFromEvent,
|
|
68
79
|
staleActiveRun,
|
|
69
80
|
canFinalizeActiveRun,
|
|
81
|
+
activeRunCompletion,
|
|
82
|
+
sessionStatusType,
|
|
70
83
|
sessionIsIdle,
|
|
71
84
|
markSessionStatus,
|
|
72
85
|
clearSessionStatus,
|
|
@@ -129,6 +142,7 @@ export function createLoopExecutor(options = {}) {
|
|
|
129
142
|
if (active?.timer) clearTimeout(active.timer)
|
|
130
143
|
compactionRuntime.clearForActiveRun(sessionID, active)
|
|
131
144
|
activeRuns.delete(sessionID)
|
|
145
|
+
retryRuns.delete(sessionID)
|
|
132
146
|
}
|
|
133
147
|
|
|
134
148
|
function disposeSession(sessionID) {
|
|
@@ -138,32 +152,116 @@ export function createLoopExecutor(options = {}) {
|
|
|
138
152
|
clearSessionStatus(sessionID)
|
|
139
153
|
}
|
|
140
154
|
|
|
155
|
+
async function persistInfrastructureRefund(directory, sessionID, active, input = {}) {
|
|
156
|
+
const state = await readState(directory, sessionID)
|
|
157
|
+
const job = (state.jobs || []).find((candidate) => candidate.id === active.jobId)
|
|
158
|
+
if (!job) return { job: undefined, delayMs: busyRetryMs }
|
|
159
|
+
refundInfrastructureRun(job, {
|
|
160
|
+
runCount: active.job?.runCount,
|
|
161
|
+
previousLastRunAt: active.previousLastRunAt,
|
|
162
|
+
disabledByMaxRuns: active.disabledByMaxRuns,
|
|
163
|
+
}, {
|
|
164
|
+
reason: input.reason,
|
|
165
|
+
error: input.error,
|
|
166
|
+
now: now(),
|
|
167
|
+
})
|
|
168
|
+
state.jobs = (state.jobs || []).map((candidate) => candidate.id === job.id ? job : candidate)
|
|
169
|
+
await writeState(directory, sessionID, state)
|
|
170
|
+
const delayMs = networkRetryDelayMs(job.infrastructureFailureCount || 1, busyRetryMs, networkRetryMaxMs)
|
|
171
|
+
return { job, delayMs }
|
|
172
|
+
}
|
|
173
|
+
|
|
141
174
|
async function recoverActiveDispatchFailure(directory, client, sessionID, jobId, runToken, error) {
|
|
142
175
|
const active = activeRuns.get(sessionID)
|
|
143
176
|
if (!active || active.jobId !== jobId || active.runToken !== runToken) return false
|
|
144
177
|
|
|
178
|
+
const transient = isTransientNetworkError(error)
|
|
179
|
+
const snapshot = active
|
|
145
180
|
clearActiveRun(sessionID)
|
|
146
181
|
clearSessionStatus(sessionID)
|
|
147
182
|
|
|
148
183
|
const message = errorMessage(error)
|
|
149
|
-
|
|
150
|
-
|
|
184
|
+
let state = await readState(directory, sessionID)
|
|
185
|
+
let job = (state.jobs || []).find((candidate) => candidate.id === jobId)
|
|
186
|
+
let retryDelay = busyRetryMs
|
|
151
187
|
if (job) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
job
|
|
158
|
-
|
|
188
|
+
if (transient) {
|
|
189
|
+
const refunded = await persistInfrastructureRefund(directory, sessionID, snapshot, {
|
|
190
|
+
reason: "dispatch_failed_transient",
|
|
191
|
+
error,
|
|
192
|
+
})
|
|
193
|
+
job = refunded.job
|
|
194
|
+
retryDelay = refunded.delayMs
|
|
195
|
+
state = await readState(directory, sessionID)
|
|
196
|
+
} else {
|
|
197
|
+
job.failureCount = (job.failureCount || 0) + 1
|
|
198
|
+
job.lastFailureReason = "dispatch_failed"
|
|
199
|
+
job.lastDispatchFailure = message.slice(0, 4000)
|
|
200
|
+
job.lastDispatchFailureAt = now()
|
|
201
|
+
if (job.maxFailures > 0 && job.failureCount >= job.maxFailures) {
|
|
202
|
+
job.paused = true
|
|
203
|
+
await notifyJob(directory, job, "dispatch_failed")
|
|
204
|
+
}
|
|
205
|
+
state.jobs = (state.jobs || []).map((candidate) => candidate.id === job.id ? job : candidate)
|
|
206
|
+
await writeState(directory, sessionID, state)
|
|
159
207
|
}
|
|
160
|
-
state.jobs = (state.jobs || []).map((candidate) => candidate.id === job.id ? job : candidate)
|
|
161
|
-
await writeState(directory, sessionID, state)
|
|
162
208
|
}
|
|
163
209
|
|
|
164
|
-
await appendLoopLog(directory, "dispatch-error"
|
|
165
|
-
|
|
166
|
-
|
|
210
|
+
await appendLoopLog(directory, transient ? "network-dispatch-error" : "dispatch-error", {
|
|
211
|
+
sessionID,
|
|
212
|
+
job: job?.name || jobId,
|
|
213
|
+
error: message,
|
|
214
|
+
...(transient ? { retryInMs: retryDelay } : {}),
|
|
215
|
+
})
|
|
216
|
+
if (transient) {
|
|
217
|
+
await toast(client, `Loop network dispatch failed; retrying when the session recovers: ${message}`, "warning")
|
|
218
|
+
} else {
|
|
219
|
+
await toast(client, `Loop dispatch failed${job?.paused ? " and paused" : ""}: ${message}`, job?.paused ? "error" : "warning")
|
|
220
|
+
}
|
|
221
|
+
await scheduleDueWork(directory, client, sessionID, retryDelay)
|
|
222
|
+
return true
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async function recoverStuckProviderRetry(directory, client, sessionID) {
|
|
226
|
+
const active = activeRuns.get(sessionID)
|
|
227
|
+
if (!active || active.compactionOnly) {
|
|
228
|
+
retryRuns.delete(sessionID)
|
|
229
|
+
return false
|
|
230
|
+
}
|
|
231
|
+
const current = retryRuns.get(sessionID)
|
|
232
|
+
if (!current || current.runToken !== active.runToken) {
|
|
233
|
+
retryRuns.set(sessionID, { runToken: active.runToken, since: now() })
|
|
234
|
+
return false
|
|
235
|
+
}
|
|
236
|
+
if (now() - current.since < providerRetryWatchdogMs) return false
|
|
237
|
+
|
|
238
|
+
const snapshot = active
|
|
239
|
+
try {
|
|
240
|
+
if (client?.session?.abort) {
|
|
241
|
+
await fireSdk(
|
|
242
|
+
client,
|
|
243
|
+
"session.abort provider retry watchdog",
|
|
244
|
+
client.session.abort.bind(client.session),
|
|
245
|
+
{ path: { id: sessionID }, body: {} },
|
|
246
|
+
{ path: { sessionID }, body: {} },
|
|
247
|
+
{ sessionID },
|
|
248
|
+
)
|
|
249
|
+
}
|
|
250
|
+
} catch {}
|
|
251
|
+
clearActiveRun(sessionID)
|
|
252
|
+
clearSessionStatus(sessionID)
|
|
253
|
+
const refunded = await persistInfrastructureRefund(directory, sessionID, snapshot, {
|
|
254
|
+
reason: "provider_retry_watchdog",
|
|
255
|
+
error: `OpenCode session.status stayed retry for ${providerRetryWatchdogMs}ms`,
|
|
256
|
+
})
|
|
257
|
+
await appendLoopLog(directory, "provider-retry-recovery", {
|
|
258
|
+
sessionID,
|
|
259
|
+
job: refunded.job?.name || snapshot.jobId,
|
|
260
|
+
retryForMs: now() - current.since,
|
|
261
|
+
retryInMs: refunded.delayMs,
|
|
262
|
+
})
|
|
263
|
+
await toast(client, "Loop provider retry exceeded the watchdog; the Loop-owned turn was released and will retry with backoff.", "warning")
|
|
264
|
+
await scheduleDueWork(directory, client, sessionID, refunded.delayMs)
|
|
167
265
|
return true
|
|
168
266
|
}
|
|
169
267
|
|
|
@@ -171,6 +269,7 @@ export function createLoopExecutor(options = {}) {
|
|
|
171
269
|
const active = activeRuns.get(sessionID)
|
|
172
270
|
if (!active) return
|
|
173
271
|
if (!await canFinalizeActiveRun(directory, client, sessionID, active, finalizeOptions)) return false
|
|
272
|
+
const completion = await activeRunCompletion(directory, client, sessionID, active)
|
|
174
273
|
const recoveredStale = staleActiveRun(sessionID)
|
|
175
274
|
if (active.compactionOnly) {
|
|
176
275
|
const pending = compactionRuntime.getPending(sessionID)
|
|
@@ -191,6 +290,31 @@ export function createLoopExecutor(options = {}) {
|
|
|
191
290
|
let job = (state.jobs || []).find((candidate) => candidate.id === active.jobId)
|
|
192
291
|
if (!job) return
|
|
193
292
|
job.lastFinishedAt = now()
|
|
293
|
+
|
|
294
|
+
if (completion === "empty" && guardsEmptyAssistantTurn(job)) {
|
|
295
|
+
const empty = refundEmptyAssistantTurn(job, active, now())
|
|
296
|
+
state.jobs = (state.jobs || []).map((candidate) => candidate.id === job.id ? job : candidate)
|
|
297
|
+
await writeState(directory, sessionID, state)
|
|
298
|
+
await appendLoopLog(directory, "empty-assistant-turn", {
|
|
299
|
+
sessionID,
|
|
300
|
+
job: job.name || job.id,
|
|
301
|
+
count: empty.count,
|
|
302
|
+
limit: empty.limit,
|
|
303
|
+
paused: empty.paused,
|
|
304
|
+
refunded: true,
|
|
305
|
+
})
|
|
306
|
+
if (empty.paused) {
|
|
307
|
+
await notifyJob(directory, job, "empty_turn")
|
|
308
|
+
await toast(client, "Loop paused after " + empty.count + " consecutive completed assistant turns with no visible output or tool activity. Resume after changing the model/prompt or use /loop-resume.", "warning")
|
|
309
|
+
await scheduleDueWork(directory, client, sessionID)
|
|
310
|
+
} else {
|
|
311
|
+
await toast(client, "Loop received an empty completed assistant turn; the logical run was refunded and will retry once.", "warning")
|
|
312
|
+
await scheduleDueWork(directory, client, sessionID, busyRetryMs)
|
|
313
|
+
}
|
|
314
|
+
return true
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
if (completion === "completed") clearEmptyAssistantTurnStreak(job)
|
|
194
318
|
if (recoveredStale) {
|
|
195
319
|
await appendLoopLog(directory, "active-stale-recovery", {
|
|
196
320
|
sessionID,
|
|
@@ -217,13 +341,22 @@ export function createLoopExecutor(options = {}) {
|
|
|
217
341
|
}
|
|
218
342
|
runLocks.set(sessionID, now())
|
|
219
343
|
let job
|
|
344
|
+
let previousLastRunAt = 0
|
|
345
|
+
let disabledByMaxRuns = false
|
|
220
346
|
try {
|
|
221
347
|
await finalizeActiveRun(directory, client, sessionID, { requireIdle: true, forceStale: true })
|
|
222
|
-
|
|
348
|
+
const statusType = await sessionStatusType(client, sessionID, directory)
|
|
349
|
+
if (statusType !== "idle") {
|
|
350
|
+
if (statusType === "retry") {
|
|
351
|
+
if (await recoverStuckProviderRetry(directory, client, sessionID)) return
|
|
352
|
+
} else {
|
|
353
|
+
retryRuns.delete(sessionID)
|
|
354
|
+
}
|
|
223
355
|
if (runOptions.force) await toast(client, "Loop queued: session is busy; it will run on the next idle check.", "info")
|
|
224
356
|
await reschedule(busyRetryMs)
|
|
225
357
|
return
|
|
226
358
|
}
|
|
359
|
+
retryRuns.delete(sessionID)
|
|
227
360
|
|
|
228
361
|
const active = activeRuns.get(sessionID)
|
|
229
362
|
const activeAge = active ? now() - (active.startedAt || 0) : 0
|
|
@@ -286,9 +419,11 @@ export function createLoopExecutor(options = {}) {
|
|
|
286
419
|
|
|
287
420
|
if (runNowRequested) delete job.runNowRequestedAt
|
|
288
421
|
job.watchTriggered = false
|
|
422
|
+
previousLastRunAt = Number(job.lastRunAt || 0)
|
|
289
423
|
job.lastRunAt = now()
|
|
290
424
|
job.runCount = (job.runCount || 0) + 1
|
|
291
|
-
|
|
425
|
+
disabledByMaxRuns = job.maxRuns > 0 && job.runCount >= job.maxRuns
|
|
426
|
+
if (disabledByMaxRuns) {
|
|
292
427
|
job.enabled = false
|
|
293
428
|
await notifyJob(directory, job, "max_runs_reached")
|
|
294
429
|
}
|
|
@@ -337,6 +472,8 @@ export function createLoopExecutor(options = {}) {
|
|
|
337
472
|
timer,
|
|
338
473
|
runToken,
|
|
339
474
|
compactionAction: result.compaction === true,
|
|
475
|
+
previousLastRunAt,
|
|
476
|
+
disabledByMaxRuns,
|
|
340
477
|
})
|
|
341
478
|
if (result.compaction && compactionRuntime.isCompleted(sessionID, job.id)) {
|
|
342
479
|
await compactionRuntime.finalize(directory, client, sessionID)
|
|
@@ -352,6 +489,24 @@ export function createLoopExecutor(options = {}) {
|
|
|
352
489
|
await reschedule(busyRetryMs)
|
|
353
490
|
} catch (error) {
|
|
354
491
|
clearActiveRun(sessionID)
|
|
492
|
+
if (isTransientNetworkError(error) && job) {
|
|
493
|
+
const stateAfterFailure = await readState(directory, sessionID)
|
|
494
|
+
const persisted = (stateAfterFailure.jobs || []).find((candidate) => candidate.id === job.id)
|
|
495
|
+
if (persisted) {
|
|
496
|
+
refundInfrastructureRun(persisted, {
|
|
497
|
+
runCount: job.runCount,
|
|
498
|
+
previousLastRunAt,
|
|
499
|
+
disabledByMaxRuns,
|
|
500
|
+
}, { reason: "action_dispatch_transient", error, now: now() })
|
|
501
|
+
stateAfterFailure.jobs = (stateAfterFailure.jobs || []).map((candidate) => candidate.id === persisted.id ? persisted : candidate)
|
|
502
|
+
await writeState(directory, sessionID, stateAfterFailure)
|
|
503
|
+
const delayMs = networkRetryDelayMs(persisted.infrastructureFailureCount || 1, busyRetryMs, networkRetryMaxMs)
|
|
504
|
+
await appendLoopLog(directory, "network-action-error", { sessionID, job: persisted.name || persisted.id, error: errorMessage(error), retryInMs: delayMs })
|
|
505
|
+
await toast(client, "Loop action hit a transient network failure; the logical run was refunded and will retry.", "warning")
|
|
506
|
+
await reschedule(delayMs)
|
|
507
|
+
return
|
|
508
|
+
}
|
|
509
|
+
}
|
|
355
510
|
await toast(client, `Loop job failed: ${error instanceof Error ? error.message : String(error)}`, "error")
|
|
356
511
|
await appendLoopLog(directory, "error", {
|
|
357
512
|
sessionID,
|
|
@@ -370,9 +525,11 @@ export function createLoopExecutor(options = {}) {
|
|
|
370
525
|
clearActiveRun,
|
|
371
526
|
disposeSession,
|
|
372
527
|
recoverActiveDispatchFailure,
|
|
528
|
+
recoverStuckProviderRetry,
|
|
373
529
|
finalizeActiveRun,
|
|
374
530
|
fireAction,
|
|
375
531
|
maybeRunDueJobs,
|
|
532
|
+
sessionStatusType,
|
|
376
533
|
sessionIsIdle,
|
|
377
534
|
updateSessionStatusFromEvent,
|
|
378
535
|
markSessionStatus,
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
const TRANSIENT_NETWORK_PATTERNS = [
|
|
2
|
+
/\b(?:408|425|429|500|502|503|504|524)\b/i,
|
|
3
|
+
/rate[\s_-]?limit|too many requests|overloaded|service[\s_-]?unavailable|provider[_ -]?unavailable/i,
|
|
4
|
+
/terminated|fetch failed|failed to fetch|network[\s_-]?error|network connection lost/i,
|
|
5
|
+
/connection (?:error|refused|lost)|socket (?:hang up|connection was closed)|reset before headers/i,
|
|
6
|
+
/\b(?:enotfound|eai_again|econnrefused|econnreset|etimedout|ehostunreach|enetunreach|epipe)\b/i,
|
|
7
|
+
/\b(?:request|response|connection|network|stream|read) (?:timeout|timed out|time out)\b/i,
|
|
8
|
+
/\btimeout(?:error)?\b/i,
|
|
9
|
+
]
|
|
10
|
+
|
|
11
|
+
function errorText(value) {
|
|
12
|
+
if (value instanceof Error) return `${value.name}: ${value.message}`
|
|
13
|
+
if (typeof value === "string") return value
|
|
14
|
+
try { return JSON.stringify(value) } catch { return String(value) }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function isTransientNetworkError(value) {
|
|
18
|
+
const text = errorText(value)
|
|
19
|
+
return TRANSIENT_NETWORK_PATTERNS.some((pattern) => pattern.test(text))
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function networkRetryDelayMs(attempt, baseMs = 5_000, maxMs = 60_000) {
|
|
23
|
+
const safeAttempt = Math.max(1, Math.floor(Number(attempt) || 1))
|
|
24
|
+
const safeBase = Math.max(1, Math.floor(Number(baseMs) || 5_000))
|
|
25
|
+
const safeMax = Math.max(safeBase, Math.floor(Number(maxMs) || 60_000))
|
|
26
|
+
return Math.min(safeMax, safeBase * (2 ** Math.min(8, safeAttempt - 1)))
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function refundInfrastructureRun(job, snapshot = {}, input = {}) {
|
|
30
|
+
const chargedCount = Number(snapshot.runCount ?? job.runCount ?? 0)
|
|
31
|
+
const currentCount = Number(job.runCount || 0)
|
|
32
|
+
if (chargedCount > 0 && currentCount >= chargedCount) job.runCount = Math.max(0, currentCount - 1)
|
|
33
|
+
if (Number.isFinite(Number(snapshot.previousLastRunAt))) job.lastRunAt = Number(snapshot.previousLastRunAt)
|
|
34
|
+
if (snapshot.disabledByMaxRuns && job.maxRuns > 0 && job.runCount < job.maxRuns) job.enabled = true
|
|
35
|
+
job.infrastructureFailureCount = (job.infrastructureFailureCount || 0) + 1
|
|
36
|
+
job.lastInfrastructureFailure = String(input.reason || "transient_network_failure").slice(0, 120)
|
|
37
|
+
job.lastInfrastructureError = errorText(input.error).slice(0, 4000)
|
|
38
|
+
job.lastInfrastructureFailureAt = Number(input.now || Date.now())
|
|
39
|
+
return job
|
|
40
|
+
}
|
|
@@ -5,6 +5,7 @@ import { appendLoopLog as defaultAppendLoopLog, runShellCommand as defaultRunShe
|
|
|
5
5
|
import { toast as defaultToast } from "../opencode/host.js"
|
|
6
6
|
import { writeGoalReport as defaultWriteGoalReport } from "./goal-runtime.js"
|
|
7
7
|
import { dangerousShell as defaultDangerousShell } from "./job-workspace.js"
|
|
8
|
+
import { applyTerminalContinuationGuard as defaultApplyTerminalContinuationGuard } from "./terminal-guard.js"
|
|
8
9
|
|
|
9
10
|
function requireFunction(value, label) {
|
|
10
11
|
if (typeof value !== "function") throw new TypeError(`createRunFinalizationRuntime requires ${label}`)
|
|
@@ -25,8 +26,22 @@ export function createRunFinalizationRuntime(options = {}) {
|
|
|
25
26
|
const toast = typeof options.toast === "function" ? options.toast : defaultToast
|
|
26
27
|
const writeGoalReport = typeof options.writeGoalReport === "function" ? options.writeGoalReport : defaultWriteGoalReport
|
|
27
28
|
const dangerousShell = typeof options.dangerousShell === "function" ? options.dangerousShell : defaultDangerousShell
|
|
29
|
+
const applyTerminalContinuationGuard = typeof options.applyTerminalContinuationGuard === "function"
|
|
30
|
+
? options.applyTerminalContinuationGuard
|
|
31
|
+
: defaultApplyTerminalContinuationGuard
|
|
28
32
|
|
|
29
33
|
async function finalizeJob(directory, client, sessionID, state, job, previousJob) {
|
|
34
|
+
// Infrastructure backoff is consecutive, not lifetime. Reaching normal
|
|
35
|
+
// finalization proves a real Loop-owned turn completed, so the next future
|
|
36
|
+
// outage should restart from the shortest backoff rather than inheriting an
|
|
37
|
+
// old 60s cap from unrelated historical network failures.
|
|
38
|
+
if (job.infrastructureFailureCount) {
|
|
39
|
+
job.infrastructureFailureCount = 0
|
|
40
|
+
job.lastInfrastructureFailure = ""
|
|
41
|
+
job.lastInfrastructureError = ""
|
|
42
|
+
job.lastInfrastructureFailureAt = 0
|
|
43
|
+
}
|
|
44
|
+
|
|
30
45
|
if (job.verifyCommand) {
|
|
31
46
|
const verify = await runShellCommand(job.verifyCommand, directory, job.timeoutMs || 300_000)
|
|
32
47
|
job.lastVerifyAt = now()
|
|
@@ -86,6 +101,19 @@ export function createRunFinalizationRuntime(options = {}) {
|
|
|
86
101
|
job = await applyGoalNoProgressGuard(directory, client, sessionID, job, previousJob)
|
|
87
102
|
}
|
|
88
103
|
|
|
104
|
+
const terminal = await applyTerminalContinuationGuard(directory, client, sessionID, job)
|
|
105
|
+
job = terminal.job
|
|
106
|
+
if (terminal.pausedNow) {
|
|
107
|
+
await appendLoopLog(directory, "terminal-no-work", {
|
|
108
|
+
sessionID,
|
|
109
|
+
job: job.name || job.id,
|
|
110
|
+
count: job.terminalNoWorkCount,
|
|
111
|
+
summary: String(terminal.text || "").slice(0, 1000),
|
|
112
|
+
})
|
|
113
|
+
await notifyJob(directory, job, "terminal_no_work")
|
|
114
|
+
await toast(client, "Loop paused: completion-bounded task reported complete with no work remaining twice.", "success")
|
|
115
|
+
}
|
|
116
|
+
|
|
89
117
|
state.jobs = (state.jobs || [])
|
|
90
118
|
.map((candidate) => candidate.id === job.id ? job : candidate)
|
|
91
119
|
.filter((candidate) => candidate.enabled !== false || isGoalJob(candidate))
|