comfyui-mcp 0.49.2 → 0.49.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/orchestrator/agent-backend.js +8 -0
- package/dist/orchestrator/agent-backend.js.map +1 -1
- package/dist/orchestrator/codex-backend.js +49 -0
- package/dist/orchestrator/codex-backend.js.map +1 -1
- package/dist/orchestrator/grok-backend.js +5 -0
- package/dist/orchestrator/grok-backend.js.map +1 -1
- package/dist/orchestrator/index.js +317 -22
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/ollama-backend.js +11 -2
- package/dist/orchestrator/ollama-backend.js.map +1 -1
- package/dist/orchestrator/panel-agent.js +591 -67
- package/dist/orchestrator/panel-agent.js.map +1 -1
- package/dist/orchestrator/panel-tools.js +518 -65
- package/dist/orchestrator/panel-tools.js.map +1 -1
- package/dist/orchestrator/run-completion-journal.js +914 -0
- package/dist/orchestrator/run-completion-journal.js.map +1 -0
- package/dist/orchestrator/session-store.js +11 -3
- package/dist/orchestrator/session-store.js.map +1 -1
- package/dist/services/asset-reconcile.js +83 -0
- package/dist/services/asset-reconcile.js.map +1 -0
- package/dist/services/asset-registry.js +9 -2
- package/dist/services/asset-registry.js.map +1 -1
- package/dist/services/download-jobs.js +178 -14
- package/dist/services/download-jobs.js.map +1 -1
- package/dist/services/download-progress.js +16 -18
- package/dist/services/download-progress.js.map +1 -1
- package/dist/services/extra-paths.js +61 -9
- package/dist/services/extra-paths.js.map +1 -1
- package/dist/services/hello-retarget.js +165 -0
- package/dist/services/hello-retarget.js.map +1 -0
- package/dist/services/job-history.js +50 -0
- package/dist/services/job-history.js.map +1 -1
- package/dist/services/job-watcher.js +38 -7
- package/dist/services/job-watcher.js.map +1 -1
- package/dist/services/manifest.js +92 -9
- package/dist/services/manifest.js.map +1 -1
- package/dist/services/model-resolver.js +616 -17
- package/dist/services/model-resolver.js.map +1 -1
- package/dist/services/output-dir.js +63 -15
- package/dist/services/output-dir.js.map +1 -1
- package/dist/services/panel-pin-guard.js +6 -62
- package/dist/services/panel-pin-guard.js.map +1 -1
- package/dist/services/ui-bridge.js +111 -14
- package/dist/services/ui-bridge.js.map +1 -1
- package/dist/services/workspace-env.js +179 -3
- package/dist/services/workspace-env.js.map +1 -1
- package/dist/tools/assets.js +34 -2
- package/dist/tools/assets.js.map +1 -1
- package/dist/tools/extra-paths.js +6 -4
- package/dist/tools/extra-paths.js.map +1 -1
- package/dist/tools/model-extras.js +26 -7
- package/dist/tools/model-extras.js.map +1 -1
- package/dist/tools/model-management.js +36 -8
- package/dist/tools/model-management.js.map +1 -1
- package/dist/tools/report-issue.js +12 -2
- package/dist/tools/report-issue.js.map +1 -1
- package/dist/tools/vocabulary.js +21 -0
- package/dist/tools/vocabulary.js.map +1 -1
- package/package.json +1 -1
- package/scripts/gen-tool-docs.ts +138 -4
- package/scripts/tool-doc-examples.ts +794 -0
|
@@ -24,6 +24,34 @@ export { fetchSupportedModels, fetchSupportedCommands };
|
|
|
24
24
|
function msgOf(err) {
|
|
25
25
|
return errorText(err);
|
|
26
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Opening clause naming WHICH run a completion belongs to (#468).
|
|
29
|
+
*
|
|
30
|
+
* The whole point is that the agent must never read a completion as the answer
|
|
31
|
+
* to its own `panel_run` unless the orchestrator PROVED it by exact prompt id.
|
|
32
|
+
* So:
|
|
33
|
+
* • matched → say so, and name the id.
|
|
34
|
+
* • foreign → real run, real id, but not one this session queued. Say
|
|
35
|
+
* UNDETERMINED and forbid treating it as the awaited render.
|
|
36
|
+
* • unidentified → no id at all. Same, plus how to find out for certain.
|
|
37
|
+
* An event with no correlation field (a legacy/simulated frame) gets the old
|
|
38
|
+
* neutral wording — unchanged behavior.
|
|
39
|
+
*/
|
|
40
|
+
function runIdentityPreamble(ev) {
|
|
41
|
+
const pid = typeof ev.prompt_id === "string" && ev.prompt_id.trim() ? ev.prompt_id.trim() : null;
|
|
42
|
+
switch (ev.run_correlation) {
|
|
43
|
+
case "matched":
|
|
44
|
+
return `This is the run YOU queued with panel_run (prompt ${pid}). `;
|
|
45
|
+
case "foreign":
|
|
46
|
+
return (`This run (prompt ${pid}) does NOT match any run you queued with panel_run — its origin is UNDETERMINED. ` +
|
|
47
|
+
`Do NOT treat it as the render you are waiting on; if you are still waiting on your own run, verify it with get_history before acting. `);
|
|
48
|
+
case "unidentified":
|
|
49
|
+
return (`The panel reported NO prompt id for this run, so it CANNOT be correlated to the render you queued — its origin is UNDETERMINED. ` +
|
|
50
|
+
`Do NOT assume it is your run; verify yours with get_history before acting on it. `);
|
|
51
|
+
default:
|
|
52
|
+
return pid ? `(prompt ${pid}) ` : ``;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
27
55
|
/** Idle window for the per-turn freeze watchdog: if a turn that's in flight
|
|
28
56
|
* receives NO events at all for this long, treat it as stalled. Generous (legit
|
|
29
57
|
* tool work is slow but still streams progress) and overridable for tests via
|
|
@@ -77,6 +105,24 @@ export class PanelAgent {
|
|
|
77
105
|
* on a clean turn result (nothing to re-queue) and on stall/rewind (abandoned on
|
|
78
106
|
* purpose). Null whenever no turn is in flight. */
|
|
79
107
|
inFlight = null;
|
|
108
|
+
/** Run-completion journal tokens carried by the turn currently in flight
|
|
109
|
+
* (#468). Acked when that turn's `result` lands; handed back to the journal
|
|
110
|
+
* when the turn is abandoned (stall watchdog) or the agent is stopped. */
|
|
111
|
+
turnEventTokens = [];
|
|
112
|
+
/** The turn marker `turnEventTokens` belongs to (#468). A completion is acked
|
|
113
|
+
* only by the result of the turn that CARRIED it — see the `result` case. */
|
|
114
|
+
turnEventTokensMarker = 0;
|
|
115
|
+
/**
|
|
116
|
+
* Has the backend produced ANY event for the turn currently in flight?
|
|
117
|
+
*
|
|
118
|
+
* This is what "carried" means for #468's bounded replay: the token is
|
|
119
|
+
* attached at DISPATCH, but the backend may not have consumed the yielded turn
|
|
120
|
+
* yet (Claude awaits output-image resolution before submitting it). Aborting
|
|
121
|
+
* in that window — a rewind, or the watchdog on a turn that produced nothing —
|
|
122
|
+
* means the completion never reached the model, so it must NOT count toward
|
|
123
|
+
* the settle bound. One event is proof of receipt. Reset at each dispatch.
|
|
124
|
+
*/
|
|
125
|
+
turnProducedEvents = false;
|
|
80
126
|
/** True while a turn is in flight (working→done). Lets the manager defer a
|
|
81
127
|
* session-restarting option change (effort) until the turn finishes, instead
|
|
82
128
|
* of interrupting and silently dropping the in-flight reply. */
|
|
@@ -98,15 +144,16 @@ export class PanelAgent {
|
|
|
98
144
|
// app-server) would otherwise leave the panel "working" forever. This is an
|
|
99
145
|
// IDLE timer (reset on every received event), NOT a hard turn cap: legit tool
|
|
100
146
|
// work is slow but still streams progress/tool events, so only a TRUE stall
|
|
101
|
-
// (no events at all for the whole window) trips it.
|
|
102
|
-
//
|
|
103
|
-
//
|
|
104
|
-
// interrupt
|
|
105
|
-
//
|
|
106
|
-
// wedged turn settling (#728).
|
|
147
|
+
// (no events at all for the whole window) trips it. A capable provider first
|
|
148
|
+
// receives an explicit harness-stall notice, never a synthetic user rejection.
|
|
149
|
+
// If that recovery is unavailable or remains silent for another window, the
|
|
150
|
+
// guarded interrupt() flow provides the existing bounded gate-release fallback
|
|
151
|
+
// so the next queued batch never runs ahead of a wedged turn settling (#728).
|
|
107
152
|
idleTimer = null;
|
|
108
153
|
/** Guards against a trip firing twice / racing a real result for one turn. */
|
|
109
154
|
idleTripped = false;
|
|
155
|
+
/** A provider-native, non-user-cancel recovery is attempted at most once per turn. */
|
|
156
|
+
stallRecoverySteered = false;
|
|
110
157
|
/** Tool calls started (item/started) but not yet ended (item/completed). A tool
|
|
111
158
|
* in flight is legitimate work even when the app-server sends nothing, so the
|
|
112
159
|
* watchdog defers while this is > 0 — the fix for the #307-review finding that
|
|
@@ -225,11 +272,38 @@ export class PanelAgent {
|
|
|
225
272
|
send(text, opts) {
|
|
226
273
|
if (opts?.title)
|
|
227
274
|
this.title = opts.title;
|
|
228
|
-
this.queue.push({
|
|
275
|
+
this.queue.push({
|
|
276
|
+
text,
|
|
277
|
+
images: opts?.images,
|
|
278
|
+
mid: opts?.mid,
|
|
279
|
+
...(opts?.completionOnly ? { completionOnly: true } : {}),
|
|
280
|
+
...(opts?.eventTokens?.length ? { eventTokens: [...opts.eventTokens] } : {}),
|
|
281
|
+
});
|
|
229
282
|
const wake = this.waiting;
|
|
230
283
|
this.waiting = null;
|
|
231
284
|
wake?.();
|
|
232
285
|
}
|
|
286
|
+
/** Hand a set of run-completion journal tokens back as UNDELIVERED (#468), so
|
|
287
|
+
* the journal re-arms them for replay instead of letting them die with this
|
|
288
|
+
* agent / this abandoned turn. Never throws into the caller's path. */
|
|
289
|
+
/** Capture-and-clear the in-flight turn in one step. Also the read that
|
|
290
|
+
* survives control-flow narrowing (channel() assigns `inFlight` from another
|
|
291
|
+
* function, so a direct read after a `= null` looks like `never` to TS). */
|
|
292
|
+
takeInFlight() {
|
|
293
|
+
const turn = this.inFlight;
|
|
294
|
+
this.inFlight = null;
|
|
295
|
+
return turn;
|
|
296
|
+
}
|
|
297
|
+
releaseEventTokens(tokens, opts = {}) {
|
|
298
|
+
if (!tokens?.length)
|
|
299
|
+
return;
|
|
300
|
+
try {
|
|
301
|
+
this.deps.onEventUndelivered?.(this.tabId, [...tokens], opts);
|
|
302
|
+
}
|
|
303
|
+
catch (err) {
|
|
304
|
+
logger.warn(`[panel-agent ${this.short()}] releasing completion tokens: ${msgOf(err)}`);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
233
307
|
/** Rewind the CONVERSATION: fork the session at `anchor` (an assistant UUID
|
|
234
308
|
* reported via onTurnAnchor) so everything after it is dropped from the agent's
|
|
235
309
|
* memory, then restart. `anchor` null forks to a fresh session. The edited
|
|
@@ -242,12 +316,42 @@ export class PanelAgent {
|
|
|
242
316
|
// A rewind deliberately DROPS everything after the anchor (the edited message
|
|
243
317
|
// arrives separately), so the interrupted turn's text must NOT be re-queued.
|
|
244
318
|
this.inFlight = null;
|
|
319
|
+
// The dropped turn may have been carrying a run completion — that is news,
|
|
320
|
+
// not conversation, so hand it back for replay rather than rewinding it away
|
|
321
|
+
// (#468).
|
|
322
|
+
const rewoundTokens = this.turnEventTokens;
|
|
323
|
+
this.turnEventTokens = [];
|
|
324
|
+
// CARRIED only if the backend actually RECEIVED this turn. A rewind during
|
|
325
|
+
// the pre-submission window (Claude resolving output images) aborts a turn
|
|
326
|
+
// the model never saw, and counting that toward the settle bound could
|
|
327
|
+
// retire a completion nobody read.
|
|
328
|
+
this.releaseEventTokens(rewoundTokens, { carried: this.turnProducedEvents });
|
|
245
329
|
// Break the current stream so start()'s loop re-enters and forks.
|
|
246
330
|
void this.backend.interrupt().catch(() => { });
|
|
247
331
|
const wake = this.waiting;
|
|
248
332
|
this.waiting = null;
|
|
249
333
|
wake?.();
|
|
250
334
|
}
|
|
335
|
+
/**
|
|
336
|
+
* Pull a still-queued INJECTED COMPLETION back off the queue by its journal
|
|
337
|
+
* token (#468). Returns true only if it was found and removed — false once the
|
|
338
|
+
* turn carrying it has started, where the text is already in the model's
|
|
339
|
+
* context and cannot be recalled.
|
|
340
|
+
*
|
|
341
|
+
* Needed because the event's wording is materialized when it is queued: if the
|
|
342
|
+
* journal later has to WEAKEN that completion's correlation (a prompt id
|
|
343
|
+
* reused, a conversation replaced), the already-queued copy would still claim
|
|
344
|
+
* "this is the run YOU queued". Revoking lets the journal re-deliver the
|
|
345
|
+
* downgraded, honest version instead. Only `completionOnly` items are eligible,
|
|
346
|
+
* so no user message is ever removed.
|
|
347
|
+
*/
|
|
348
|
+
revokeEvent(token) {
|
|
349
|
+
const i = this.queue.findIndex((item) => item.completionOnly && item.eventTokens?.includes(token));
|
|
350
|
+
if (i < 0)
|
|
351
|
+
return false;
|
|
352
|
+
this.queue.splice(i, 1);
|
|
353
|
+
return true;
|
|
354
|
+
}
|
|
251
355
|
/** Drop a still-queued message (the user cancelled/edited it before the agent
|
|
252
356
|
* got to it). Returns true if it was found and removed; false if it was
|
|
253
357
|
* already dequeued (the turn started — too late to cancel). */
|
|
@@ -277,7 +381,14 @@ export class PanelAgent {
|
|
|
277
381
|
* reached the agent." Only meaningful when a session is live (the manager only
|
|
278
382
|
* calls this for an existing agent, so we never spawn one just for an event).
|
|
279
383
|
*/
|
|
280
|
-
injectEvent(ev) {
|
|
384
|
+
injectEvent(ev, opts) {
|
|
385
|
+
// A closed agent's queue is never drained again, so accepting an event here
|
|
386
|
+
// would silently swallow it (#468). REFUSE — and deliberately do NOT hand the
|
|
387
|
+
// token back through onEventUndelivered: the caller is the journal's own
|
|
388
|
+
// flush, which keeps a refused entry pending from the `false` return. Calling
|
|
389
|
+
// back would recurse (release → flush → inject → release → …).
|
|
390
|
+
if (this.closed)
|
|
391
|
+
return false;
|
|
281
392
|
let text = null;
|
|
282
393
|
let images;
|
|
283
394
|
if (ev.kind === "executed") {
|
|
@@ -289,6 +400,23 @@ export class PanelAgent {
|
|
|
289
400
|
const note = typeof ev.note === "string" && ev.note.trim() ? ev.note.trim() : null;
|
|
290
401
|
text =
|
|
291
402
|
`[panel event] ` +
|
|
403
|
+
// #468 — a completion that was journaled and re-delivered says so, so the
|
|
404
|
+
// agent reads it as "this landed late", not as a second render.
|
|
405
|
+
(ev.replayed
|
|
406
|
+
? `(RE-DELIVERED — this completion could not be handed to you when it arrived.) `
|
|
407
|
+
: ``) +
|
|
408
|
+
// An eviction dropped older completions for this tab — say so rather than
|
|
409
|
+
// let them disappear (#468). The agent must treat those runs as unknown.
|
|
410
|
+
(typeof ev.dropped_completions === "number" && ev.dropped_completions > 0
|
|
411
|
+
? `⚠️ ${ev.dropped_completions} EARLIER completion(s) for this tab could not be delivered and were dropped — treat the outcome of those runs as UNDETERMINED and check get_history if you were waiting on one. `
|
|
412
|
+
: ``) +
|
|
413
|
+
// An id-less completion whose content matches one already reported. We
|
|
414
|
+
// will NOT swallow it (identical content is not proof of identity, and a
|
|
415
|
+
// swallowed render is a silent loss), so hand the judgement to the agent.
|
|
416
|
+
(ev.possible_repeat
|
|
417
|
+
? `⚠️ POSSIBLE REPEAT: a completion with identical outputs was already reported to you recently, and this one carries no prompt id to tell them apart. It may be the same event re-sent, or a second render that produced identical filenames — do NOT count it twice without checking (get_history). `
|
|
418
|
+
: ``) +
|
|
419
|
+
runIdentityPreamble(ev) +
|
|
292
420
|
(note
|
|
293
421
|
? `${note} `
|
|
294
422
|
: `A run on the user's canvas just finished and produced ${imgs.length} output image(s): ${names}. `) +
|
|
@@ -322,29 +450,42 @@ export class PanelAgent {
|
|
|
322
450
|
// wakes the agent ONCE, not per file.
|
|
323
451
|
const dl = (ev.downloads ?? []).filter((d) => d && d.name);
|
|
324
452
|
if (dl.length === 0)
|
|
325
|
-
return;
|
|
453
|
+
return false;
|
|
326
454
|
const done = dl.filter((d) => d.status === "done").map((d) => d.name);
|
|
327
455
|
const failed = dl.filter((d) => d.status !== "done").map((d) => d.name);
|
|
328
456
|
const parts = [];
|
|
457
|
+
// This event is raised by the TRANSFER, which finishes BEFORE the placement
|
|
458
|
+
// check against the connected ComfyUI does. Saying "finished" here would be a
|
|
459
|
+
// bare success claim during that window (#369) — a model can land in an
|
|
460
|
+
// install the running server never reads — so the wording says only what the
|
|
461
|
+
// event actually proves and points at download_status for the verdict.
|
|
329
462
|
if (done.length)
|
|
330
|
-
parts.push(`
|
|
463
|
+
parts.push(`transfer completed: ${done.join(", ")}`);
|
|
331
464
|
if (failed.length)
|
|
332
465
|
parts.push(`FAILED: ${failed.join(", ")}`);
|
|
333
466
|
const plural = dl.length > 1 ? "these downloads" : "it";
|
|
334
467
|
text =
|
|
335
468
|
`[panel event] Model download ${parts.join("; ")}. ` +
|
|
336
|
-
`
|
|
337
|
-
|
|
469
|
+
`The bytes finished transferring; whether the connected ComfyUI can actually LOAD ` +
|
|
470
|
+
`${plural} is confirmed separately. If you were waiting on ${plural} to continue a task, ` +
|
|
471
|
+
`call download_status FIRST for the verified path and placement verdict${failed.length ? " or the error detail" : ""} — ` +
|
|
472
|
+
`do not tell the user a model is ready until download_status confirms it. ` +
|
|
338
473
|
`Otherwise reply with ONE short sentence acknowledging it and no tool calls.`;
|
|
339
474
|
}
|
|
340
475
|
if (!text)
|
|
341
|
-
return;
|
|
476
|
+
return false;
|
|
342
477
|
this.busy = true;
|
|
343
478
|
this.deps.onTurn?.(this.tabId, "working"); // event triggers a turn — show working
|
|
344
|
-
this.queue.push({
|
|
479
|
+
this.queue.push({
|
|
480
|
+
text,
|
|
481
|
+
images,
|
|
482
|
+
completionOnly: true, // the whole item IS the event — safe to drop wholesale
|
|
483
|
+
...(opts?.eventToken ? { eventTokens: [opts.eventToken] } : {}),
|
|
484
|
+
});
|
|
345
485
|
const wake = this.waiting;
|
|
346
486
|
this.waiting = null;
|
|
347
487
|
wake?.();
|
|
488
|
+
return true;
|
|
348
489
|
}
|
|
349
490
|
/** Push a ComfyUI EXECUTION error into the session with urgency — the "hey,
|
|
350
491
|
* look at me" path. Renders fail ASYNC (minutes after the agent queued them via
|
|
@@ -418,7 +559,9 @@ export class PanelAgent {
|
|
|
418
559
|
}
|
|
419
560
|
/** Remove and return any unsent queued messages — so a session restart can hand
|
|
420
561
|
* them to the replacement agent instead of dropping them. Items keep their
|
|
421
|
-
* panel `mid`, so re-delivery still flips the right bubble on dequeue (seen)
|
|
562
|
+
* panel `mid`, so re-delivery still flips the right bubble on dequeue (seen),
|
|
563
|
+
* AND their run-completion tokens (#468), so a carried-over completion is
|
|
564
|
+
* acked by the replacement agent rather than replayed as a duplicate. */
|
|
422
565
|
takePending() {
|
|
423
566
|
const items = this.queue;
|
|
424
567
|
this.queue = [];
|
|
@@ -453,10 +596,27 @@ export class PanelAgent {
|
|
|
453
596
|
// the user wants BOTH the interrupted message and the new one answered. A plain
|
|
454
597
|
// Stop / Ctrl+C / Esc (requeueInFlight=false) must NOT re-queue, or it would
|
|
455
598
|
// silently re-run the turn the user just stopped (double tool actions).
|
|
599
|
+
// #468 — the interrupted turn's run-completion tokens travel with its text.
|
|
600
|
+
// Re-queued: they ride the re-queued item and are acked when THAT turn ends.
|
|
601
|
+
// Dropped (plain Stop): hand them back so the completion is replayed rather
|
|
602
|
+
// than dying with the turn the user cancelled.
|
|
603
|
+
const interruptedTokens = this.turnEventTokens;
|
|
604
|
+
this.turnEventTokens = [];
|
|
456
605
|
if (interrupted && opts.requeueInFlight) {
|
|
457
606
|
// Front of the queue: the interrupted work is addressed before whatever the
|
|
458
|
-
// user sends next (which is appended after this interrupt is handled).
|
|
459
|
-
|
|
607
|
+
// user sends next (which is appended after this interrupt is handled). The
|
|
608
|
+
// ORIGINAL items go back (not one merged item) — the next splice re-joins
|
|
609
|
+
// them into the same text, while an injected completion stays a separate
|
|
610
|
+
// `completionOnly` item that a later detach can remove cleanly (#468).
|
|
611
|
+
this.queue.unshift(...interrupted.items);
|
|
612
|
+
}
|
|
613
|
+
else {
|
|
614
|
+
// UNCARRIED on purpose. This is a plain Stop — a deliberate human act that
|
|
615
|
+
// does not repeat on its own, so it cannot form the automatic loop the
|
|
616
|
+
// bound exists to break; settling a completion on the user's third Stop
|
|
617
|
+
// would be a surprise, not a safeguard. (The stall watchdog and rewind DO
|
|
618
|
+
// auto-repeat, so those are carried.)
|
|
619
|
+
this.releaseEventTokens(interruptedTokens);
|
|
460
620
|
}
|
|
461
621
|
// Track the turn this interrupt is aborting (the one holding the gate). The
|
|
462
622
|
// fallback only force-releases while THIS turn is still the one parked on the
|
|
@@ -532,6 +692,16 @@ export class PanelAgent {
|
|
|
532
692
|
async stop() {
|
|
533
693
|
this.closed = true;
|
|
534
694
|
this.inFlight = null; // teardown must not leave a turn that could be re-queued
|
|
695
|
+
// #468 — every run completion this agent still holds (queued-but-unread, or
|
|
696
|
+
// carried by the turn we're tearing down) goes BACK to the journal. A tab
|
|
697
|
+
// that is genuinely gone has its entries dropped explicitly by the
|
|
698
|
+
// orchestrator (forget); everything else is replayed into the replacement
|
|
699
|
+
// agent. Done before the awaits below so a concurrent spawn can pick them up.
|
|
700
|
+
const orphanedTokens = [...this.queue.flatMap((it) => it.eventTokens ?? []), ...this.turnEventTokens];
|
|
701
|
+
this.turnEventTokens = [];
|
|
702
|
+
for (const item of this.queue)
|
|
703
|
+
delete item.eventTokens;
|
|
704
|
+
this.releaseEventTokens(orphanedTokens);
|
|
535
705
|
this.clearIdleWatchdog(); // don't let a turn watchdog fire after teardown
|
|
536
706
|
const wake = this.waiting;
|
|
537
707
|
this.waiting = null;
|
|
@@ -625,11 +795,35 @@ export class PanelAgent {
|
|
|
625
795
|
return;
|
|
626
796
|
// Remember the in-flight turn's user text so an interrupt mid-reply can
|
|
627
797
|
// re-queue it (send-now must address BOTH the interrupted and new message).
|
|
628
|
-
|
|
798
|
+
// #468 — the run-completion tokens this batch carries. They ride with the
|
|
799
|
+
// in-flight capture so a crash-requeue keeps them attached, and are ACKED
|
|
800
|
+
// only when this turn's `result` lands (proof the agent actually read the
|
|
801
|
+
// completion), not merely because it was spliced off the queue.
|
|
802
|
+
const carriedTokens = batch.flatMap((it) => it.eventTokens ?? []);
|
|
803
|
+
// SAFETY NET: the previous turn ended without acking (no result at all, or
|
|
804
|
+
// only a traceless one the ack gate rejected). Its completions are about to
|
|
805
|
+
// be overwritten here, so hand them back first — the flush re-queues them
|
|
806
|
+
// into a LATER turn rather than letting them vanish at the handover.
|
|
807
|
+
if (this.turnEventTokens.length) {
|
|
808
|
+
const stale = this.turnEventTokens;
|
|
809
|
+
this.turnEventTokens = [];
|
|
810
|
+
// CARRIED only if that turn produced events (proof the backend received
|
|
811
|
+
// it) — the same rule as the result and abandon paths.
|
|
812
|
+
this.releaseEventTokens(stale, { carried: this.turnProducedEvents });
|
|
813
|
+
}
|
|
814
|
+
this.turnEventTokens = carriedTokens;
|
|
815
|
+
this.inFlight = {
|
|
816
|
+
text,
|
|
817
|
+
...(images.length ? { images } : {}),
|
|
818
|
+
...(carriedTokens.length ? { eventTokens: carriedTokens } : {}),
|
|
819
|
+
items: batch,
|
|
820
|
+
};
|
|
629
821
|
this.yieldedTurns += 1; // this batch is turn N
|
|
630
822
|
// Mirror the backend's turn-marker mint: the Nth turn yielded here is the
|
|
631
823
|
// Nth turn the backend reads — its events carry marker N (#728 r3).
|
|
632
824
|
this.currentTurnMarker += 1;
|
|
825
|
+
this.turnEventTokensMarker = this.currentTurnMarker;
|
|
826
|
+
this.turnProducedEvents = false; // no proof of receipt yet (#468)
|
|
633
827
|
// Mark the turn in flight AT DISPATCH (not on the first event). Without this
|
|
634
828
|
// the watchdog's `busy` guard would be false for the exact zero-event freeze
|
|
635
829
|
// it's meant to catch, so onTurnStalled() would no-op. (handleEvent's later
|
|
@@ -705,6 +899,9 @@ export class PanelAgent {
|
|
|
705
899
|
this.turnWaiter = null;
|
|
706
900
|
this.currentTurnMarker = 0;
|
|
707
901
|
this.abandonedTurnMarker = 0;
|
|
902
|
+
// The #468 ack gate mirrors the marker reset: no marker minted by the dead
|
|
903
|
+
// run can satisfy the new one's first turn.
|
|
904
|
+
this.turnEventTokensMarker = 0;
|
|
708
905
|
// Drop the prior session's last assistant UUID so a fork can't report a
|
|
709
906
|
// stale (pre-fork) anchor for the first turn of the new session.
|
|
710
907
|
this.lastAssistantUuid = null;
|
|
@@ -766,10 +963,17 @@ export class PanelAgent {
|
|
|
766
963
|
// in-flight message so the restarted/fresh session actually re-runs it.
|
|
767
964
|
// Idempotent enough: a duplicate render beats a lost request, and the
|
|
768
965
|
// quickRestarts give-up guard still bounds a message that crash-loops.
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
966
|
+
// Read through takeInFlight(): channel() assigns this.inFlight from
|
|
967
|
+
// another function, which control-flow analysis can't see, so a direct
|
|
968
|
+
// read here is narrowed to `null` by the `= null` at the top of the loop.
|
|
969
|
+
const interrupted = this.takeInFlight();
|
|
970
|
+
if (interrupted) {
|
|
971
|
+
// Its run-completion tokens (#468) ride the re-queued items, so they
|
|
972
|
+
// are acked when the RE-RUN turn ends — not left dangling on a turn
|
|
973
|
+
// whose session died. The ORIGINAL items go back, so a completion
|
|
974
|
+
// stays its own `completionOnly` item (never welded into user text).
|
|
975
|
+
this.turnEventTokens = [];
|
|
976
|
+
this.queue.unshift(...interrupted.items);
|
|
773
977
|
logger.warn(`[panel-agent ${this.short()}] crash mid-turn — re-queued the interrupted message so it isn't lost`);
|
|
774
978
|
}
|
|
775
979
|
}
|
|
@@ -780,6 +984,18 @@ export class PanelAgent {
|
|
|
780
984
|
// first turn — gate run-ahead). The gate counters are reset next iteration.
|
|
781
985
|
this.clearIdleWatchdog();
|
|
782
986
|
this.clearInterruptReleaseFallback();
|
|
987
|
+
// A turn that never produced a `result` (the session just ended) never
|
|
988
|
+
// acked its run completions. Hand them back so the restarted session
|
|
989
|
+
// replays them instead of the completion dying with the dead session
|
|
990
|
+
// (#468). No-op after a clean result or the crash re-queue above.
|
|
991
|
+
const strandedTokens = this.turnEventTokens;
|
|
992
|
+
const strandedWereReceived = this.turnProducedEvents;
|
|
993
|
+
this.turnEventTokens = [];
|
|
994
|
+
// Same rule again: a session that DID receive this turn (it emitted marked
|
|
995
|
+
// events) and then died without a terminal result is a bounded replay
|
|
996
|
+
// cycle — a session that kept dropping before receiving it is not, and
|
|
997
|
+
// must never count toward the settle bound.
|
|
998
|
+
this.releaseEventTokens(strandedTokens, { carried: strandedWereReceived });
|
|
783
999
|
if (this.closed)
|
|
784
1000
|
break;
|
|
785
1001
|
// Session ended on its own — bound rapid failure loops so a persistently
|
|
@@ -823,20 +1039,101 @@ export class PanelAgent {
|
|
|
823
1039
|
this.idleTimer = null;
|
|
824
1040
|
}
|
|
825
1041
|
this.idleTripped = false;
|
|
1042
|
+
this.stallRecoverySteered = false;
|
|
826
1043
|
// The turn ended — drop any tool-busy state so a start whose matching end was
|
|
827
1044
|
// never seen (errored/interrupted turn) can't defer the NEXT turn's watchdog.
|
|
828
1045
|
this.openToolCalls = 0;
|
|
829
1046
|
this.toolBusySince = 0;
|
|
830
1047
|
}
|
|
1048
|
+
/** The exact notice injected into capable providers when a harness watchdog
|
|
1049
|
+
* sees a frozen turn. It must never be conflated with a user cancellation. */
|
|
1050
|
+
static STALL_RECOVERY_NOTICE = "[harness stall notice] This turn stalled with no activity and was detected by the harness. " +
|
|
1051
|
+
"The user did NOT reject or cancel this tool call. Do not tell the user that they stopped it. " +
|
|
1052
|
+
"Inspect side effects if the call may have started; otherwise it is safe to retry the same call.";
|
|
1053
|
+
/** Finish the legacy interruption path after no native stall recovery is
|
|
1054
|
+
* available. This retains the existing gate/restart safety for old providers. */
|
|
1055
|
+
finishStalledTurn(alreadyReported) {
|
|
1056
|
+
this.busy = false;
|
|
1057
|
+
// The stalled turn is abandoned — don't re-queue its text (a wedged message
|
|
1058
|
+
// could otherwise loop on every interrupt, and it may already have performed
|
|
1059
|
+
// tool side effects).
|
|
1060
|
+
this.inFlight = null;
|
|
1061
|
+
// …but a run COMPLETION the abandoned turn was carrying is not the agent's
|
|
1062
|
+
// work, it's news the agent still needs (#468). Hand its tokens back so the
|
|
1063
|
+
// journal replays them into the next turn instead of losing them with the
|
|
1064
|
+
// turn we just wrote off.
|
|
1065
|
+
//
|
|
1066
|
+
// THIS is the abandon path. The #587 steer path deliberately does NOT come
|
|
1067
|
+
// here: there the turn stays live and keeps carrying its tokens, so releasing
|
|
1068
|
+
// them would replay a completion the live turn still holds.
|
|
1069
|
+
//
|
|
1070
|
+
// CARRIED only if the backend produced at least one event for this turn —
|
|
1071
|
+
// i.e. it demonstrably received the completion and then went silent. A turn
|
|
1072
|
+
// that produced NOTHING never reached the model, so it must not count toward
|
|
1073
|
+
// the settle bound (that freeze is bounded by the self-restart give-up
|
|
1074
|
+
// machinery instead, which tears the agent down rather than quietly retiring
|
|
1075
|
+
// a completion nobody saw).
|
|
1076
|
+
const stalledTokens = this.turnEventTokens;
|
|
1077
|
+
this.turnEventTokens = [];
|
|
1078
|
+
this.releaseEventTokens(stalledTokens, { carried: this.turnProducedEvents });
|
|
1079
|
+
if (!alreadyReported) {
|
|
1080
|
+
this.deps.onSay(this.tabId, "⚠️ The agent stopped responding (the turn stalled with no activity). I've cleared it — please try again.");
|
|
1081
|
+
}
|
|
1082
|
+
this.deps.onTurn?.(this.tabId, "done");
|
|
1083
|
+
// Do NOT completeTurn() here: the gate must stay held until the turn genuinely
|
|
1084
|
+
// ends. interrupt() arms the bounded release fallback and stops the wedged
|
|
1085
|
+
// backend; the aborted turn's `result` (or that fallback) releases the next
|
|
1086
|
+
// queued batch at the right moment. The self-restart loop in start() recovers
|
|
1087
|
+
// the session.
|
|
1088
|
+
void this.interrupt();
|
|
1089
|
+
}
|
|
1090
|
+
/** Attempt one provider-native recovery before the legacy interrupt path.
|
|
1091
|
+
* A steer is an explicit agent-facing harness notice, not a synthetic user
|
|
1092
|
+
* rejection. Its short re-arm preserves the ordinary bounded fallback if the
|
|
1093
|
+
* provider remains frozen after accepting it. */
|
|
1094
|
+
async recoverStalledTurn(stalledMarker, alreadyReported) {
|
|
1095
|
+
const recover = this.backend.recoverStalledTurn;
|
|
1096
|
+
if (!recover || this.stallRecoverySteered) {
|
|
1097
|
+
this.finishStalledTurn(alreadyReported);
|
|
1098
|
+
return;
|
|
1099
|
+
}
|
|
1100
|
+
this.stallRecoverySteered = true;
|
|
1101
|
+
let steered = false;
|
|
1102
|
+
try {
|
|
1103
|
+
steered = await recover.call(this.backend, PanelAgent.STALL_RECOVERY_NOTICE);
|
|
1104
|
+
}
|
|
1105
|
+
catch (err) {
|
|
1106
|
+
logger.debug(`[panel-agent ${this.short()}] stalled-turn recovery: ${msgOf(err)}`);
|
|
1107
|
+
}
|
|
1108
|
+
// The active turn may have completed while the asynchronous steer request was
|
|
1109
|
+
// in flight. Never resurrect a settled or replacement turn.
|
|
1110
|
+
const stillCurrent = !this.closed &&
|
|
1111
|
+
this.busy &&
|
|
1112
|
+
this.currentTurnMarker === stalledMarker &&
|
|
1113
|
+
this.completedTurns < this.yieldedTurns;
|
|
1114
|
+
if (!steered || !stillCurrent) {
|
|
1115
|
+
if (stillCurrent)
|
|
1116
|
+
this.finishStalledTurn(alreadyReported);
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
// The provider accepted the precise notice. Keep the turn authoritative and
|
|
1120
|
+
// allow a second full idle window; if it remains frozen, the next watchdog
|
|
1121
|
+
// trip falls through to the established bounded interrupt/restart recovery.
|
|
1122
|
+
this.idleTripped = false;
|
|
1123
|
+
this.bumpIdleWatchdog();
|
|
1124
|
+
if (!alreadyReported) {
|
|
1125
|
+
this.deps.onSay(this.tabId, "⚠️ The agent stalled with no activity. I sent it a harness-stall notice — you did NOT cancel it; it can inspect state and retry safely.");
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
831
1128
|
/** The current turn produced NO events for the whole idle window → it's frozen.
|
|
832
|
-
*
|
|
833
|
-
*
|
|
834
|
-
* guarded interrupt() flow so the turn-gate opens
|
|
835
|
-
* ends (the aborted turn's terminal result, or the
|
|
836
|
-
* fallback if none arrives) — not synchronously here,
|
|
837
|
-
* batch run before the wedged turn had settled.
|
|
838
|
-
* interruptRequested suppress the follow-up interrupted
|
|
839
|
-
* line (#728). Idempotent per
|
|
1129
|
+
* First send a capable provider an explicit non-user-cancel notice. If that
|
|
1130
|
+
* is unavailable (or it remains silent through another full window), surface
|
|
1131
|
+
* the legacy clear + use the guarded interrupt() flow so the turn-gate opens
|
|
1132
|
+
* only when the turn GENUINELY ends (the aborted turn's terminal result, or the
|
|
1133
|
+
* bounded interrupt-release fallback if none arrives) — not synchronously here,
|
|
1134
|
+
* which let the next queued batch run before the wedged turn had settled.
|
|
1135
|
+
* errorSurfaced + interruptRequested suppress the follow-up interrupted
|
|
1136
|
+
* result's "turn failed" line (#728). Idempotent per trip via idleTripped. */
|
|
840
1137
|
onTurnStalled() {
|
|
841
1138
|
if (this.closed || this.idleTripped || !this.busy)
|
|
842
1139
|
return;
|
|
@@ -860,23 +1157,18 @@ export class PanelAgent {
|
|
|
860
1157
|
// failure. (interrupt() also sets interruptRequested, so the result case
|
|
861
1158
|
// treats that result as the interrupt landing, not a new failure.)
|
|
862
1159
|
const alreadyReported = this.errorSurfaced;
|
|
863
|
-
|
|
1160
|
+
const recovery = this.stallRecoverySteered || !this.backend.recoverStalledTurn
|
|
1161
|
+
? " via legacy interrupt"
|
|
1162
|
+
: " with provider notice";
|
|
1163
|
+
logger.error(`[panel-agent ${this.short()}] turn stalled — no events for ${Math.round(TURN_IDLE_MS / 1000)}s; recovering${recovery}${alreadyReported ? " (failure already reported)" : " and surfacing error"}`);
|
|
864
1164
|
this.errorSurfaced = true;
|
|
865
|
-
|
|
866
|
-
//
|
|
867
|
-
//
|
|
868
|
-
//
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
}
|
|
873
|
-
this.deps.onTurn?.(this.tabId, "done");
|
|
874
|
-
// Do NOT completeTurn() here: the gate must stay held until the turn genuinely
|
|
875
|
-
// ends. interrupt() arms the bounded release fallback and stops the wedged
|
|
876
|
-
// backend; the aborted turn's `result` (or that fallback) releases the next
|
|
877
|
-
// queued batch at the right moment. The self-restart loop in start() recovers
|
|
878
|
-
// the session.
|
|
879
|
-
void this.interrupt();
|
|
1165
|
+
const stalledMarker = this.currentTurnMarker;
|
|
1166
|
+
// #468 note: the run-completion tokens this turn is carrying are handed back
|
|
1167
|
+
// in finishStalledTurn(), NOT here. #587 introduced a path where the stall is
|
|
1168
|
+
// recovered by STEERING the provider and the turn stays live — releasing the
|
|
1169
|
+
// tokens on that path would replay the completion into a turn that is still
|
|
1170
|
+
// holding it, i.e. a duplicate. Only the genuinely-abandoned path releases.
|
|
1171
|
+
void this.recoverStalledTurn(stalledMarker, alreadyReported);
|
|
880
1172
|
}
|
|
881
1173
|
// Handle a canonical AgentEvent from the backend. This is the provider-agnostic
|
|
882
1174
|
// half of what used to be route(SDKMessage): all the panel orchestration (turn
|
|
@@ -904,6 +1196,21 @@ export class PanelAgent {
|
|
|
904
1196
|
logger.debug(`[panel-agent ${this.short()}] dead-lettered a straggler ${ev.type} from turn ${ev.turn} (current=${this.currentTurnMarker}, abandoned≤${this.abandonedTurnMarker})`);
|
|
905
1197
|
return;
|
|
906
1198
|
}
|
|
1199
|
+
// Proof the backend RECEIVED the in-flight turn (#468). On a backend that
|
|
1200
|
+
// DECLARES turn markers, only an event stamped with THIS turn counts: #728
|
|
1201
|
+
// deliberately lets UNMARKED events past the dead-letter guard for gate
|
|
1202
|
+
// liveness, and Claude emits an unmarked terminal for a result it cannot
|
|
1203
|
+
// match to any submitted turn — a stale one of those, arriving while the new
|
|
1204
|
+
// turn is still pre-submission, would otherwise "prove" a receipt that never
|
|
1205
|
+
// happened and let the settle bound retire a completion nobody saw.
|
|
1206
|
+
if (this.backend.capabilities.turnMarkers === true) {
|
|
1207
|
+
if (typeof ev.turn === "number" && ev.turn === this.currentTurnMarker) {
|
|
1208
|
+
this.turnProducedEvents = true;
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
else {
|
|
1212
|
+
this.turnProducedEvents = true; // nothing to compare against
|
|
1213
|
+
}
|
|
907
1214
|
// Any event means the turn is alive — reset the idle watchdog. The `result`
|
|
908
1215
|
// case below disarms it entirely (turn ended). Placed before the switch so it
|
|
909
1216
|
// covers every event type without per-case bumps.
|
|
@@ -1031,6 +1338,50 @@ export class PanelAgent {
|
|
|
1031
1338
|
// Turn completed → nothing to re-queue on a later interrupt. (A clean
|
|
1032
1339
|
// completion must NOT have its message re-queued.)
|
|
1033
1340
|
this.inFlight = null;
|
|
1341
|
+
// #468 — the turn that CARRIED the run completion(s) has ended, so they
|
|
1342
|
+
// demonstrably reached the model's context. Ack them: the journal drops
|
|
1343
|
+
// them and settles their run tickets. This is the ONLY ack point; every
|
|
1344
|
+
// other exit from a turn hands the tokens back for replay.
|
|
1345
|
+
//
|
|
1346
|
+
// ACK GATE: only THIS turn's own result may ack. On a backend that
|
|
1347
|
+
// DECLARES turn markers, an UNMARKED result is a traceless straggler
|
|
1348
|
+
// (Claude emits one for a result it cannot match to a submitted turn)
|
|
1349
|
+
// that the #728 dead-letter deliberately lets through for gate liveness
|
|
1350
|
+
// — it must not also retire a completion the CURRENT turn is carrying,
|
|
1351
|
+
// or a turn that then stalls would have nothing left to hand back.
|
|
1352
|
+
//
|
|
1353
|
+
// The capability is DECLARED, never inferred from "have I seen a marker
|
|
1354
|
+
// yet": a zero-output turn's straggler arrives BEFORE the replacement
|
|
1355
|
+
// turn stamps anything, so an observation-based gate is unsound in
|
|
1356
|
+
// exactly the window it exists to protect. A backend that declares no
|
|
1357
|
+
// markers keeps the pre-#468 behavior (nothing to compare against).
|
|
1358
|
+
//
|
|
1359
|
+
// Not ackable → hand the tokens BACK right here, so a completion can
|
|
1360
|
+
// never dangle on a turn that has already ended. The journal re-queues
|
|
1361
|
+
// it into a later turn: a duplicate at worst, never a loss.
|
|
1362
|
+
if (this.turnEventTokens.length) {
|
|
1363
|
+
const carried = this.turnEventTokens;
|
|
1364
|
+
this.turnEventTokens = [];
|
|
1365
|
+
const ackable = this.backend.capabilities.turnMarkers !== true ||
|
|
1366
|
+
(typeof ev.turn === "number" && ev.turn === this.turnEventTokensMarker);
|
|
1367
|
+
if (ackable) {
|
|
1368
|
+
try {
|
|
1369
|
+
this.deps.onEventDelivered?.(this.tabId, carried);
|
|
1370
|
+
}
|
|
1371
|
+
catch (err) {
|
|
1372
|
+
logger.warn(`[panel-agent ${this.short()}] acking completion tokens: ${msgOf(err)}`);
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
else {
|
|
1376
|
+
logger.warn(`[panel-agent ${this.short()}] an unmarked result cannot ack turn ${this.turnEventTokensMarker}'s run completion(s) — handing ${carried.length} back for replay (#468)`);
|
|
1377
|
+
// CARRIED only with PROOF the backend received this turn — the same
|
|
1378
|
+
// rule as every other release. An UNMARKED result bypasses the #728
|
|
1379
|
+
// dead-letter gate, so it may be a straggler from an abandoned turn
|
|
1380
|
+
// arriving while this one is still pre-submission; counting it would
|
|
1381
|
+
// let three such stragglers settle a completion nobody ever saw.
|
|
1382
|
+
this.releaseEventTokens(carried, { carried: this.turnProducedEvents });
|
|
1383
|
+
}
|
|
1384
|
+
}
|
|
1034
1385
|
// Turn ended cleanly → disarm the freeze watchdog. (If it already tripped,
|
|
1035
1386
|
// completeTurn() is a capped no-op, so the gate can't double-advance.)
|
|
1036
1387
|
this.clearIdleWatchdog();
|
|
@@ -1175,6 +1526,8 @@ export class PanelAgentManager {
|
|
|
1175
1526
|
onThinking: this.opts.onThinking,
|
|
1176
1527
|
onToolCall: this.opts.onToolCall,
|
|
1177
1528
|
onSeen: this.opts.onSeen,
|
|
1529
|
+
onEventDelivered: this.opts.onEventDelivered,
|
|
1530
|
+
onEventUndelivered: this.opts.onEventUndelivered,
|
|
1178
1531
|
panelServer: this.opts.makePanelServer?.(tabId),
|
|
1179
1532
|
pluginPath: this.opts.pluginPath,
|
|
1180
1533
|
}, backend);
|
|
@@ -1337,8 +1690,19 @@ export class PanelAgentManager {
|
|
|
1337
1690
|
const resume = oldAgent.sessionId ?? undefined;
|
|
1338
1691
|
const pending = oldAgent.takePending();
|
|
1339
1692
|
const fresh = this.spawn(tabId, resume); // new agent owns the tab now
|
|
1340
|
-
for (const item of pending)
|
|
1341
|
-
fresh.send(item.text, {
|
|
1693
|
+
for (const item of pending) {
|
|
1694
|
+
fresh.send(item.text, {
|
|
1695
|
+
images: item.images,
|
|
1696
|
+
mid: item.mid,
|
|
1697
|
+
// #468 — carry the run-completion tokens over so the replacement agent
|
|
1698
|
+
// acks the completion it inherited (rather than the journal replaying it
|
|
1699
|
+
// as a second copy), AND the completionOnly marker with them: an item
|
|
1700
|
+
// that loses it can no longer be removed from held mail later, so its
|
|
1701
|
+
// text would outlive its token.
|
|
1702
|
+
...(item.completionOnly ? { completionOnly: true } : {}),
|
|
1703
|
+
...(item.eventTokens?.length ? { eventTokens: item.eventTokens } : {}),
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1342
1706
|
if (nudge)
|
|
1343
1707
|
fresh.send(nudge);
|
|
1344
1708
|
void oldAgent.stop(); // retire the old one; it's no longer mapped
|
|
@@ -1349,13 +1713,23 @@ export class PanelAgentManager {
|
|
|
1349
1713
|
return this.agents.get(tabId)?.lastStatus ?? null;
|
|
1350
1714
|
}
|
|
1351
1715
|
/** Feed a ComfyUI execution event to an EXISTING agent (no-op if none — we
|
|
1352
|
-
* never spawn an agent just to react to an event). Returns whether
|
|
1353
|
-
|
|
1716
|
+
* never spawn an agent just to react to an event). Returns whether the agent
|
|
1717
|
+
* TOOK it onto its queue — not that it was read; a run completion carrying an
|
|
1718
|
+
* `eventToken` is only acked when the turn that delivered it ends (#468). */
|
|
1719
|
+
injectEvent(tabId, ev, opts) {
|
|
1354
1720
|
const agent = this.agents.get(tabId);
|
|
1355
1721
|
if (!agent || agent.isStopped)
|
|
1356
1722
|
return false; // best-effort; don't enqueue into a closed agent
|
|
1357
|
-
agent.injectEvent(ev);
|
|
1358
|
-
|
|
1723
|
+
return agent.injectEvent(ev, opts);
|
|
1724
|
+
}
|
|
1725
|
+
/** Pull a still-queued injected completion back off a tab's agent by journal
|
|
1726
|
+
* token (#468), so a weakened correlation can be re-delivered honestly.
|
|
1727
|
+
* False when there is no such agent or the turn carrying it already started. */
|
|
1728
|
+
revokeEvent(tabId, token) {
|
|
1729
|
+
const agent = this.agents.get(tabId);
|
|
1730
|
+
if (!agent || agent.isStopped)
|
|
1731
|
+
return false;
|
|
1732
|
+
return agent.revokeEvent(token);
|
|
1359
1733
|
}
|
|
1360
1734
|
/** Push a ComfyUI execution error to a tab's agent — interrupt the live turn
|
|
1361
1735
|
* and front-queue the error so the agent stops and addresses it. */
|
|
@@ -1378,10 +1752,29 @@ export class PanelAgentManager {
|
|
|
1378
1752
|
const held = this.heldMessages.get(tabId);
|
|
1379
1753
|
if (held?.length) {
|
|
1380
1754
|
this.heldMessages.delete(tabId);
|
|
1381
|
-
for (const item of held)
|
|
1382
|
-
agent.send(item.text, {
|
|
1755
|
+
for (const item of held) {
|
|
1756
|
+
agent.send(item.text, {
|
|
1757
|
+
images: item.images,
|
|
1758
|
+
mid: item.mid,
|
|
1759
|
+
// #468 — preserve the completionOnly marker across EVERY re-delivery.
|
|
1760
|
+
// A second consecutive failed start would otherwise return this item to
|
|
1761
|
+
// held mail unmarked, making its text un-removable by a later detach.
|
|
1762
|
+
...(item.completionOnly ? { completionOnly: true } : {}),
|
|
1763
|
+
...(item.eventTokens?.length ? { eventTokens: item.eventTokens } : {}),
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1383
1766
|
logger.info(`[panel-orchestrator] tab ${tabId.slice(0, 8)} re-delivering ${held.length} message(s) held from the previous failed start`);
|
|
1384
1767
|
}
|
|
1768
|
+
// #468 — a fresh agent is mapped and can take mail: replay any run
|
|
1769
|
+
// completions journaled while this key had no live agent. Fired BEFORE
|
|
1770
|
+
// start() so the replay is queued ahead of the session coming up, and after
|
|
1771
|
+
// held mail so ordering stays chronological.
|
|
1772
|
+
try {
|
|
1773
|
+
this.opts.onAgentReady?.(tabId);
|
|
1774
|
+
}
|
|
1775
|
+
catch (err) {
|
|
1776
|
+
logger.warn(`[panel-orchestrator] tab ${tabId.slice(0, 8)} completion replay: ${msgOf(err)}`);
|
|
1777
|
+
}
|
|
1385
1778
|
// start() now SELF-RESTARTS internally on session-end, so it only settles on
|
|
1386
1779
|
// an intentional stop() or after it gives up (repeated immediate failures),
|
|
1387
1780
|
// or it rejects on a hard start failure. In the give-up / reject cases, drop
|
|
@@ -1416,11 +1809,7 @@ export class PanelAgentManager {
|
|
|
1416
1809
|
// agent's queue — including the one that triggered it. Capture them
|
|
1417
1810
|
// (BEFORE stop(), which closes the agent) for re-delivery by the next
|
|
1418
1811
|
// spawn on this key, so nothing dies silently with the doomed agent.
|
|
1419
|
-
|
|
1420
|
-
if (orphaned.length) {
|
|
1421
|
-
this.heldMessages.set(key, [...(this.heldMessages.get(key) ?? []), ...orphaned]);
|
|
1422
|
-
logger.warn(`[panel-orchestrator] tab ${key.slice(0, 8)} holding ${orphaned.length} undelivered message(s) — re-delivered on the next successful start`);
|
|
1423
|
-
}
|
|
1812
|
+
this.holdOrphanedMail(key, agent.takePending(), "a failed start");
|
|
1424
1813
|
// PER-TAB degradation (issue #250): a hard start failure is almost
|
|
1425
1814
|
// always a tab-local configuration error — an invalid API key (the
|
|
1426
1815
|
// endpoint 401s in prepare()), an unreachable base URL, a missing CLI
|
|
@@ -1443,8 +1832,17 @@ export class PanelAgentManager {
|
|
|
1443
1832
|
void agent.stop().catch(() => { });
|
|
1444
1833
|
}
|
|
1445
1834
|
else if (gaveUp) {
|
|
1446
|
-
// The bounded self-restart loop gave up — the session keeps dropping.
|
|
1447
|
-
//
|
|
1835
|
+
// The bounded self-restart loop gave up — the session keeps dropping.
|
|
1836
|
+
// Same treatment as the hard-start failure above for the agent's QUEUE
|
|
1837
|
+
// (#468): its still-unread messages — a run completion among them — were
|
|
1838
|
+
// dying here. `settle` only unmapped the agent, so a completion sitting
|
|
1839
|
+
// in its queue stayed `handed_off` in the journal, a state deliverPending
|
|
1840
|
+
// skips: no replay, ever. Capture the queue into held mail (its tokens
|
|
1841
|
+
// ride along, so a respawn re-delivers exactly once) and stop() the agent
|
|
1842
|
+
// so anything left over is handed back.
|
|
1843
|
+
this.holdOrphanedMail(key, agent.takePending(), "an agent that gave up");
|
|
1844
|
+
void agent.stop().catch(() => { });
|
|
1845
|
+
// Fatal signal: let the orchestrator self-exit + respawn.
|
|
1448
1846
|
this.opts.onAgentFatal?.(key, "agent session kept dropping (self-restart gave up)");
|
|
1449
1847
|
}
|
|
1450
1848
|
};
|
|
@@ -1656,13 +2054,126 @@ export class PanelAgentManager {
|
|
|
1656
2054
|
}
|
|
1657
2055
|
return { model: this.modelFor(tabId), effort: this.effortFor(tabId), restarted, deferred };
|
|
1658
2056
|
}
|
|
2057
|
+
/**
|
|
2058
|
+
* THE SINGLE TEARDOWN SEAM for unbinding a key's agent (#468). Every path that
|
|
2059
|
+
* stops an agent must go through here, so a future third teardown can't
|
|
2060
|
+
* silently re-open the hole that `retire()` had: it preserved `heldMessages`
|
|
2061
|
+
* without releasing the run-completion tokens parked in them, leaving those
|
|
2062
|
+
* entries `handed_off` — a state `deliverPending()` skips — with no agent left
|
|
2063
|
+
* to consume them and no disclosure.
|
|
2064
|
+
*
|
|
2065
|
+
* Held mail is DETACHED from its completion tokens either way: the message
|
|
2066
|
+
* text keeps whatever preservation semantics the caller wants, while ownership
|
|
2067
|
+
* of the completion returns to the journal, which replays it to whatever agent
|
|
2068
|
+
* serves this panel tab next (a different provider's key included — the
|
|
2069
|
+
* journal is keyed by panel tab).
|
|
2070
|
+
*
|
|
2071
|
+
* `dropHeldMail` distinguishes the two callers: reset() is an explicit fresh
|
|
2072
|
+
* start and discards the mail; retire() preserves it for when the workflow is
|
|
2073
|
+
* reopened. Returns the unbound agent (if any) for the caller to stop.
|
|
2074
|
+
*/
|
|
2075
|
+
unbindAgent(key, opts) {
|
|
2076
|
+
const agent = this.agents.get(key);
|
|
2077
|
+
this.agents.delete(key);
|
|
2078
|
+
this.detachHeldCompletions(key, opts.reason);
|
|
2079
|
+
if (opts.dropHeldMail)
|
|
2080
|
+
this.heldMessages.delete(key);
|
|
2081
|
+
return agent;
|
|
2082
|
+
}
|
|
2083
|
+
/**
|
|
2084
|
+
* Hand a key's HELD-MAIL run completions back to the journal (#468).
|
|
2085
|
+
*
|
|
2086
|
+
* The completion must leave held mail ENTIRELY, not just lose its token.
|
|
2087
|
+
* Stripping the token alone left the event's TEXT sitting in preserved mail:
|
|
2088
|
+
* `retire()` keeps that mail, so switching Claude→Codex handed the journal's
|
|
2089
|
+
* token to Codex (delivered, correctly) and then switching BACK to Claude
|
|
2090
|
+
* re-delivered the retained text as an ordinary user message — no token, no
|
|
2091
|
+
* `possible_repeat` flag, indistinguishable from a real second completion.
|
|
2092
|
+
* Preservation semantics are for the user's own mail; a completion whose token
|
|
2093
|
+
* has moved on is not that.
|
|
2094
|
+
*
|
|
2095
|
+
* Dropping the item is safe precisely because an injected event is always its
|
|
2096
|
+
* own `completionOnly` item — a re-queued turn restores the original items
|
|
2097
|
+
* rather than one merged blob — so no user message is ever removed with it.
|
|
2098
|
+
*
|
|
2099
|
+
* Every release here is UNCARRIED (the default): nobody read these, so they
|
|
2100
|
+
* must not count toward the journal's bounded replay cycle. Only a turn that
|
|
2101
|
+
* ran and ended is `carried`.
|
|
2102
|
+
*/
|
|
2103
|
+
/**
|
|
2104
|
+
* Park a dead agent's unsent mail for the next spawn — but NEVER its injected
|
|
2105
|
+
* completions (#468).
|
|
2106
|
+
*
|
|
2107
|
+
* Held mail is unbounded and the journal's revocation can only reach a LIVE
|
|
2108
|
+
* agent's queue, so a completion parked here is a copy the journal can no
|
|
2109
|
+
* longer count or cap: each failed-start/retry cycle would strand another
|
|
2110
|
+
* capped batch in held mail, and the whole pile eventually drains into one
|
|
2111
|
+
* turn. Completions are revocable by design and the JOURNAL is their record —
|
|
2112
|
+
* so they are handed back instead, and replayed (bounded) into whatever agent
|
|
2113
|
+
* comes next. Only the user's own messages are held.
|
|
2114
|
+
*/
|
|
2115
|
+
holdOrphanedMail(key, orphaned, reason) {
|
|
2116
|
+
if (!orphaned.length)
|
|
2117
|
+
return;
|
|
2118
|
+
const completions = orphaned.filter((it) => it.completionOnly);
|
|
2119
|
+
const mail = orphaned.filter((it) => !it.completionOnly);
|
|
2120
|
+
if (mail.length) {
|
|
2121
|
+
this.heldMessages.set(key, [...(this.heldMessages.get(key) ?? []), ...mail]);
|
|
2122
|
+
logger.warn(`[panel-orchestrator] tab ${key.slice(0, 8)} holding ${mail.length} undelivered message(s) from ${reason} — re-delivered on the next successful start`);
|
|
2123
|
+
}
|
|
2124
|
+
const tokens = completions.flatMap((it) => it.eventTokens ?? []);
|
|
2125
|
+
if (!tokens.length)
|
|
2126
|
+
return;
|
|
2127
|
+
logger.warn(`[panel-orchestrator] tab ${key.slice(0, 8)}: ${tokens.length} run completion(s) were queued on ${reason} — returned to the journal (never held) for bounded replay (#468)`);
|
|
2128
|
+
try {
|
|
2129
|
+
this.opts.onEventUndelivered?.(key, tokens);
|
|
2130
|
+
}
|
|
2131
|
+
catch (err) {
|
|
2132
|
+
logger.warn(`[panel-orchestrator] tab ${key.slice(0, 8)} returning orphaned completions: ${msgOf(err)}`);
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
detachHeldCompletions(key, reason) {
|
|
2136
|
+
const held = this.heldMessages.get(key);
|
|
2137
|
+
if (!held?.length)
|
|
2138
|
+
return;
|
|
2139
|
+
const tokens = [];
|
|
2140
|
+
const keep = [];
|
|
2141
|
+
for (const item of held) {
|
|
2142
|
+
if (!item.eventTokens?.length) {
|
|
2143
|
+
keep.push(item);
|
|
2144
|
+
continue;
|
|
2145
|
+
}
|
|
2146
|
+
tokens.push(...item.eventTokens);
|
|
2147
|
+
delete item.eventTokens; // the journal owns it now — never ack it twice
|
|
2148
|
+
if (item.completionOnly)
|
|
2149
|
+
continue; // …and the event's text goes with it
|
|
2150
|
+
// Defensive: a token on a non-completionOnly item would mean the item
|
|
2151
|
+
// carries user text too, so it must survive — but then its embedded event
|
|
2152
|
+
// text could be re-delivered untracked. Construction prevents this; log
|
|
2153
|
+
// loudly if it ever happens rather than silently allowing a phantom.
|
|
2154
|
+
logger.error(`[panel-orchestrator] tab ${key.slice(0, 8)} ${reason}: held item carried a completion token but is not completionOnly — keeping its text (#468)`);
|
|
2155
|
+
keep.push(item);
|
|
2156
|
+
}
|
|
2157
|
+
if (keep.length !== held.length)
|
|
2158
|
+
this.heldMessages.set(key, keep);
|
|
2159
|
+
if (!tokens.length)
|
|
2160
|
+
return;
|
|
2161
|
+
logger.warn(`[panel-orchestrator] tab ${key.slice(0, 8)} ${reason}: ${tokens.length} run completion(s) were parked in held mail — returned to the journal for replay (#468)`);
|
|
2162
|
+
try {
|
|
2163
|
+
this.opts.onEventUndelivered?.(key, tokens);
|
|
2164
|
+
}
|
|
2165
|
+
catch (err) {
|
|
2166
|
+
logger.warn(`[panel-orchestrator] tab ${key.slice(0, 8)} releasing held completion tokens: ${msgOf(err)}`);
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
1659
2169
|
/** Forget a tab's agent so the next message starts a brand-new session. The
|
|
1660
2170
|
* map mutation is synchronous and the old agent is stopped fire-and-forget,
|
|
1661
2171
|
* so the caller (e.g. resume_session) can set a new pendingResume right after
|
|
1662
2172
|
* without a concurrent send() spawning a non-resumed agent in an await gap. */
|
|
1663
2173
|
reset(tabId) {
|
|
1664
|
-
|
|
1665
|
-
|
|
2174
|
+
// Unbind through the SHARED teardown seam (#468) — it is what guarantees a
|
|
2175
|
+
// run completion parked in held mail is handed back rather than discarded.
|
|
2176
|
+
const agent = this.unbindAgent(tabId, { dropHeldMail: true, reason: "reset" });
|
|
1666
2177
|
this.pendingResume.delete(tabId);
|
|
1667
2178
|
// Forget the durable session too — a NEW chat must start fresh, so the disk
|
|
1668
2179
|
// fallback in send() can't resurrect the conversation the user just cleared.
|
|
@@ -1671,7 +2182,6 @@ export class PanelAgentManager {
|
|
|
1671
2182
|
this.opts.sessionStore?.clear(tabId);
|
|
1672
2183
|
this.pendingEffortRestart.delete(tabId); // a reset supersedes any deferred restart
|
|
1673
2184
|
this.pendingMcpRestart.delete(tabId);
|
|
1674
|
-
this.heldMessages.delete(tabId); // a reset is an explicit fresh start — drop held mail
|
|
1675
2185
|
// Drop this key's picker override so a provider switch (which reset()s the old
|
|
1676
2186
|
// key) can't carry the old provider's model/effort into the new backend's spawn.
|
|
1677
2187
|
this.modelByKey.delete(tabId);
|
|
@@ -1687,12 +2197,18 @@ export class PanelAgentManager {
|
|
|
1687
2197
|
* chat — this preserves sessionStore/pendingResume/held mail, so the retired
|
|
1688
2198
|
* workflow resumes exactly where it left off when reopened, while its now-stopped
|
|
1689
2199
|
* agent can no longer push frames that the bridge migration alias would leak into
|
|
1690
|
-
* the newly-targeted view.
|
|
2200
|
+
* the newly-targeted view.
|
|
2201
|
+
*
|
|
2202
|
+
* Held mail is PRESERVED (that is the point) but its run completions are NOT
|
|
2203
|
+
* (#468): with the agent gone they would sit `handed_off` forever — a state
|
|
2204
|
+
* `deliverPending()` skips — so a provider switch that retires the key would
|
|
2205
|
+
* silently swallow them. The shared teardown detaches and returns them, and it
|
|
2206
|
+
* runs even when no live agent owns the key (a retire with only held mail is
|
|
2207
|
+
* exactly the case that lost them). */
|
|
1691
2208
|
retire(tabId) {
|
|
1692
|
-
const agent = this.
|
|
2209
|
+
const agent = this.unbindAgent(tabId, { dropHeldMail: false, reason: "retire" });
|
|
1693
2210
|
if (!agent)
|
|
1694
2211
|
return;
|
|
1695
|
-
this.agents.delete(tabId);
|
|
1696
2212
|
void agent.stop();
|
|
1697
2213
|
logger.info(`[panel-orchestrator] tab ${tabId.slice(0, 8)} retired (workflow switch on the same socket) — durable session preserved`);
|
|
1698
2214
|
}
|
|
@@ -1702,6 +2218,14 @@ export class PanelAgentManager {
|
|
|
1702
2218
|
async stopAll() {
|
|
1703
2219
|
this.pendingEffortRestart.clear();
|
|
1704
2220
|
this.pendingMcpRestart.clear();
|
|
2221
|
+
// #468 — go through the same detach seam as reset()/retire() for EVERY key
|
|
2222
|
+
// before dropping held mail. The journal is about to be reported on by the
|
|
2223
|
+
// orchestrator's shutdown disclosure, and an entry still marked `handed_off`
|
|
2224
|
+
// because its token died inside discarded held mail would be missed by the
|
|
2225
|
+
// replay AND read as merely "in flight" rather than lost.
|
|
2226
|
+
for (const key of [...this.heldMessages.keys()]) {
|
|
2227
|
+
this.detachHeldCompletions(key, "shutdown");
|
|
2228
|
+
}
|
|
1705
2229
|
this.heldMessages.clear();
|
|
1706
2230
|
await Promise.all([...this.agents.values()].map((a) => a.stop()));
|
|
1707
2231
|
this.agents.clear();
|