@crouton-kit/crouter 0.3.70 → 0.3.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +1 -1
  2. package/dist/build-root.d.ts +12 -4
  3. package/dist/build-root.js +25 -6
  4. package/dist/builtin-memory/00-runtime-base.md +3 -10
  5. package/dist/builtin-memory/04-base-worker.md +18 -0
  6. package/dist/builtin-memory/04-orchestration-kernel.md +1 -1
  7. package/dist/builtin-memory/crouter-development/plugins.md +82 -5
  8. package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +13 -34
  9. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/provider-rotation.test.ts +1115 -1
  10. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +234 -71
  11. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +2 -9
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/lib/subscription-state.ts +2 -516
  13. package/dist/builtin-pi-packages/pi-crtr-extensions/package-lock.json +2 -2
  14. package/dist/builtin-pi-packages/pi-mode-switch/README.md +11 -39
  15. package/dist/builtin-pi-packages/pi-mode-switch/extensions/index.ts +20 -15
  16. package/dist/builtin-pi-packages/pi-mode-switch/package.json +1 -1
  17. package/dist/builtin-views/canvas/tui.mjs +8 -9
  18. package/dist/builtin-views/chat/tui.mjs +12 -12
  19. package/dist/builtin-views/git-pr/tui.mjs +7 -8
  20. package/dist/builtin-views/inbox/tui.mjs +27 -41
  21. package/dist/builtin-views/linkedin/tui.mjs +23 -37
  22. package/dist/builtin-views/prompt-review/tui.mjs +11 -11
  23. package/dist/builtin-views/settings/tui.mjs +11 -11
  24. package/dist/builtin-views/workspace-sidebar/tui.mjs +8 -9
  25. package/dist/clients/attach/__tests__/attach-chrome-remote.test.js +11 -6
  26. package/dist/clients/attach/__tests__/attach-keybindings.test.d.ts +1 -0
  27. package/dist/clients/attach/__tests__/attach-keybindings.test.js +113 -0
  28. package/dist/clients/attach/__tests__/crtr-output-render.test.js +1 -1
  29. package/dist/clients/attach/__tests__/mermaid-render.test.d.ts +1 -0
  30. package/dist/clients/attach/__tests__/mermaid-render.test.js +28 -0
  31. package/dist/clients/attach/__tests__/oauth-dialog-lifecycle.test.d.ts +1 -0
  32. package/dist/clients/attach/__tests__/oauth-dialog-lifecycle.test.js +110 -0
  33. package/dist/clients/attach/attach-cmd.d.ts +9 -1
  34. package/dist/clients/attach/attach-cmd.js +841 -792
  35. package/dist/clients/attach/auth-pickers.d.ts +0 -12
  36. package/dist/clients/attach/auth-pickers.js +64 -15
  37. package/dist/clients/attach/canvas-panels.js +2 -3
  38. package/dist/clients/attach/chat-view.d.ts +7 -8
  39. package/dist/clients/attach/chat-view.js +119 -79
  40. package/dist/clients/attach/edit-diff-render.d.ts +6 -0
  41. package/dist/clients/attach/edit-diff-render.js +263 -0
  42. package/dist/clients/attach/graph-overlay.d.ts +12 -2
  43. package/dist/clients/attach/graph-overlay.js +83 -33
  44. package/dist/clients/attach/input-controller.d.ts +28 -2
  45. package/dist/clients/attach/input-controller.js +38 -5
  46. package/dist/clients/attach/mermaid-render.js +5 -2
  47. package/dist/clients/attach/pickers.d.ts +8 -7
  48. package/dist/clients/attach/pickers.js +11 -17
  49. package/dist/clients/attach/slash-commands.d.ts +9 -0
  50. package/dist/clients/attach/slash-commands.js +127 -7
  51. package/dist/clients/attach/titled-editor.d.ts +12 -1
  52. package/dist/clients/attach/titled-editor.js +103 -8
  53. package/dist/commands/canvas-browse.js +2 -2
  54. package/dist/commands/human/queue.js +3 -4
  55. package/dist/commands/memory/lint.js +39 -5
  56. package/dist/commands/memory/write.js +1 -0
  57. package/dist/commands/node.js +9 -2
  58. package/dist/commands/pkg/plugin-inspect.js +22 -1
  59. package/dist/commands/pkg/plugin-manage.js +31 -9
  60. package/dist/commands/surface-tmux-spread.js +1 -3
  61. package/dist/commands/sys/__tests__/config-keybindings.test.d.ts +1 -0
  62. package/dist/commands/sys/__tests__/config-keybindings.test.js +55 -0
  63. package/dist/commands/sys/__tests__/config-model-ladders.test.d.ts +1 -0
  64. package/dist/commands/sys/__tests__/config-model-ladders.test.js +121 -0
  65. package/dist/commands/sys/__tests__/setup-core.test.js +158 -1
  66. package/dist/commands/sys/config.js +18 -21
  67. package/dist/commands/sys/doctor.js +42 -4
  68. package/dist/commands/sys/setup-core.d.ts +49 -1
  69. package/dist/commands/sys/setup-core.js +161 -4
  70. package/dist/commands/sys/setup.d.ts +88 -0
  71. package/dist/commands/sys/setup.js +943 -169
  72. package/dist/commands/view-pick.d.ts +4 -0
  73. package/dist/commands/view-pick.js +17 -7
  74. package/dist/core/__tests__/base-worker-prompt.test.d.ts +1 -0
  75. package/dist/core/__tests__/base-worker-prompt.test.js +24 -0
  76. package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +232 -1
  77. package/dist/core/__tests__/canvas-inbox-watcher.test.js +34 -9
  78. package/dist/core/__tests__/command-plugins-surfaces.test.d.ts +1 -0
  79. package/dist/core/__tests__/command-plugins-surfaces.test.js +298 -0
  80. package/dist/core/__tests__/command-plugins.test.d.ts +1 -0
  81. package/dist/core/__tests__/command-plugins.test.js +444 -0
  82. package/dist/core/__tests__/fault-classifier.test.js +15 -0
  83. package/dist/core/__tests__/fixtures/fake-engine.d.ts +6 -0
  84. package/dist/core/__tests__/fixtures/fake-engine.js +9 -1
  85. package/dist/core/__tests__/full/broker-dialogs.test.js +7 -2
  86. package/dist/core/__tests__/host-teardown-process-group.test.js +15 -4
  87. package/dist/core/__tests__/preview-registry-sync.test.js +30 -1
  88. package/dist/core/__tests__/scope-crouter-home-fence.test.d.ts +1 -0
  89. package/dist/core/__tests__/scope-crouter-home-fence.test.js +55 -0
  90. package/dist/core/__tests__/stream-watchdog.test.d.ts +1 -0
  91. package/dist/core/__tests__/stream-watchdog.test.js +70 -0
  92. package/dist/core/__tests__/tmux-surface.test.js +72 -0
  93. package/dist/core/canvas/browse/__tests__/model.test.js +23 -5
  94. package/dist/core/canvas/browse/__tests__/render.test.js +1 -0
  95. package/dist/core/canvas/browse/app.d.ts +6 -0
  96. package/dist/core/canvas/browse/app.js +230 -41
  97. package/dist/core/canvas/browse/model.d.ts +10 -6
  98. package/dist/core/canvas/browse/model.js +20 -12
  99. package/dist/core/canvas/browse/pins.d.ts +4 -0
  100. package/dist/core/canvas/browse/pins.js +29 -0
  101. package/dist/core/canvas/browse/render.d.ts +13 -1
  102. package/dist/core/canvas/browse/render.js +72 -15
  103. package/dist/core/canvas/nav-model.js +5 -11
  104. package/dist/core/canvas/paths.d.ts +5 -0
  105. package/dist/core/canvas/paths.js +7 -0
  106. package/dist/core/command-plugins/adapter.d.ts +15 -0
  107. package/dist/core/command-plugins/adapter.js +145 -0
  108. package/dist/core/command-plugins/compose.d.ts +5 -0
  109. package/dist/core/command-plugins/compose.js +56 -0
  110. package/dist/core/command-plugins/discovery.d.ts +104 -0
  111. package/dist/core/command-plugins/discovery.js +565 -0
  112. package/dist/core/config.d.ts +48 -0
  113. package/dist/core/config.js +228 -110
  114. package/dist/core/fault-classifier.js +1 -1
  115. package/dist/core/fs-utils.d.ts +1 -0
  116. package/dist/core/fs-utils.js +15 -1
  117. package/dist/core/keybindings/__tests__/bespoke-consumers.test.d.ts +1 -0
  118. package/dist/core/keybindings/__tests__/bespoke-consumers.test.js +40 -0
  119. package/dist/core/keybindings/__tests__/resolve.test.d.ts +1 -0
  120. package/dist/core/keybindings/__tests__/resolve.test.js +224 -0
  121. package/dist/core/keybindings/catalog.d.ts +14 -0
  122. package/dist/core/keybindings/catalog.js +257 -0
  123. package/dist/core/keybindings/index.d.ts +5 -0
  124. package/dist/core/keybindings/index.js +4 -0
  125. package/dist/core/keybindings/match.d.ts +29 -0
  126. package/dist/core/keybindings/match.js +99 -0
  127. package/dist/core/keybindings/persistence.d.ts +14 -0
  128. package/dist/core/keybindings/persistence.js +31 -0
  129. package/dist/core/keybindings/resolve.d.ts +14 -0
  130. package/dist/core/keybindings/resolve.js +305 -0
  131. package/dist/core/keybindings/types.d.ts +37 -0
  132. package/dist/core/keybindings/types.js +1 -0
  133. package/dist/core/memory-resolver.d.ts +1 -1
  134. package/dist/core/memory-resolver.js +10 -3
  135. package/dist/core/predicate.d.ts +5 -3
  136. package/dist/core/predicate.js +5 -3
  137. package/dist/core/profiles/select.d.ts +6 -0
  138. package/dist/core/profiles/select.js +86 -52
  139. package/dist/core/provider-management.d.ts +12 -0
  140. package/dist/core/provider-management.js +24 -0
  141. package/dist/core/runtime/broker-protocol.d.ts +33 -5
  142. package/dist/core/runtime/broker.js +164 -14
  143. package/dist/core/runtime/launch.d.ts +39 -6
  144. package/dist/core/runtime/launch.js +78 -20
  145. package/dist/core/runtime/naming.js +3 -3
  146. package/dist/core/runtime/pi-cli.d.ts +6 -0
  147. package/dist/core/runtime/pi-cli.js +16 -2
  148. package/dist/core/runtime/pi-vendored.d.ts +8 -0
  149. package/dist/core/runtime/pi-vendored.js +14 -0
  150. package/dist/core/runtime/placement.d.ts +2 -2
  151. package/dist/core/runtime/placement.js +4 -1
  152. package/dist/core/runtime/promote.js +4 -0
  153. package/dist/core/runtime/recap.d.ts +3 -3
  154. package/dist/core/runtime/recap.js +75 -47
  155. package/dist/core/runtime/recycle.js +6 -1
  156. package/dist/core/runtime/reset.js +5 -0
  157. package/dist/core/runtime/session-list-cache.d.ts +33 -0
  158. package/dist/core/runtime/session-list-cache.js +338 -0
  159. package/dist/core/runtime/spawn.js +52 -45
  160. package/dist/core/runtime/stream-watchdog.d.ts +26 -0
  161. package/dist/core/runtime/stream-watchdog.js +75 -0
  162. package/dist/core/runtime/tmux-chrome.d.ts +1 -1
  163. package/dist/core/runtime/tmux-chrome.js +2 -2
  164. package/dist/core/runtime/tmux.d.ts +18 -15
  165. package/dist/core/runtime/tmux.js +217 -110
  166. package/dist/core/scope.js +27 -4
  167. package/dist/core/subscription-state.d.ts +90 -0
  168. package/dist/core/subscription-state.js +762 -0
  169. package/dist/core/tui/__tests__/host-keybindings.test.d.ts +1 -0
  170. package/dist/core/tui/__tests__/host-keybindings.test.js +112 -0
  171. package/dist/core/tui/host.d.ts +30 -1
  172. package/dist/core/tui/host.js +148 -34
  173. package/dist/core/view/contract.d.ts +17 -3
  174. package/dist/daemon/crtrd.js +253 -12
  175. package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +41 -0
  176. package/dist/pi-extensions/__tests__/canvas-stophook-context-nudge.test.d.ts +1 -0
  177. package/dist/pi-extensions/__tests__/canvas-stophook-context-nudge.test.js +121 -0
  178. package/dist/pi-extensions/canvas-inbox-watcher.d.ts +1 -1
  179. package/dist/pi-extensions/canvas-inbox-watcher.js +203 -25
  180. package/dist/pi-extensions/canvas-recap.d.ts +4 -0
  181. package/dist/pi-extensions/canvas-recap.js +73 -46
  182. package/dist/pi-extensions/canvas-stophook.d.ts +11 -0
  183. package/dist/pi-extensions/canvas-stophook.js +33 -4
  184. package/dist/types.d.ts +9 -26
  185. package/dist/types.js +1 -28
  186. package/dist/web-client/assets/{index-DiFuLcp6.js → index--md2ylfi.js} +1 -1
  187. package/dist/web-client/index.html +1 -1
  188. package/dist/web-client/sw.js +1 -1
  189. package/package.json +8 -4
  190. package/dist/pi-extensions/canvas-nav.d.ts +0 -43
  191. package/dist/pi-extensions/canvas-nav.js +0 -640
  192. package/dist/pi-extensions/widget-order-bus.d.ts +0 -6
  193. package/dist/pi-extensions/widget-order-bus.js +0 -34
@@ -139,6 +139,160 @@ function clearStrandedRelaunchState(id) {
139
139
  strandedRelaunchAttempts.delete(id);
140
140
  strandedRelaunchLastAttempt.delete(id);
141
141
  }
142
+ // Idle-release/inbox retry cap (2026-07-10 diagnosis, round 2): the SAME
143
+ // livelock shape as the stranded-relaunch case above, but on the SEPARATE
144
+ // pass-2 path that resumes a dormant idle-released node the instant its inbox
145
+ // gains an unseen entry. That path called reviveNode() unconditionally every
146
+ // poll with no grace, no backoff, and no attempt cap.
147
+ //
148
+ // ROUND 1 keyed the retry counter on row-state residency (pid liveness),
149
+ // cleared the instant the pid-alive branch observed a live engine. That
150
+ // caught a broker that never boots at all — but the RECURRING incident's
151
+ // actual failure is TURN-level, not launch-level: the revive succeeds (broker
152
+ // launches, records a pid — clearing the round-1 counter), the turn then
153
+ // faults against the provider, the engine exits with the inbox CURSOR
154
+ // UNADVANCED, the node re-enters idle/idle-release, pass-2 sees the same
155
+ // unseen entries again, and — because the counter was just cleared by the
156
+ // transiently-alive engine — starts over at attempt 1/6 every single cycle.
157
+ // crtrd.err showed exactly this: hundreds of `attempt 1/6` lines, never
158
+ // `2/6`, interleaved with `fatal-fault <id> (other, pi→provider)`.
159
+ //
160
+ // ROUND 2 fix: key the counter on INBOX-CURSOR PROGRESS, not row-state
161
+ // residency or pid liveness. `readCursor(id)` is recorded as the state's
162
+ // baseline the first time a stuck delivery is observed; a transiently-alive
163
+ // engine does NOT reset it (there is no pid-alive clear for this state
164
+ // anymore — see handleNodeLiveness). Only an ADVANCED cursor on the NEXT
165
+ // observation — proof the wake actually delivered something — resets the
166
+ // counter (via `progressed` below); an unchanged cursor is the SAME stuck
167
+ // delivery and increments/backs off exactly like the stranded-relaunch path.
168
+ const IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS = 6;
169
+ const IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS = 10 * 60_000; // 10 minutes
170
+ const idleReleaseRelaunchState = new Map();
171
+ function idleReleaseRelaunchBackoffMs(attempts) {
172
+ return Math.min(REVIVE_GRACE_MS * 2 ** attempts, IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS);
173
+ }
174
+ /** Cleared ONLY when the node row itself is gone, or once the counter gives
175
+ * up and terminalizes the node — NEVER on a transiently-alive engine (that
176
+ * was round 1's bug) and never on a bare idle-release status flap (the
177
+ * fault-recovery cycle flaps status EVERY episode, so clearing on that would
178
+ * reproduce the exact same livelock one level up). */
179
+ function clearIdleReleaseRelaunchState(id) {
180
+ idleReleaseRelaunchState.delete(id);
181
+ }
182
+ // ---------------------------------------------------------------------------
183
+ // Concurrency cap (2026-07-10 incident): nothing previously bounded how many
184
+ // live brokers the daemon's automatic supervision sweep would launch. Each pi
185
+ // broker is tens-to-hundreds of MB resident and grows with context; on one
186
+ // machine 223 accumulated, exhausted ~19GB of swap, and froze the host — all
187
+ // parked idle in kevent, not doing work, just wedged. Cap the FIVE reviveNode
188
+ // call sites in THIS file (the daemon's automatic sweep) at a sane per-host
189
+ // default; override via CRTR_MAX_LIVE_BROKERS for a beefier host. Deliberately
190
+ // does NOT apply to `node lifecycle revive` / `node focus` — those call
191
+ // reviveNode directly from their own command files, never through this file,
192
+ // so an explicit human ask for a specific node is never blocked by this cap.
193
+ const DEFAULT_MAX_LIVE_BROKERS = 64;
194
+ function maxLiveBrokers() {
195
+ const raw = process.env['CRTR_MAX_LIVE_BROKERS'];
196
+ if (raw === undefined || raw === '')
197
+ return DEFAULT_MAX_LIVE_BROKERS;
198
+ const n = Number(raw);
199
+ return Number.isFinite(n) && n >= 1 ? Math.floor(n) : DEFAULT_MAX_LIVE_BROKERS;
200
+ }
201
+ // Row-census blind spot (2026-07-10 incident, round 2): the cap's `count`
202
+ // originally seeded ONLY from rows whose recorded `pi_pid` reads confirmed-
203
+ // alive. A broker that spawns but never re-records its pid (the exact
204
+ // stranded-relaunch shape REVIVE_GRACE_MS/strandedRelaunch* exist to recover
205
+ // from) is INVISIBLE to that census — the row still shows pid=null or a stale
206
+ // dead pid. So the row census stayed near-zero while ~190 such brokers piled
207
+ // up as real OS processes, every 2s tick believed it had headroom under
208
+ // max=16, and kept launching straight past the cap.
209
+ //
210
+ // Fix: also census the ACTUAL OS processes. Every broker is launched as
211
+ // `<hostExecPath> <.../broker-cli.js> <nodeId>` (host.ts's one launch()
212
+ // call site) — `broker-cli.js` is a stable, distinctive substring of every
213
+ // broker's full command line and nothing else's. One `ps` shell-out per tick
214
+ // (same primitive/cost class as the wedge detector's own `ps -ax` sample
215
+ // just below) counts them directly, with no dependency on any canvas-row
216
+ // state at all. `count` is then max(rowCensus, osCensus) — see superviseTick
217
+ // — so a failed/unavailable probe (null here) can only ever make the daemon
218
+ // MORE permissive than the row census alone, never less; it is never treated
219
+ // as "zero live brokers" and can only widen the effective cap toward the row
220
+ // census, matching the "never undercount, fail toward the row census" brief.
221
+ function countLiveBrokerProcesses() {
222
+ try {
223
+ const r = spawnSync('ps', ['-ax', '-o', 'command='], { encoding: 'utf8', timeout: 2000 });
224
+ if (r.status !== 0 || typeof r.stdout !== 'string')
225
+ return null;
226
+ let count = 0;
227
+ for (const line of r.stdout.split('\n')) {
228
+ if (line.includes('broker-cli.js'))
229
+ count++;
230
+ }
231
+ return count;
232
+ }
233
+ catch {
234
+ return null;
235
+ }
236
+ }
237
+ // Over-cap surfacing (2026-07-10 incident follow-up): the daemon's own launch
238
+ // sites are all capacity-gated, so the OS census EXCEEDING the cap means
239
+ // brokers are leaking or arriving out-of-band (explicit `node lifecycle
240
+ // revive`/`node focus` runs, the ungated fresh_revive clock pass, or a
241
+ // stranded-spawn pile-up). That condition was invisible in crtrd.err unless
242
+ // someone went looking — and the last time it went unnoticed it ended in swap
243
+ // exhaustion and a frozen host. Surface it where the human actually is: a
244
+ // throttled `tmux display-message` on every attached client (tmux is crtr's
245
+ // one guaranteed viewer surface). No tmux server / no clients → silent no-op;
246
+ // the stderr line still records the condition either way.
247
+ const OVER_CAP_WARN_THROTTLE_MS = 5 * 60_000;
248
+ let lastOverCapWarnAt = Number.NEGATIVE_INFINITY;
249
+ function warnOverCapInTmux(count, max) {
250
+ const now = Date.now();
251
+ if (now - lastOverCapWarnAt < OVER_CAP_WARN_THROTTLE_MS)
252
+ return;
253
+ lastOverCapWarnAt = now;
254
+ process.stderr.write(`[crtrd] over-cap: ${count} live brokers exceed max ${max} — surfacing to tmux clients\n`);
255
+ const msg = `⚠ crtrd: ${count} live brokers (cap ${max}) — possible broker leak. ps -ax | grep broker-cli.js`;
256
+ try {
257
+ const clients = spawnSync('tmux', ['list-clients', '-F', '#{client_name}'], {
258
+ encoding: 'utf8',
259
+ timeout: 2000,
260
+ });
261
+ if (clients.status !== 0 || typeof clients.stdout !== 'string')
262
+ return;
263
+ for (const client of clients.stdout.split('\n')) {
264
+ if (client === '')
265
+ continue;
266
+ // -d 10000 holds the message for 10s (tmux ≥3.2); if this tmux predates
267
+ // -d, fall back to the default display-time rather than not warning.
268
+ const r = spawnSync('tmux', ['display-message', '-d', '10000', '-c', client, msg], { encoding: 'utf8', timeout: 2000 });
269
+ if (r.status !== 0) {
270
+ spawnSync('tmux', ['display-message', '-c', client, msg], {
271
+ encoding: 'utf8',
272
+ timeout: 2000,
273
+ });
274
+ }
275
+ }
276
+ }
277
+ catch {
278
+ /* no tmux server — the stderr line above still records the condition */
279
+ }
280
+ }
281
+ // Throttle for the "at capacity" log line: without it, every deferred revive
282
+ // on every 2s poll would spam crtrd.err as badly as the livelock this cap
283
+ // exists to prevent.
284
+ const CAPACITY_LOG_THROTTLE_MS = 30_000;
285
+ let lastCapacityLogAt = Number.NEGATIVE_INFINITY;
286
+ function hasBrokerCapacity(capacity) {
287
+ if (capacity.count < capacity.max)
288
+ return true;
289
+ const now = Date.now();
290
+ if (now - lastCapacityLogAt >= CAPACITY_LOG_THROTTLE_MS) {
291
+ lastCapacityLogAt = now;
292
+ process.stderr.write(`[crtrd] at broker capacity (${capacity.count}/${capacity.max}) — deferring automatic revives to next poll\n`);
293
+ }
294
+ return false;
295
+ }
142
296
  // §H refresh-authority grace: how long a node may sit with intent='refresh', its
143
297
  // turn OVER (busy marker absent) and its engine still ALIVE before the daemon
144
298
  // concludes the refresh stalled and force-kills the engine. The healthy window is
@@ -697,7 +851,7 @@ export function retryResumeMode(meta) {
697
851
  * dead engine's last launch attempt was itself a cycling revive that never
698
852
  * reached session_start (cycle_pending) — then retry it AS a cycle instead
699
853
  * (retryResumeMode). */
700
- async function handleNodeLiveness(row, now, revivedThisTick) {
854
+ async function handleNodeLiveness(row, now, revivedThisTick, capacity) {
701
855
  const id = row.node_id;
702
856
  const pid = row.pi_pid;
703
857
  // The engine is live → nothing pending; clear any boot/grace timer. THEN
@@ -725,6 +879,11 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
725
879
  if (pid != null && recordedPidLiveness(pid, row.pi_pid_identity) === 'alive') {
726
880
  unhealthySince.delete(id);
727
881
  clearStrandedRelaunchState(id); // a live engine means the stranding resolved
882
+ // Deliberately NOT clearing idleReleaseRelaunchState here (2026-07-10,
883
+ // round 2): a TRANSIENTLY-alive engine — one that boots, then faults its
884
+ // turn and dies again with the inbox cursor unadvanced — is exactly the
885
+ // recurrence this state now guards against. Only cursor PROGRESS (checked
886
+ // in pass 2, keyed on readCursor) may reset that counter.
728
887
  handleYieldStall(row, pid, now);
729
888
  handleWedgeDetection(id, pid, now);
730
889
  handleFatalFault(id, now);
@@ -800,10 +959,13 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
800
959
  const backoff = strandedRelaunchBackoffMs(attempts);
801
960
  if (lastAttempt !== undefined && now - lastAttempt < backoff)
802
961
  return; // backing off before the next attempt
962
+ if (!hasBrokerCapacity(capacity))
963
+ return; // deferred, not attempted — costs no retry budget
803
964
  strandedRelaunchAttempts.set(id, attempts + 1);
804
965
  strandedRelaunchLastAttempt.set(id, now);
805
966
  process.stderr.write(`[crtrd] revive ${id} (stranded relaunch — pid never re-recorded, attempt ${attempts + 1}/${STRANDED_RELAUNCH_MAX_ATTEMPTS})\n`);
806
967
  reviveNode(id, { resume: retryResumeMode(meta) });
968
+ capacity.count++;
807
969
  revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
808
970
  return;
809
971
  }
@@ -824,9 +986,13 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
824
986
  yieldTermAt.delete(id);
825
987
  if (row.intent === 'refresh') {
826
988
  // Clean yield — or the §H force-kill above just landed — → respawn FRESH,
827
- // immediately (no grace wait).
989
+ // immediately (no grace wait), UNLESS the daemon is already at its live-
990
+ // broker cap — then defer to the next poll rather than launching over it.
991
+ if (!hasBrokerCapacity(capacity))
992
+ return;
828
993
  process.stderr.write(`[crtrd] revive ${id} (refresh-yield)\n`);
829
994
  reviveNode(id, { resume: false });
995
+ capacity.count++;
830
996
  revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
831
997
  return;
832
998
  }
@@ -883,8 +1049,11 @@ async function handleNodeLiveness(row, now, revivedThisTick) {
883
1049
  }
884
1050
  return;
885
1051
  }
1052
+ if (!hasBrokerCapacity(capacity))
1053
+ return; // deferred — retried plainly next poll
886
1054
  process.stderr.write(`[crtrd] revive ${id} (engine dead, intent=${String(row.intent)})\n`);
887
1055
  reviveNode(id, { resume: retryResumeMode(meta) });
1056
+ capacity.count++;
888
1057
  revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
889
1058
  }
890
1059
  /** Fail loud for a drifted/broken trigger (design §6.6/Q5): wake the ARMER
@@ -1137,6 +1306,24 @@ export async function superviseTick(now = Date.now()) {
1137
1306
  // Node-ids revived in pass 1 + pass 2 THIS tick — the third pass's bare branch
1138
1307
  // skips them so it never launches a second pi on the same .jsonl (Maj-4).
1139
1308
  const revivedThisTick = new Set();
1309
+ // Concurrency-cap budget for this tick (2026-07-10 incident, round 2 — see
1310
+ // countLiveBrokerProcesses's doc for the row-census blind spot this closes).
1311
+ // Seed from max(rowCensus, osCensus): the row census alone undercounts any
1312
+ // broker that spawned but never re-recorded its pid, so it can never be the
1313
+ // sole source of truth for a HARD cap; `?? 0` on a failed/unavailable OS
1314
+ // probe degrades exactly to the row census (never lower — a null probe must
1315
+ // never be read as "zero live brokers"). Threaded by reference through both
1316
+ // automatic-revive passes below so a burst of due revives in one tick can't
1317
+ // collectively launch past the cap.
1318
+ const rowLiveBrokerCount = rows.filter((r) => r.pi_pid != null && recordedPidLiveness(r.pi_pid, r.pi_pid_identity) === 'alive').length;
1319
+ const capacity = {
1320
+ count: Math.max(rowLiveBrokerCount, countLiveBrokerProcesses() ?? 0),
1321
+ max: maxLiveBrokers(),
1322
+ };
1323
+ // Already OVER the cap at tick start — the daemon can't have done this
1324
+ // (every launch site below is gated), so tell the human about the leak.
1325
+ if (capacity.count > capacity.max)
1326
+ warnOverCapInTmux(capacity.count, capacity.max);
1140
1327
  for (const row of rows) {
1141
1328
  try {
1142
1329
  // Every node is broker-hosted (host_kind always 'broker'; pane/window/
@@ -1144,7 +1331,7 @@ export async function superviseTick(now = Date.now()) {
1144
1331
  // pane, if any, is just a viewer — the daemon never consults it and a viewer
1145
1332
  // closing is not a node death. There is no inline-root carve-out and no
1146
1333
  // pane-gone reaping: one liveness path for all.
1147
- await handleNodeLiveness(row, now, revivedThisTick);
1334
+ await handleNodeLiveness(row, now, revivedThisTick, capacity);
1148
1335
  }
1149
1336
  catch (err) {
1150
1337
  // One bad node must never kill the loop.
@@ -1155,13 +1342,32 @@ export async function superviseTick(now = Date.now()) {
1155
1342
  // The in-process inbox-watcher dies with pi, so the daemon owns wake-on-message
1156
1343
  // for dormant nodes. readCursor is the cursor the watcher persisted before
1157
1344
  // exit; any entry past it is undelivered work — resume the node to handle it.
1345
+ //
1346
+ // Backoff + attempt cap, keyed on CURSOR PROGRESS (2026-07-10 incident, round
1347
+ // 2 — see the idleReleaseRelaunchState doc above for the full failure-mode
1348
+ // writeup): a node whose delivery keeps failing — whether the broker never
1349
+ // boots, OR it boots and then faults its turn before advancing the cursor —
1350
+ // used to be relaunched here every poll (round 1) or every fault-recovery
1351
+ // cycle (round 1's fix, still exploitable), forever. The counter now only
1352
+ // resets when readCursor(id) has actually moved since the last time this
1353
+ // node's stuck delivery was considered — proof the wake accomplished
1354
+ // something — never on a bare status flap or a transiently-alive engine.
1355
+ // Capacity-checked too, so a burst of due idle-release wakes can't stampede
1356
+ // past the live-broker cap either.
1158
1357
  for (const row of rows) {
1358
+ const id = row.node_id;
1159
1359
  try {
1160
1360
  // Re-read the ROW for fresh runtime (the first pass may have mutated it);
1161
1361
  // no meta needed — status/intent live in the row.
1162
- const r = getRow(row.node_id);
1163
- if (r === null)
1362
+ const r = getRow(id);
1363
+ if (r === null) {
1364
+ clearIdleReleaseRelaunchState(id); // node gone — nothing left to track
1164
1365
  continue;
1366
+ }
1367
+ // NOT idle-release right now (e.g. mid fault-recovery, briefly active) —
1368
+ // leave idleReleaseRelaunchState UNTOUCHED. Clearing here would let the
1369
+ // fault-recovery cycle (idle-release → briefly alive → fault → idle-
1370
+ // release again) reset the counter every episode — the exact round-2 bug.
1165
1371
  if (r.status !== 'idle' || r.intent !== 'idle-release')
1166
1372
  continue;
1167
1373
  // The in-process inbox-watcher only owns delivery while the engine is LIVE.
@@ -1169,18 +1375,53 @@ export async function superviseTick(now = Date.now()) {
1169
1375
  // has no watcher and the daemon must wake it. Gate the skip on engine
1170
1376
  // liveness alone — viewer panes are irrelevant here. Identity-aware: a
1171
1377
  // bare isPidAlive would misread a reused pid as "still alive" and skip
1172
- // the revive forever, stranding the node on its unseen inbox entry.
1378
+ // the revive forever, stranding the node on its unseen inbox entry. Same
1379
+ // no-clear rule as above — a transiently-alive engine proves nothing about
1380
+ // delivery progress.
1173
1381
  if (r.pi_pid != null && isRecordedPidAlive(r.pi_pid, r.pi_pid_identity))
1174
1382
  continue;
1175
- const entries = readInboxSince(row.node_id, readCursor(row.node_id));
1176
- if (entries.length > 0) {
1177
- process.stderr.write(`[crtrd] revive ${row.node_id} (idle-release, inbox)\n`);
1178
- reviveNode(row.node_id, { resume: true });
1179
- revivedThisTick.add(row.node_id); // third-pass bare double-spawn guard (Maj-4)
1383
+ const cursorNow = readCursor(id);
1384
+ const entries = readInboxSince(id, cursorNow);
1385
+ if (entries.length === 0) {
1386
+ clearIdleReleaseRelaunchState(id); // fully delivered — no stuck episode to track
1387
+ continue;
1388
+ }
1389
+ const prior = idleReleaseRelaunchState.get(id);
1390
+ // Cursor progress since the LAST time this episode was considered (or no
1391
+ // prior episode at all) — the ONLY thing allowed to reset the counter.
1392
+ const progressed = prior === undefined || prior.cursor !== cursorNow;
1393
+ const attempts = progressed ? 0 : prior.attempts;
1394
+ if (attempts >= IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS) {
1395
+ clearIdleReleaseRelaunchState(id);
1396
+ const meta = getNode(id);
1397
+ process.stderr.write(`[crtrd] giving up on ${id} after ${attempts} idle-release relaunch attempts (no inbox-cursor progress)\n`);
1398
+ transition(id, 'crash');
1399
+ if (meta !== null) {
1400
+ try {
1401
+ fanDoctrineWake(id, subscribersOf(id), `Child crashed — ${fullName(meta)} (${id}) failed to relaunch after ${attempts} attempts while trying to ` +
1402
+ `deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays ` +
1403
+ `dead until you revive it — \`crtr node lifecycle revive ${id}\`.`, { reason: 'child-crashed', child: id });
1404
+ }
1405
+ catch {
1406
+ /* best-effort, mirrors every other fan-out in this file */
1407
+ }
1408
+ }
1409
+ continue;
1180
1410
  }
1411
+ const lastAttempt = progressed ? undefined : prior?.lastAttempt;
1412
+ const backoff = idleReleaseRelaunchBackoffMs(attempts);
1413
+ if (lastAttempt !== undefined && now - lastAttempt < backoff)
1414
+ continue; // backing off before the next attempt
1415
+ if (!hasBrokerCapacity(capacity))
1416
+ continue; // deferred, not attempted — costs no retry budget
1417
+ idleReleaseRelaunchState.set(id, { cursor: cursorNow, attempts: attempts + 1, lastAttempt: now });
1418
+ process.stderr.write(`[crtrd] revive ${id} (idle-release, inbox, attempt ${attempts + 1}/${IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS})\n`);
1419
+ reviveNode(id, { resume: true });
1420
+ capacity.count++;
1421
+ revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
1181
1422
  }
1182
1423
  catch (err) {
1183
- process.stderr.write(`[crtrd] error polling inbox ${row.node_id}: ${err.message}\n`);
1424
+ process.stderr.write(`[crtrd] error polling inbox ${id}: ${err.message}\n`);
1184
1425
  }
1185
1426
  }
1186
1427
  // Third pass: FIRE DUE CLOCK TRIGGERS (the triggers engine, migration v9).
@@ -21,6 +21,7 @@ import { closeDb } from '../../core/canvas/db.js';
21
21
  import { reportsDir } from '../../core/canvas/paths.js';
22
22
  import { readInboxSince } from '../../core/feed/inbox.js';
23
23
  import { STALL_REPROMPT } from '../../core/runtime/stop-guard.js';
24
+ import { recordFault, readFault } from '../../core/runtime/fault.js';
24
25
  let home;
25
26
  let origNode;
26
27
  let origPane;
@@ -262,6 +263,46 @@ test('refresh-yield (intent=refresh) writes NO push — silent to subscribers',
262
263
  assert.equal(readInboxSince('root').length, 0, 'a yield is silent: NO inbox pointer');
263
264
  assert.equal(pi.injected.length, 0, 'no reprompt on a refresh-yield');
264
265
  });
266
+ // ---------------------------------------------------------------------------
267
+ // turn_end — stale-⚠ fix: reaching turn_end proves the current provider
268
+ // round-trip and any wedge episode are both over, so both fault links clear.
269
+ // ---------------------------------------------------------------------------
270
+ /** A turn_end event carrying one assistant text block. */
271
+ function turnEndEvent(text) {
272
+ return { message: { role: 'assistant', content: [{ type: 'text', text }] } };
273
+ }
274
+ test('turn_end clears a stale pi→provider fatal fault (a completed round-trip proves the episode is over)', () => {
275
+ createNode(node('leaf', { parent: null, lifecycle: 'terminal' }));
276
+ recordFault('leaf', {
277
+ link: 'pi→provider',
278
+ op: 'generation turn',
279
+ kind: 'other',
280
+ retry: { disposition: 'fatal' },
281
+ message: 'terminated',
282
+ });
283
+ assert.notEqual(readFault('leaf'), null, 'precondition: fault marker recorded');
284
+ process.env['CRTR_NODE_ID'] = 'leaf';
285
+ const pi = makeFakePi();
286
+ registerCanvasStophook(pi);
287
+ pi.fire('turn_end', turnEndEvent('all good now'), {});
288
+ assert.equal(readFault('leaf'), null, 'the pi→provider fault is cleared at turn_end');
289
+ });
290
+ test('turn_end still clears a daemon→node wedge fault (existing behavior, unchanged)', () => {
291
+ createNode(node('leaf', { parent: null, lifecycle: 'terminal' }));
292
+ recordFault('leaf', {
293
+ link: 'daemon→node',
294
+ op: 'wedge detection',
295
+ kind: 'wedged',
296
+ retry: { disposition: 'manual' },
297
+ message: 'no progress for 20m',
298
+ });
299
+ assert.notEqual(readFault('leaf'), null, 'precondition: wedge fault recorded');
300
+ process.env['CRTR_NODE_ID'] = 'leaf';
301
+ const pi = makeFakePi();
302
+ registerCanvasStophook(pi);
303
+ pi.fire('turn_end', turnEndEvent('turn completed'), {});
304
+ assert.equal(readFault('leaf'), null, 'the daemon→node wedge fault is still cleared at turn_end');
305
+ });
265
306
  test('stalled leaf (nothing live to await, no final) is still reprompted', () => {
266
307
  createNode(node('mgr', { parent: null, lifecycle: 'terminal', mode: 'orchestrator' }));
267
308
  createNode(node('leaf', { parent: 'mgr', lifecycle: 'terminal', status: 'active' }));
@@ -0,0 +1,121 @@
1
+ // Run with: node --import tsx/esm --test src/pi-extensions/__tests__/canvas-stophook-context-nudge.test.ts
2
+ //
3
+ // The context-size nudge's delivery mode splits on whether the agent run
4
+ // continues past the turn. The regression this pins: the nudge used to be an
5
+ // unconditional STEER, and pi's agent loop drains the steering queue right
6
+ // after turn_end — so a band crossing on the run's FINAL message (no tool
7
+ // calls) extended the run past the agent's closing reply, burying the summary
8
+ // a human comes back to read once the conversation goes quiet.
9
+ // • turn_end with toolCall content (mid-run) → sendUserMessage steer
10
+ // • turn_end with text-only content (final) → sendMessage 'nextTurn' hold
11
+ //
12
+ // Second regression pinned here: a tool-calling turn ends with stopReason
13
+ // 'toolUse' (pi-ai's StopReason), NEVER 'stop'. Gating the steer path on
14
+ // stopReason 'stop'/'length' made runContinues unconditionally false, so the
15
+ // nudge was ALWAYS appended as nextTurn and never steered mid-run.
16
+ import { test, before, after, beforeEach } from 'node:test';
17
+ import assert from 'node:assert/strict';
18
+ import { mkdtempSync, rmSync } from 'node:fs';
19
+ import { tmpdir } from 'node:os';
20
+ import { join } from 'node:path';
21
+ import registerCanvasStophook from '../canvas-stophook.js';
22
+ import { createNode } from '../../core/canvas/canvas.js';
23
+ import { closeDb } from '../../core/canvas/db.js';
24
+ let home;
25
+ let origNode;
26
+ function node(id) {
27
+ return {
28
+ node_id: id,
29
+ name: id,
30
+ created: new Date().toISOString(),
31
+ cwd: '/tmp/work',
32
+ kind: 'general',
33
+ mode: 'base',
34
+ lifecycle: 'terminal',
35
+ status: 'active',
36
+ };
37
+ }
38
+ function makeFakePi() {
39
+ const handlers = {};
40
+ return {
41
+ steered: [],
42
+ held: [],
43
+ on(e, h) { handlers[e] = h; },
44
+ sendUserMessage(content, options) { this.steered.push({ content, deliverAs: options?.deliverAs }); },
45
+ sendMessage(m, options) {
46
+ this.held.push({ customType: m.customType, content: m.content, deliverAs: options?.deliverAs, triggerTurn: options?.triggerTurn });
47
+ },
48
+ fire(e, ev, ctx) { handlers[e]?.(ev, ctx); },
49
+ };
50
+ }
51
+ /** A turn_end event whose assistant message crossed a steering band. A
52
+ * tool-calling turn really ends with stopReason 'toolUse'; text-only finals
53
+ * end with 'stop'. */
54
+ function turnEnd(content, stopReason = 'stop') {
55
+ return { message: { role: 'assistant', stopReason, content, usage: { input: 1, output: 1 } }, toolResults: [] };
56
+ }
57
+ const ctxAt = (tokens) => ({ getContextUsage: () => ({ tokens }) });
58
+ before(() => { origNode = process.env['CRTR_NODE_ID']; });
59
+ beforeEach(() => {
60
+ closeDb();
61
+ if (home)
62
+ rmSync(home, { recursive: true, force: true });
63
+ home = mkdtempSync(join(tmpdir(), 'crtr-stophook-nudge-'));
64
+ process.env['CRTR_HOME'] = home;
65
+ });
66
+ after(() => {
67
+ closeDb();
68
+ if (home)
69
+ rmSync(home, { recursive: true, force: true });
70
+ delete process.env['CRTR_HOME'];
71
+ if (origNode === undefined)
72
+ delete process.env['CRTR_NODE_ID'];
73
+ else
74
+ process.env['CRTR_NODE_ID'] = origNode;
75
+ });
76
+ test('band crossing mid-run (tool calls present) → context nudge is a STEER', () => {
77
+ createNode(node('midrun'));
78
+ process.env['CRTR_NODE_ID'] = 'midrun';
79
+ const pi = makeFakePi();
80
+ registerCanvasStophook(pi);
81
+ pi.fire('turn_end', turnEnd([{ type: 'toolCall', id: 't1', name: 'bash', arguments: {} }], 'toolUse'), ctxAt(150_000));
82
+ assert.equal(pi.steered.length, 1, 'exactly one steer delivered');
83
+ assert.equal(pi.steered[0].deliverAs, 'steer');
84
+ assert.match(pi.steered[0].content, /^\[crtr\] Context ~150k/);
85
+ assert.equal(pi.held.length, 0, 'nothing held for nextTurn');
86
+ });
87
+ test('band crossing on the FINAL message (no tool calls) → nudge is HELD as nextTurn, never a steer', () => {
88
+ createNode(node('final'));
89
+ process.env['CRTR_NODE_ID'] = 'final';
90
+ const pi = makeFakePi();
91
+ registerCanvasStophook(pi);
92
+ pi.fire('turn_end', turnEnd([{ type: 'text', text: 'here is my summary' }]), ctxAt(150_000));
93
+ assert.equal(pi.steered.length, 0, 'no steer — a steer here would extend the run past the closing reply');
94
+ assert.equal(pi.held.length, 1, 'nudge held for the next user message');
95
+ assert.equal(pi.held[0].deliverAs, 'nextTurn');
96
+ assert.equal(pi.held[0].customType, 'crtr-context-nudge');
97
+ assert.match(pi.held[0].content, /^\[crtr\] Context ~150k/);
98
+ assert.notEqual(pi.held[0].triggerTurn, true, 'must not trigger a turn');
99
+ });
100
+ test('a truncated tool-calling turn (stopReason length) still steers — the loop continues past failed calls', () => {
101
+ createNode(node('lengthrun'));
102
+ process.env['CRTR_NODE_ID'] = 'lengthrun';
103
+ const pi = makeFakePi();
104
+ registerCanvasStophook(pi);
105
+ pi.fire('turn_end', turnEnd([{ type: 'toolCall', id: 't1', name: 'bash', arguments: {} }], 'length'), ctxAt(150_000));
106
+ assert.equal(pi.steered.length, 1);
107
+ assert.equal(pi.steered[0].deliverAs, 'steer');
108
+ assert.equal(pi.held.length, 0);
109
+ });
110
+ test('an aborted final turn also holds (the loop ends unconditionally on abort)', () => {
111
+ createNode(node('aborted'));
112
+ process.env['CRTR_NODE_ID'] = 'aborted';
113
+ const pi = makeFakePi();
114
+ registerCanvasStophook(pi);
115
+ // Aborted mid-tool-call: toolCall content present, but the run ends anyway —
116
+ // a steer would sit queued and leak into the NEXT run's start instead.
117
+ pi.fire('turn_end', turnEnd([{ type: 'toolCall', id: 't1', name: 'bash', arguments: {} }], 'aborted'), ctxAt(150_000));
118
+ assert.equal(pi.steered.length, 0);
119
+ assert.equal(pi.held.length, 1);
120
+ assert.equal(pi.held[0].deliverAs, 'nextTurn');
121
+ });
@@ -1,4 +1,4 @@
1
- type PiEvents = 'session_start' | 'turn_end' | 'agent_start' | 'agent_end' | 'session_shutdown';
1
+ type PiEvents = 'session_start' | 'turn_end' | 'agent_start' | 'agent_end' | 'agent_settled' | 'session_shutdown';
2
2
  interface PiLike {
3
3
  on: (event: PiEvents, handler: (event: any, ctx: any) => void | Promise<void>) => void;
4
4
  sendUserMessage: (content: string, options?: {