@crouton-kit/crouter 0.3.28 → 0.3.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/build-root.d.ts +15 -7
- package/dist/build-root.js +48 -34
- package/dist/builtin-memory/crouter-development/marketplaces.md +7 -6
- package/dist/builtin-memory/crouter-development/personas/base-prompt.md +1 -1
- package/dist/builtin-memory/crouter-development/personas/orchestrator-prompt.md +1 -1
- package/dist/builtin-memory/crouter-development/personas.md +1 -1
- package/dist/builtin-memory/crouter-development/plugins.md +20 -18
- package/dist/builtin-memory/design.md +1 -1
- package/dist/builtin-memory/development.md +1 -1
- package/dist/builtin-memory/internal/INDEX.md +1 -1
- package/dist/builtin-memory/internal/examples/INDEX.md +1 -1
- package/dist/builtin-memory/internal/examples/imessage-assistant.md +2 -2
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-memory/internal/storage-tiers.md +1 -1
- package/dist/builtin-memory/planning.md +1 -1
- package/dist/builtin-memory/spec.md +6 -4
- package/dist/builtin-personas/orchestration-kernel.md +4 -5
- package/dist/builtin-personas/runtime-base.md +5 -4
- package/dist/builtin-personas/spec/PERSONA.md +8 -4
- package/dist/builtin-personas/spec/orchestrator.md +5 -3
- package/dist/builtin-personas/spec/requirements/PERSONA.md +10 -0
- package/dist/builtin-views/canvas/core.mjs +586 -0
- package/dist/builtin-views/canvas/text.mjs +58 -0
- package/dist/builtin-views/canvas/tui.mjs +165 -0
- package/dist/builtin-views/canvas/web.jsx +120 -0
- package/dist/builtin-views/git-pr/core.mjs +673 -0
- package/dist/builtin-views/git-pr/text.mjs +84 -0
- package/dist/builtin-views/git-pr/tui.mjs +302 -0
- package/dist/builtin-views/git-pr/web.jsx +216 -0
- package/dist/builtin-views/inbox/_lib/render.mjs +1 -3
- package/dist/builtin-views/inbox/core.mjs +1273 -0
- package/dist/builtin-views/inbox/text.mjs +73 -0
- package/dist/builtin-views/inbox/tui.mjs +312 -0
- package/dist/builtin-views/inbox/web.jsx +188 -0
- package/dist/builtin-views/linkedin/core.mjs +906 -0
- package/dist/builtin-views/linkedin/text.mjs +69 -0
- package/dist/builtin-views/linkedin/tui.mjs +427 -0
- package/dist/builtin-views/linkedin/web.jsx +206 -0
- package/dist/builtin-views/workspace-sidebar/core.mjs +621 -0
- package/dist/builtin-views/workspace-sidebar/text.mjs +53 -0
- package/dist/builtin-views/workspace-sidebar/tui.mjs +142 -0
- package/dist/builtin-views/workspace-sidebar/web.jsx +109 -0
- package/dist/cli.js +22 -17
- package/dist/clients/attach/__tests__/action-parity.test.js +48 -0
- package/dist/clients/attach/__tests__/clipboard-image.test.js +76 -0
- package/dist/clients/attach/__tests__/editor-newline.test.js +57 -0
- package/dist/clients/attach/__tests__/git-info.test.js +31 -0
- package/dist/clients/attach/__tests__/onrequest-wired.test.js +20 -0
- package/dist/clients/attach/__tests__/slash-quit-copy.test.js +42 -0
- package/dist/clients/attach/__tests__/titled-editor.test.js +39 -0
- package/dist/clients/attach/attach-cmd.js +282 -46
- package/dist/clients/attach/auth-pickers.d.ts +9 -0
- package/dist/clients/attach/auth-pickers.js +194 -0
- package/dist/clients/attach/canvas-panels.d.ts +4 -2
- package/dist/clients/attach/canvas-panels.js +45 -17
- package/dist/clients/attach/chat-view.d.ts +27 -2
- package/dist/clients/attach/chat-view.js +98 -9
- package/dist/clients/attach/clipboard-image.d.ts +47 -8
- package/dist/clients/attach/clipboard-image.js +150 -26
- package/dist/clients/attach/clipboard-text.d.ts +3 -0
- package/dist/clients/attach/clipboard-text.js +39 -0
- package/dist/clients/attach/config-load.d.ts +54 -7
- package/dist/clients/attach/config-load.js +99 -3
- package/dist/clients/attach/context-message.d.ts +12 -0
- package/dist/clients/attach/context-message.js +72 -0
- package/dist/clients/attach/git-info.d.ts +20 -0
- package/dist/clients/attach/git-info.js +44 -0
- package/dist/clients/attach/graph-overlay.d.ts +13 -4
- package/dist/clients/attach/graph-overlay.js +59 -19
- package/dist/clients/attach/input-controller.d.ts +75 -13
- package/dist/clients/attach/input-controller.js +305 -67
- package/dist/clients/attach/pickers.d.ts +58 -0
- package/dist/clients/attach/pickers.js +171 -0
- package/dist/clients/attach/slash-commands.d.ts +22 -0
- package/dist/clients/attach/slash-commands.js +91 -25
- package/dist/clients/attach/titled-editor.d.ts +33 -0
- package/dist/clients/attach/titled-editor.js +84 -0
- package/dist/clients/attach/view-socket.d.ts +18 -1
- package/dist/clients/attach/view-socket.js +70 -1
- package/dist/clients/web/dev-server.d.ts +7 -0
- package/dist/clients/web/dev-server.js +59 -0
- package/dist/clients/web/events.d.ts +14 -0
- package/dist/clients/web/events.js +151 -0
- package/dist/clients/web/server.d.ts +18 -0
- package/dist/clients/web/server.js +450 -0
- package/dist/clients/web/web-cmd.d.ts +2 -0
- package/dist/clients/web/web-cmd.js +120 -0
- package/dist/commands/canvas.js +1 -2
- package/dist/commands/chord.js +4 -25
- package/dist/commands/dashboard.js +5 -1
- package/dist/commands/memory/__tests__/lint-schema.test.js +6 -6
- package/dist/commands/memory/lint.js +14 -2
- package/dist/commands/memory/read.js +2 -2
- package/dist/commands/memory/shared.d.ts +7 -1
- package/dist/commands/memory/shared.js +18 -3
- package/dist/commands/memory/write.js +8 -4
- package/dist/commands/memory.js +4 -4
- package/dist/commands/node.d.ts +1 -1
- package/dist/commands/node.js +64 -65
- package/dist/commands/pkg.js +1 -2
- package/dist/commands/revive.js +11 -3
- package/dist/commands/sys/sync.d.ts +1 -0
- package/dist/commands/sys/sync.js +187 -0
- package/dist/commands/sys.js +3 -2
- package/dist/commands/view-cycle.js +2 -2
- package/dist/commands/view-list.js +8 -8
- package/dist/commands/view-new.js +21 -17
- package/dist/commands/view-pick.js +1 -1
- package/dist/commands/view-run.js +35 -14
- package/dist/commands/view.js +8 -6
- package/dist/commands/workspace.d.ts +2 -0
- package/dist/commands/workspace.js +161 -0
- package/dist/core/__tests__/broker-double-spawn.test.d.ts +1 -0
- package/dist/core/__tests__/broker-double-spawn.test.js +143 -0
- package/dist/core/__tests__/broker-fork-seam.test.d.ts +1 -0
- package/dist/core/__tests__/broker-fork-seam.test.js +102 -0
- package/dist/core/__tests__/broker-sdk-wiring.test.js +2 -0
- package/dist/core/__tests__/child-death-wake.test.js +56 -111
- package/dist/core/__tests__/close.test.js +14 -20
- package/dist/core/__tests__/context-intro.test.js +19 -19
- package/dist/core/__tests__/daemon-boot.test.js +30 -23
- package/dist/core/__tests__/draw-style.test.js +1 -1
- package/dist/core/__tests__/error-stall-recycle.test.d.ts +1 -0
- package/dist/core/__tests__/error-stall-recycle.test.js +141 -0
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +17 -1
- package/dist/core/__tests__/fixtures/fake-engine.js +22 -2
- package/dist/core/__tests__/fixtures/fake-pi-host.js +2 -2
- package/dist/core/__tests__/focuses.test.js +25 -13
- package/dist/core/__tests__/fork.test.js +22 -5
- package/dist/core/__tests__/full/broker-control-preempt.test.d.ts +1 -0
- package/dist/core/__tests__/full/broker-control-preempt.test.js +61 -0
- package/dist/core/__tests__/full/broker-model-changed-broadcast.test.d.ts +1 -0
- package/dist/core/__tests__/full/broker-model-changed-broadcast.test.js +57 -0
- package/dist/core/__tests__/full/broker-navigate-tree-rewelcome.test.d.ts +1 -0
- package/dist/core/__tests__/full/broker-navigate-tree-rewelcome.test.js +80 -0
- package/dist/core/__tests__/full/broker-pane-resolution.test.js +90 -0
- package/dist/core/__tests__/full/cascade-close.test.js +22 -8
- package/dist/core/__tests__/full/detach-focus.test.js +66 -169
- package/dist/core/__tests__/helpers/harness.d.ts +6 -9
- package/dist/core/__tests__/helpers/harness.js +19 -36
- package/dist/core/__tests__/human-node-not-supervised.test.d.ts +1 -0
- package/dist/core/__tests__/human-node-not-supervised.test.js +83 -0
- package/dist/core/__tests__/lifecycle.test.js +1 -19
- package/dist/core/__tests__/listing-completeness.test.js +2 -2
- package/dist/core/__tests__/live-mutation-verbs.test.js +5 -5
- package/dist/core/__tests__/memory-resolver.test.js +1 -1
- package/dist/core/__tests__/on-read-dedup-resume.test.js +2 -2
- package/dist/core/__tests__/refresh-stall-recycle.test.d.ts +1 -0
- package/dist/core/__tests__/refresh-stall-recycle.test.js +88 -0
- package/dist/core/__tests__/relaunch-root.test.d.ts +1 -0
- package/dist/core/__tests__/relaunch-root.test.js +114 -0
- package/dist/core/__tests__/reset.test.js +19 -79
- package/dist/core/__tests__/steer-note.test.js +9 -7
- package/dist/core/__tests__/tmux-surface.test.js +13 -7
- package/dist/core/bootstrap.d.ts +0 -3
- package/dist/core/bootstrap.js +1 -143
- package/dist/core/canvas/browse/__tests__/model.test.js +86 -1
- package/dist/core/canvas/browse/__tests__/render.test.js +2 -1
- package/dist/core/canvas/browse/app.js +131 -8
- package/dist/core/canvas/browse/model.d.ts +23 -5
- package/dist/core/canvas/browse/model.js +68 -13
- package/dist/core/canvas/browse/render.d.ts +7 -1
- package/dist/core/canvas/browse/render.js +178 -47
- package/dist/core/canvas/canvas.js +1 -1
- package/dist/core/canvas/focuses.d.ts +4 -4
- package/dist/core/canvas/focuses.js +7 -7
- package/dist/core/canvas/nav-model.js +8 -7
- package/dist/core/canvas/render.d.ts +68 -9
- package/dist/core/canvas/render.js +195 -61
- package/dist/core/canvas/types.d.ts +16 -23
- package/dist/core/config.js +1 -4
- package/dist/core/memory-resolver.js +43 -5
- package/dist/core/runtime/bearings.d.ts +1 -1
- package/dist/core/runtime/bearings.js +8 -8
- package/dist/core/runtime/branded-host.d.ts +16 -0
- package/dist/core/runtime/branded-host.js +127 -0
- package/dist/core/runtime/broker-protocol.d.ts +212 -6
- package/dist/core/runtime/broker.d.ts +4 -1
- package/dist/core/runtime/broker.js +494 -58
- package/dist/core/runtime/close.d.ts +12 -2
- package/dist/core/runtime/close.js +37 -16
- package/dist/core/runtime/front-door.js +6 -14
- package/dist/core/runtime/host.d.ts +14 -34
- package/dist/core/runtime/host.js +14 -50
- package/dist/core/runtime/launch.d.ts +12 -9
- package/dist/core/runtime/launch.js +27 -15
- package/dist/core/runtime/lifecycle.d.ts +1 -1
- package/dist/core/runtime/lifecycle.js +15 -19
- package/dist/core/runtime/nodes.d.ts +0 -55
- package/dist/core/runtime/nodes.js +6 -74
- package/dist/core/runtime/placement.d.ts +91 -335
- package/dist/core/runtime/placement.js +291 -858
- package/dist/core/runtime/promote.d.ts +2 -0
- package/dist/core/runtime/promote.js +24 -6
- package/dist/core/runtime/recap.d.ts +8 -0
- package/dist/core/runtime/recap.js +107 -0
- package/dist/core/runtime/recycle.js +25 -61
- package/dist/core/runtime/reset.d.ts +43 -37
- package/dist/core/runtime/reset.js +131 -218
- package/dist/core/runtime/revive.d.ts +9 -29
- package/dist/core/runtime/revive.js +40 -139
- package/dist/core/runtime/spawn.d.ts +15 -16
- package/dist/core/runtime/spawn.js +165 -194
- package/dist/core/runtime/surface-bg.d.ts +11 -0
- package/dist/core/runtime/surface-bg.js +68 -0
- package/dist/core/runtime/tmux.d.ts +15 -52
- package/dist/core/runtime/tmux.js +35 -99
- package/dist/core/scope.d.ts +1 -2
- package/dist/core/scope.js +1 -5
- package/dist/core/skill-sync/__tests__/dry-run-wrote-count.test.d.ts +1 -0
- package/dist/core/skill-sync/__tests__/dry-run-wrote-count.test.js +57 -0
- package/dist/core/skill-sync/builtins.d.ts +42 -0
- package/dist/core/skill-sync/builtins.js +112 -0
- package/dist/core/skill-sync/claude-plugins.d.ts +23 -0
- package/dist/core/skill-sync/claude-plugins.js +71 -0
- package/dist/core/skill-sync/engine.d.ts +42 -0
- package/dist/core/skill-sync/engine.js +633 -0
- package/dist/core/skill-sync/export.d.ts +23 -0
- package/dist/core/skill-sync/export.js +86 -0
- package/dist/core/skill-sync/manifest.d.ts +64 -0
- package/dist/core/skill-sync/manifest.js +181 -0
- package/dist/core/skill-sync/profile.d.ts +76 -0
- package/dist/core/skill-sync/profile.js +173 -0
- package/dist/core/skill-sync/snapshot.d.ts +57 -0
- package/dist/core/skill-sync/snapshot.js +120 -0
- package/dist/core/substrate/index.d.ts +1 -1
- package/dist/core/substrate/index.js +1 -1
- package/dist/core/substrate/injected-store.js +3 -3
- package/dist/core/substrate/on-read.js +68 -6
- package/dist/core/substrate/render.d.ts +8 -11
- package/dist/core/substrate/render.js +29 -43
- package/dist/core/substrate/schema.d.ts +10 -3
- package/dist/core/substrate/schema.js +6 -3
- package/dist/core/tui/host.d.ts +12 -4
- package/dist/core/tui/host.js +280 -149
- package/dist/core/view/bridge.d.ts +10 -0
- package/dist/core/view/bridge.js +31 -0
- package/dist/core/view/chrome.d.ts +9 -0
- package/dist/core/view/chrome.js +22 -0
- package/dist/core/view/contract.d.ts +171 -0
- package/dist/core/view/contract.js +23 -0
- package/dist/core/view/loader.d.ts +31 -0
- package/dist/core/view/loader.js +188 -0
- package/dist/core/view/transport-local.d.ts +7 -0
- package/dist/core/view/transport-local.js +70 -0
- package/dist/core/view/transport.d.ts +4 -0
- package/dist/core/view/transport.js +15 -0
- package/dist/daemon/crtrd.d.ts +38 -27
- package/dist/daemon/crtrd.js +303 -376
- package/dist/daemon/manage.js +6 -1
- package/dist/index.js +1 -1
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +28 -20
- package/dist/pi-extensions/canvas-context-intro.js +4 -4
- package/dist/pi-extensions/canvas-doc-substrate.js +16 -16
- package/dist/pi-extensions/canvas-nav.js +5 -0
- package/dist/pi-extensions/canvas-recap.d.ts +37 -0
- package/dist/pi-extensions/canvas-recap.js +236 -0
- package/dist/pi-extensions/canvas-resume.js +2 -1
- package/dist/pi-extensions/canvas-stophook.d.ts +14 -12
- package/dist/pi-extensions/canvas-stophook.js +87 -101
- package/dist/pi-extensions/canvas-view.js +2 -1
- package/dist/pi-extensions/widget-order-bus.d.ts +6 -0
- package/dist/pi-extensions/widget-order-bus.js +34 -0
- package/dist/prompts/view.d.ts +2 -2
- package/dist/prompts/view.js +145 -73
- package/dist/types.d.ts +1 -6
- package/dist/types.js +1 -3
- package/dist/web/ViewChrome.d.ts +7 -0
- package/dist/web/ViewChrome.js +28 -0
- package/dist/web/ViewPane.d.ts +39 -0
- package/dist/web/ViewPane.js +48 -0
- package/dist/web/index.d.ts +6 -0
- package/dist/web/index.js +16 -0
- package/dist/web/runtime.d.ts +39 -0
- package/dist/web/runtime.js +133 -0
- package/dist/web/states.d.ts +24 -0
- package/dist/web/states.js +24 -0
- package/dist/web/transport-http.d.ts +5 -0
- package/dist/web/transport-http.js +28 -0
- package/dist/web-client/assets/index-BUvQb4hR.css +2 -0
- package/dist/web-client/assets/index-ClLQXYAE.js +10 -0
- package/dist/web-client/index.html +13 -0
- package/package.json +20 -6
- package/dist/builtin-views/canvas/client.mjs +0 -303
- package/dist/builtin-views/canvas/view.mjs +0 -576
- package/dist/builtin-views/git-pr/client.mjs +0 -440
- package/dist/builtin-views/git-pr/view.mjs +0 -675
- package/dist/builtin-views/inbox/sources/gmail.mjs +0 -965
- package/dist/builtin-views/inbox/sources/linkedin.mjs +0 -427
- package/dist/builtin-views/inbox/view.mjs +0 -889
- package/dist/builtin-views/linkedin/client.mjs +0 -610
- package/dist/builtin-views/linkedin/view.mjs +0 -1171
- package/dist/commands/pkg/bridge.d.ts +0 -1
- package/dist/commands/pkg/bridge.js +0 -137
- package/dist/commands/skill/author.d.ts +0 -3
- package/dist/commands/skill/author.js +0 -140
- package/dist/commands/skill/shared.d.ts +0 -3
- package/dist/commands/skill/shared.js +0 -19
- package/dist/commands/skill.d.ts +0 -2
- package/dist/commands/skill.js +0 -21
- package/dist/commands/tmux-spread.d.ts +0 -2
- package/dist/commands/tmux-spread.js +0 -144
- package/dist/core/__tests__/full/daemon-liveness-pane.full.test.js +0 -292
- package/dist/core/__tests__/full/placement-focus.test.js +0 -309
- package/dist/core/__tests__/full/placement-reconcile.test.js +0 -212
- package/dist/core/__tests__/full/placement-revive.test.js +0 -238
- package/dist/core/__tests__/full/placement-teardown.test.js +0 -297
- package/dist/core/__tests__/home-session.test.js +0 -186
- package/dist/core/__tests__/relaunch.test.js +0 -335
- package/dist/core/bridge-map.d.ts +0 -19
- package/dist/core/bridge-map.js +0 -73
- package/dist/core/tui/contract.d.ts +0 -83
- package/dist/core/tui/contract.js +0 -8
- package/dist/core/tui/loader.d.ts +0 -16
- package/dist/core/tui/loader.js +0 -94
- package/dist/prompts/skill.d.ts +0 -2
- package/dist/prompts/skill.js +0 -650
- /package/dist/{core/__tests__/full/daemon-liveness-pane.full.test.d.ts → clients/attach/__tests__/action-parity.test.d.ts} +0 -0
- /package/dist/{core/__tests__/full/placement-focus.test.d.ts → clients/attach/__tests__/clipboard-image.test.d.ts} +0 -0
- /package/dist/{core/__tests__/full/placement-reconcile.test.d.ts → clients/attach/__tests__/editor-newline.test.d.ts} +0 -0
- /package/dist/{core/__tests__/full/placement-revive.test.d.ts → clients/attach/__tests__/git-info.test.d.ts} +0 -0
- /package/dist/{core/__tests__/full/placement-teardown.test.d.ts → clients/attach/__tests__/onrequest-wired.test.d.ts} +0 -0
- /package/dist/{core/__tests__/home-session.test.d.ts → clients/attach/__tests__/slash-quit-copy.test.d.ts} +0 -0
- /package/dist/{core/__tests__/relaunch.test.d.ts → clients/attach/__tests__/titled-editor.test.d.ts} +0 -0
package/dist/daemon/crtrd.js
CHANGED
|
@@ -1,45 +1,60 @@
|
|
|
1
1
|
// crtrd — the thin supervisor daemon. One instance per canvas.
|
|
2
2
|
//
|
|
3
|
-
// Sole responsibility: supervise engine
|
|
4
|
-
// headless broker process — `host_kind`) and revive nodes. No orchestration
|
|
3
|
+
// Sole responsibility: supervise engine exit and revive nodes. No orchestration
|
|
5
4
|
// logic lives here. The daemon is a process-lifecycle watcher.
|
|
6
5
|
//
|
|
7
|
-
// Model (
|
|
8
|
-
//
|
|
9
|
-
//
|
|
6
|
+
// Model (broker-is-the-only-host): EVERY node runs on a detached headless broker
|
|
7
|
+
// process; a tmux pane, when present, is only a VIEWER of that engine and never
|
|
8
|
+
// hosts it. So liveness is pid-only — the broker pid the stophook records on
|
|
9
|
+
// session_start. A viewer window closing is not a node death; the daemon never
|
|
10
|
+
// reaps a node on pane/window state. All node liveness routes through
|
|
11
|
+
// handleNodeLiveness:
|
|
10
12
|
// • Poll every intervalMs (default 2000ms).
|
|
11
|
-
// •
|
|
12
|
-
//
|
|
13
|
-
// •
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// •
|
|
17
|
-
//
|
|
18
|
-
// gains an unseen entry.
|
|
19
|
-
// •
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
//
|
|
13
|
+
// • pid alive → leave; but enforce a stuck refresh-yield (§H) and a
|
|
14
|
+
// wedged-engine error-stall (§I, see below).
|
|
15
|
+
// • pid null + pi_session_id set → relaunch in flight; leave.
|
|
16
|
+
// • pid null + pi_session_id null → after a boot grace, crash + surface boot
|
|
17
|
+
// failure (the broker exited before session_start).
|
|
18
|
+
// • pid dead + intent==='refresh' → fresh respawn (node asked to yield).
|
|
19
|
+
// • pid dead + intent==='idle-release' → dormant by choice; the second pass
|
|
20
|
+
// revives (resume) when its inbox gains an unseen entry.
|
|
21
|
+
// • pid dead + any other intent → grace-revive RESUME on the saved session.
|
|
22
|
+
//
|
|
23
|
+
// §H refresh-authority: a stale in-process stophook (pi extensions never reload,
|
|
24
|
+
// so a days-old resident carries an old hook) can leave intent='refresh' on a
|
|
25
|
+
// LIVE engine forever — the hook never calls shutdown, so the pid never dies and
|
|
26
|
+
// the normal dead-pid refresh path never fires. The daemon is the authority:
|
|
27
|
+
// when intent='refresh' persists on a live engine whose turn is over, past a
|
|
28
|
+
// grace, it force-kills the engine itself (handleYieldStall) so the dead-pid
|
|
29
|
+
// refresh path then revives it fresh.
|
|
30
|
+
//
|
|
31
|
+
// §I error-stall: an engine whose turn ends with stopReason 'error' (e.g. a
|
|
32
|
+
// provider outage exhausted the SDK retry budget) is left PARKED by the
|
|
33
|
+
// stophook ("stay alive for re-steering") — but a headless node has no human to
|
|
34
|
+
// re-steer it, so the broker stays alive, status stays 'active', and the pid-only
|
|
35
|
+
// liveness check reads it healthy FOREVER (gh issue #4: a 9h zombie). The daemon
|
|
36
|
+
// is the authority here too: when a live, not-busy node with no pending intent
|
|
37
|
+
// has a session file that is QUIET past a grace AND whose last assistant message
|
|
38
|
+
// stopped with 'error', it kills the engine (handleErrorStall) so the ordinary
|
|
39
|
+
// dead-pid crash path grace-revives RESUME on the saved session — the exact
|
|
40
|
+
// manual recovery (kill broker + canvas revive) that fixes these by hand.
|
|
25
41
|
//
|
|
26
42
|
// Single-instance guarantee
|
|
27
43
|
// A PID file at crtrHome()/crtrd.pid prevents double-runs. On start, if the
|
|
28
44
|
// file exists and the recorded pid is alive, we refuse to start (exit 0).
|
|
29
45
|
// On stop (SIGINT/SIGTERM/exit) we remove the file.
|
|
30
|
-
import { writeFileSync, readFileSync, rmSync, existsSync, mkdirSync, statSync, } from 'node:fs';
|
|
46
|
+
import { writeFileSync, readFileSync, rmSync, existsSync, mkdirSync, statSync, openSync, readSync, closeSync, } from 'node:fs';
|
|
31
47
|
import { join } from 'node:path';
|
|
32
48
|
import { crtrHome, nodeDir, jobDir } from '../core/canvas/paths.js';
|
|
33
|
-
import { listNodes, getRow,
|
|
49
|
+
import { listNodes, getRow, getNode, dueWakes, consumeWake, advanceWake, listFocuses, closeFocusRow, } from '../core/canvas/index.js';
|
|
34
50
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
35
51
|
import { isBusy } from '../core/runtime/busy.js';
|
|
36
52
|
import { isPidAlive } from '../core/canvas/pid.js';
|
|
37
|
-
import {
|
|
38
|
-
import { hostFor } from '../core/runtime/host.js';
|
|
53
|
+
import { listLivePanes, tearDownNode } from '../core/runtime/placement.js';
|
|
39
54
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
40
55
|
import { spawnChild } from '../core/runtime/spawn.js';
|
|
41
56
|
import { wakeOriginFrom } from '../core/runtime/bearings.js';
|
|
42
|
-
import { pushUrgent
|
|
57
|
+
import { pushUrgent } from '../core/feed/feed.js';
|
|
43
58
|
import { appendInbox, readInboxSince, readCursor } from '../core/feed/inbox.js';
|
|
44
59
|
import { nextSlotAfter } from '../core/wake.js';
|
|
45
60
|
/** Surface a vehicle that never booted.
|
|
@@ -62,54 +77,27 @@ async function surfaceBootFailure(meta) {
|
|
|
62
77
|
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`;
|
|
63
78
|
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
64
79
|
}
|
|
65
|
-
/** Wake an inbox-waiting PARENT when the daemon ends a child the child did NOT
|
|
66
|
-
* end itself — a mid-generation CRASH (the pane died inside a turn) or a quiet-
|
|
67
|
-
* turn FINALIZE (the pane was dismissed after the turn ended with nothing live
|
|
68
|
-
* to wait for). Mirrors surfaceBootFailure: a single push fanned to
|
|
69
|
-
* subscribersOf(child), so a purely-inbox-waiting parent is woken on the SAME
|
|
70
|
-
* channel as a `push final` — its live inbox-watcher, or this daemon's dormant-
|
|
71
|
-
* revive second pass. Without it a parent that delegated and just stopped hangs
|
|
72
|
-
* forever on these outcomes (D-1 finding: today only `push final` / never-booted
|
|
73
|
-
* wake it). The doctrine "delegate and stop; the runtime wakes you" requires
|
|
74
|
-
* EVERY terminal child outcome to reach the parent.
|
|
75
|
-
*
|
|
76
|
-
* Fires ONLY on genuine death — NEVER on healthy dormancy. The CALLER owns that
|
|
77
|
-
* boundary: this is invoked from the crash + finalize branches only; a child
|
|
78
|
-
* that ended its turn still awaiting a live grandchild OR holding a pending
|
|
79
|
-
* self-wake routes to `release` (revivable) and never reaches here. A CRASH is
|
|
80
|
-
* URGENT (a fault, like a boot failure); a quiet FINALIZE is a normal update (a
|
|
81
|
-
* clean dismissal — wake the dormant parent without interrupting a live one
|
|
82
|
-
* mid-turn). */
|
|
83
|
-
async function surfaceChildDeath(meta, cause) {
|
|
84
|
-
if (cause === 'crash') {
|
|
85
|
-
await pushUrgent(meta.node_id, `⚠ Child died — \`${meta.name}\` (${meta.kind}) was killed mid-task.\n\n` +
|
|
86
|
-
`Its pi vehicle went away mid-generation (the pane was closed or crashed while it was ` +
|
|
87
|
-
`working), so it never pushed a final report. Re-spawn it if the work still needs doing.`, { from: meta.node_id });
|
|
88
|
-
}
|
|
89
|
-
else {
|
|
90
|
-
await pushUpdate(meta.node_id, `\`${meta.name}\` (${meta.kind}) ended without a final report.\n\n` +
|
|
91
|
-
`Its turn ended and its pane was closed with nothing live left to wait for, so the runtime ` +
|
|
92
|
-
`finalized it. It pushed no \`final\` — check its reports/ for anything it left, and re-spawn ` +
|
|
93
|
-
`if the result is incomplete.`, { from: meta.node_id });
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
80
|
const DEFAULT_INTERVAL_MS = 2000;
|
|
97
|
-
// How long a node's
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
//
|
|
101
|
-
//
|
|
81
|
+
// How long a node's broker pid may be observed dead before the daemon revives
|
|
82
|
+
// it. MUST exceed worst-case broker boot time: a refresh / crash-revive
|
|
83
|
+
// transiently shows a dead OLD pid for the gap between the old broker exiting
|
|
84
|
+
// and the fresh broker booting + re-recording its pid, and we must not
|
|
85
|
+
// double-spawn into that gap.
|
|
102
86
|
const REVIVE_GRACE_MS = 20_000;
|
|
103
87
|
// Per-node first-observed-dead timestamps, for the grace window above. In-memory
|
|
104
88
|
// only — a daemon restart resets it (worst case: one extra grace interval).
|
|
105
89
|
const unhealthySince = new Map();
|
|
106
|
-
//
|
|
107
|
-
// absent) and its
|
|
108
|
-
// and kills the
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
90
|
+
// §H refresh-authority grace: how long a node may sit with intent='refresh', its
|
|
91
|
+
// turn OVER (busy marker absent) and its engine still ALIVE before the daemon
|
|
92
|
+
// concludes the refresh stalled and force-kills the engine. The healthy window is
|
|
93
|
+
// seconds: a sound stophook calls shutdown() the instant the refresh turn ends,
|
|
94
|
+
// the engine pid dies, and the dead-pid refresh path revives fresh. Minutes of
|
|
95
|
+
// that state means the engine never exited — either the shutdown/respawn path
|
|
96
|
+
// hung (observed at ~200k-token contexts) OR a STALE in-process stophook (pi
|
|
97
|
+
// extensions never reload, so a days-old resident carries an old hook that
|
|
98
|
+
// ignores intent='refresh' at stop) never called shutdown at all. Nothing else
|
|
99
|
+
// will ever unstick it — the daemon only revives DEAD engines — so the daemon is
|
|
100
|
+
// the authority: kill the engine here, then the dead-pid path revives.
|
|
113
101
|
export const YIELD_STALL_GRACE_MS = 3 * 60_000;
|
|
114
102
|
// After a yield-stall SIGTERM, how long the pi gets to die gracefully before
|
|
115
103
|
// escalating to SIGKILL (a pi hung at context overflow may not run handlers).
|
|
@@ -125,58 +113,21 @@ const yieldTermAt = new Map();
|
|
|
125
113
|
// only, like unhealthySince; a daemon restart resets it (worst case: one repeat
|
|
126
114
|
// notice). A successful spawn clears its entry so a future failure re-notifies.
|
|
127
115
|
const notifiedWakeFailures = new Set();
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
* forever; the parent waited on a node that was never going to run. Pure core:
|
|
139
|
-
* a node that has ever booted (session recorded) or ever been revived (cycles
|
|
140
|
-
* bumped) is NOT never-launched — the cycles gate also bounds this to ONE
|
|
141
|
-
* automatic retry (the relaunch bumps cycles), so a persistently failing
|
|
142
|
-
* launch can't loop. Age below the grace is a normal in-flight boot. */
|
|
143
|
-
export function neverLaunchedVerdict(piSessionId, cycles, ageMs) {
|
|
144
|
-
if (piSessionId != null || cycles > 0)
|
|
145
|
-
return 'leave';
|
|
146
|
-
return ageMs >= NEVER_LAUNCHED_GRACE_MS ? 'relaunch' : 'leave';
|
|
147
|
-
}
|
|
148
|
-
/** Enact the never-launched relaunch for a pane-alive, pid-less node (Bug 4):
|
|
149
|
-
* verify against meta (pi_session_id is identity, read on demand), reap the
|
|
150
|
-
* stale spawn window, and relaunch fresh via reviveNode — whose kickoff
|
|
151
|
-
* carries the node's goal (context/initial-prompt.md) when no roadmap exists,
|
|
152
|
-
* so the relaunched worker receives its original mandate. */
|
|
153
|
-
function handleNeverLaunched(row, now, revivedThisTick) {
|
|
154
|
-
const age = now - Date.parse(row.created); // NaN compares false → leave
|
|
155
|
-
if (!(age >= NEVER_LAUNCHED_GRACE_MS))
|
|
156
|
-
return;
|
|
157
|
-
const meta = getNode(row.node_id);
|
|
158
|
-
if (meta === null)
|
|
159
|
-
return;
|
|
160
|
-
if (neverLaunchedVerdict(meta.pi_session_id ?? null, meta.cycles ?? 0, age) !== 'relaunch') {
|
|
161
|
-
return;
|
|
162
|
-
}
|
|
163
|
-
process.stderr.write(`[crtrd] relaunch ${row.node_id} (spawned but never launched: 0 cycles, no session, age ${Math.round(age / 1000)}s)\n`);
|
|
164
|
-
tearDownNode(row.node_id); // reap the stale spawn pane — reviveNode opens a fresh window
|
|
165
|
-
reviveNode(row.node_id, { resume: false });
|
|
166
|
-
revivedThisTick.add(row.node_id);
|
|
167
|
-
}
|
|
168
|
-
/** Yield-stall decision (audit 2026-06-09, Bug 1 — node mq5v9hfa-74493d45 and
|
|
169
|
-
* three sibling orchestrators): a `node yield` records intent='refresh' and
|
|
170
|
-
* agent_end dispatches the in-place respawn, but at very large contexts the pi
|
|
171
|
-
* process can HANG instead of exiting — leaving pi alive, the turn over, and
|
|
172
|
-
* the intent pending forever. The daemon only revives DEAD pis, so without
|
|
173
|
-
* this verdict the node is permanently stuck (no revive, no wake). Pure core,
|
|
174
|
-
* mirroring livenessVerdict; the kill side effects live in handleYieldStall.
|
|
116
|
+
/** Refresh-stall decision (§H refresh-authority; audit 2026-06-09, Bug 1 — node
|
|
117
|
+
* mq5v9hfa-74493d45 and siblings; stuck-refresh — parent mq7s5o93): a `node
|
|
118
|
+
* yield` / refresh records intent='refresh' and relies on the in-process
|
|
119
|
+
* stophook to call shutdown at stop, but the engine can be left ALIVE — either
|
|
120
|
+
* it HANGS instead of exiting (very large contexts) or a STALE stophook (pi
|
|
121
|
+
* extensions never reload) ignores intent='refresh' entirely and never calls
|
|
122
|
+
* shutdown. Either way: engine alive, turn over, intent pending forever. The
|
|
123
|
+
* daemon only revives DEAD engines, so without this verdict the node is
|
|
124
|
+
* permanently stuck (no revive, no wake). Pure core, mirroring livenessVerdict;
|
|
125
|
+
* the kill side effects live in handleYieldStall.
|
|
175
126
|
*
|
|
176
|
-
* 'kill' ONLY when ALL of:
|
|
177
|
-
* marker absent — a node may legitimately keep working for many minutes
|
|
178
|
-
* running `node yield` mid-turn, and a working
|
|
179
|
-
* the state has persisted past YIELD_STALL_GRACE_MS. */
|
|
127
|
+
* 'kill' ONLY when ALL of: engine alive, intent='refresh', the turn is over
|
|
128
|
+
* (busy marker absent — a node may legitimately keep working for many minutes
|
|
129
|
+
* after running `node yield` mid-turn, and a working engine must never be
|
|
130
|
+
* killed), and the state has persisted past YIELD_STALL_GRACE_MS. */
|
|
180
131
|
export function yieldStallVerdict(piPidAlive, intent, busy, stalledFor) {
|
|
181
132
|
if (piPidAlive !== true)
|
|
182
133
|
return 'leave';
|
|
@@ -188,14 +139,16 @@ export function yieldStallVerdict(piPidAlive, intent, busy, stalledFor) {
|
|
|
188
139
|
return 'pending';
|
|
189
140
|
return 'kill';
|
|
190
141
|
}
|
|
191
|
-
/** Enact a
|
|
192
|
-
*
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
142
|
+
/** Enact a refresh-stall (§H refresh-authority): clock the stalled state, then
|
|
143
|
+
* SIGTERM the hung engine (escalating to SIGKILL if it survives
|
|
144
|
+
* KILL_ESCALATE_MS — an engine hung at context overflow, or a stale hook that
|
|
145
|
+
* never wired shutdown, may never run its handlers, so a graceful frame can't be
|
|
146
|
+
* trusted; the signal escalation is what guarantees the engine actually dies).
|
|
147
|
+
* Once the engine is dead the ORDINARY machinery finishes the job: the dead-pid
|
|
148
|
+
* refresh branch in handleNodeLiveness revives with resume=false (intent is
|
|
149
|
+
* still 'refresh'), i.e. the fresh roadmap-revive the yield asked for. Safe
|
|
150
|
+
* against a graceful shutdown racing us: markCleanExitDone no-ops while intent
|
|
151
|
+
* is pending, so a dying engine can't mis-finalize the node. */
|
|
199
152
|
function handleYieldStall(row, pid, now) {
|
|
200
153
|
const id = row.node_id;
|
|
201
154
|
if (row.intent !== 'refresh' || isBusy(id)) {
|
|
@@ -231,124 +184,219 @@ function handleYieldStall(row, pid, now) {
|
|
|
231
184
|
}
|
|
232
185
|
}
|
|
233
186
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
export
|
|
241
|
-
|
|
187
|
+
// §I error-stall grace: how long a node's session .jsonl must have been QUIET
|
|
188
|
+
// (no append — mtime is the clock, so it survives daemon restarts) before a
|
|
189
|
+
// trailing-error engine is declared wedged and killed. Generous enough that an
|
|
190
|
+
// in-flight recovery (the human re-steering, a parent's message waking a new
|
|
191
|
+
// turn via the live inbox-watcher — both of which append and reset the clock)
|
|
192
|
+
// always wins, while a true zombie recovers in minutes instead of never.
|
|
193
|
+
export const ERROR_STALL_QUIET_MS = 5 * 60_000;
|
|
194
|
+
// Per-node SIGTERM timestamps for §I kill escalation, mirroring yieldTermAt.
|
|
195
|
+
const errorTermAt = new Map();
|
|
196
|
+
// Memoized tail inspection per node, keyed on the session file's mtime so the
|
|
197
|
+
// (rare) 64KB tail read happens once per file change, not once per 2s tick.
|
|
198
|
+
const errorTailMemo = new Map();
|
|
199
|
+
const TAIL_BYTES = 64 * 1024;
|
|
200
|
+
/** Read the last TAIL_BYTES of a file. null on any failure (absent file reads
|
|
201
|
+
* as "nothing to inspect", never as evidence of a wedge). */
|
|
202
|
+
function readTail(path) {
|
|
203
|
+
try {
|
|
204
|
+
const size = statSync(path).size;
|
|
205
|
+
const len = Math.min(TAIL_BYTES, size);
|
|
206
|
+
if (len === 0)
|
|
207
|
+
return null;
|
|
208
|
+
const fd = openSync(path, 'r');
|
|
209
|
+
try {
|
|
210
|
+
const buf = Buffer.alloc(len);
|
|
211
|
+
readSync(fd, buf, 0, len, size - len);
|
|
212
|
+
return buf.toString('utf8');
|
|
213
|
+
}
|
|
214
|
+
finally {
|
|
215
|
+
closeSync(fd);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/** The §I wedge signature, from a session-jsonl tail: the LAST assistant
|
|
223
|
+
* message stopped with 'error' (retry budget exhausted — pi appends the error
|
|
224
|
+
* turn and parks). Returns its errorMessage for the daemon log (the "why it
|
|
225
|
+
* wedged" trace), or null when the last assistant turn ended any other way —
|
|
226
|
+
* 'stop'/'length' is a healthy park, 'aborted' is a human Esc, and trailing
|
|
227
|
+
* non-assistant lines (toolResults, an injected user message) are walked past.
|
|
228
|
+
* The first line of a mid-file tail cut fails JSON.parse and is skipped. */
|
|
229
|
+
export function trailingEngineError(tail) {
|
|
230
|
+
if (tail === null)
|
|
231
|
+
return null;
|
|
232
|
+
const lines = tail.split('\n');
|
|
233
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
234
|
+
const line = lines[i].trim();
|
|
235
|
+
if (line === '')
|
|
236
|
+
continue;
|
|
237
|
+
let msg;
|
|
238
|
+
try {
|
|
239
|
+
msg = JSON.parse(line)?.message;
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
continue; // a cut fragment (tail boundary) or non-JSON line
|
|
243
|
+
}
|
|
244
|
+
if (msg?.role !== 'assistant')
|
|
245
|
+
continue;
|
|
246
|
+
if (msg.stopReason !== 'error')
|
|
247
|
+
return null;
|
|
248
|
+
return typeof msg.errorMessage === 'string' && msg.errorMessage !== ''
|
|
249
|
+
? msg.errorMessage
|
|
250
|
+
: 'engine error (no errorMessage recorded)';
|
|
251
|
+
}
|
|
252
|
+
return null;
|
|
253
|
+
}
|
|
254
|
+
/** Error-stall decision (§I; gh issue #4 — node mqaeqzdu-0746b3b7, a 9h zombie
|
|
255
|
+
* after an Anthropic outage). Pure, mirroring yieldStallVerdict; the kill side
|
|
256
|
+
* effects live in handleErrorStall.
|
|
257
|
+
*
|
|
258
|
+
* 'kill' ONLY when ALL of: engine alive, NO pending intent (refresh is §H's
|
|
259
|
+
* domain; idle-release chose dormancy), the turn is over (busy absent — a node
|
|
260
|
+
* wedged mid-bash must be recovered by killing the subprocess, never the node),
|
|
261
|
+
* the session file has been quiet past ERROR_STALL_QUIET_MS, and its last
|
|
262
|
+
* assistant message stopped with 'error'. */
|
|
263
|
+
export function errorStallVerdict(piPidAlive, intent, busy, quietForMs, trailingError) {
|
|
264
|
+
if (piPidAlive !== true)
|
|
242
265
|
return 'leave';
|
|
243
|
-
if (
|
|
244
|
-
return '
|
|
245
|
-
|
|
266
|
+
if (intent !== null)
|
|
267
|
+
return 'leave';
|
|
268
|
+
if (busy)
|
|
269
|
+
return 'leave';
|
|
270
|
+
if (quietForMs === null || quietForMs < ERROR_STALL_QUIET_MS)
|
|
271
|
+
return 'leave';
|
|
272
|
+
if (!trailingError)
|
|
273
|
+
return 'leave';
|
|
274
|
+
return 'kill';
|
|
246
275
|
}
|
|
247
|
-
/**
|
|
248
|
-
*
|
|
249
|
-
*
|
|
250
|
-
*
|
|
251
|
-
|
|
276
|
+
/** Enact an error-stall (§I): when the verdict is 'kill', SIGTERM the parked
|
|
277
|
+
* engine (escalating to SIGKILL after KILL_ESCALATE_MS, like §H — a broker
|
|
278
|
+
* wedged after an outage may not run graceful handlers). intent is null, so
|
|
279
|
+
* once the pid dies the ordinary crash branch in handleNodeLiveness grace-
|
|
280
|
+
* revives RESUME on the saved session — the conversation is preserved and the
|
|
281
|
+
* revive kickoff prompts the node to continue. No grace clock of our own: the
|
|
282
|
+
* session file's mtime IS the clock (any append — a re-steer, an inbox-watcher
|
|
283
|
+
* wake, a new turn — resets it). */
|
|
284
|
+
function handleErrorStall(row, pid, now) {
|
|
252
285
|
const id = row.node_id;
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
// inbox push (the second pass below); grace-reviving would pre-empt that and
|
|
256
|
-
// churn the pane. (Focused-await nodes now stay pi-LIVE instead of releasing —
|
|
257
|
-
// see the stophook awaiting branch — so reaching this with a live pane is rare;
|
|
258
|
-
// the guard stays as a cheap safety net, e.g. a remain-on-exit pane frozen for
|
|
259
|
-
// a done-node's manager handoff.)
|
|
260
|
-
if (row.intent === 'idle-release') {
|
|
261
|
-
unhealthySince.delete(id);
|
|
262
|
-
yieldStallSince.delete(id);
|
|
263
|
-
yieldTermAt.delete(id);
|
|
286
|
+
if (row.intent !== null || isBusy(id)) {
|
|
287
|
+
errorTermAt.delete(id);
|
|
264
288
|
return;
|
|
265
289
|
}
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
if (piPidAlive !== false) {
|
|
269
|
-
unhealthySince.delete(id); // alive, or no pid to judge — nothing pending
|
|
270
|
-
if (piPidAlive === null) {
|
|
271
|
-
// No pid recorded: a legacy row, a relaunch in flight — or a spawn whose
|
|
272
|
-
// pi NEVER came up while its window lingers (Bug 4). Judge the last case.
|
|
273
|
-
handleNeverLaunched(row, now, revivedThisTick);
|
|
274
|
-
}
|
|
275
|
-
else {
|
|
276
|
-
// pi ALIVE: judge a hung refresh-yield (Bug 1) — alive is not healthy
|
|
277
|
-
// when the node yielded, its turn ended, and the pi never exited.
|
|
278
|
-
handleYieldStall(row, pid, now);
|
|
279
|
-
}
|
|
290
|
+
const file = getNode(id)?.pi_session_file;
|
|
291
|
+
if (file == null)
|
|
280
292
|
return;
|
|
293
|
+
let mtimeMs;
|
|
294
|
+
try {
|
|
295
|
+
mtimeMs = statSync(file).mtimeMs;
|
|
281
296
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
if (verdict === 'pending') {
|
|
289
|
-
if (since === undefined)
|
|
290
|
-
unhealthySince.set(id, now);
|
|
297
|
+
catch {
|
|
298
|
+
return; // session file unreadable — no evidence, never kill on "can't tell"
|
|
299
|
+
}
|
|
300
|
+
const quietFor = now - mtimeMs;
|
|
301
|
+
if (quietFor < ERROR_STALL_QUIET_MS) {
|
|
302
|
+
errorTermAt.delete(id);
|
|
291
303
|
return;
|
|
292
304
|
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
305
|
+
let memo = errorTailMemo.get(id);
|
|
306
|
+
if (memo === undefined || memo.mtimeMs !== mtimeMs) {
|
|
307
|
+
memo = { mtimeMs, error: trailingEngineError(readTail(file)) };
|
|
308
|
+
errorTailMemo.set(id, memo);
|
|
309
|
+
}
|
|
310
|
+
if (errorStallVerdict(true, row.intent, false, quietFor, memo.error !== null) !== 'kill')
|
|
311
|
+
return;
|
|
312
|
+
const termed = errorTermAt.get(id);
|
|
313
|
+
if (termed === undefined) {
|
|
314
|
+
process.stderr.write(`[crtrd] kill ${id} (error-stall: engine parked on a trailing error turn, session quiet ${Math.round(quietFor / 1000)}s — SIGTERM; last error: ${String(memo.error).slice(0, 200)})\n`);
|
|
315
|
+
try {
|
|
316
|
+
process.kill(pid, 'SIGTERM');
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
/* already gone */
|
|
320
|
+
}
|
|
321
|
+
errorTermAt.set(id, now);
|
|
322
|
+
}
|
|
323
|
+
else if (now - termed >= KILL_ESCALATE_MS) {
|
|
324
|
+
process.stderr.write(`[crtrd] kill ${id} (error-stall: pi ${pid} survived SIGTERM — SIGKILL)\n`);
|
|
325
|
+
try {
|
|
326
|
+
process.kill(pid, 'SIGKILL');
|
|
327
|
+
}
|
|
328
|
+
catch {
|
|
329
|
+
/* already gone */
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
/** Decide what to do with a node whose engine pid is DEAD, from how long it's
|
|
334
|
+
* been dead. Pure — the time/revive side effects live in handleNodeLiveness;
|
|
335
|
+
* this is the unit-testable core. (piPidAlive carries dead/alive/unknown, but
|
|
336
|
+
* only `false` ever reaches a revive decision here — alive and null are routed
|
|
337
|
+
* upstream by handleNodeLiveness.)
|
|
338
|
+
* deadFor: ms since first observed dead, or null on the first observation. */
|
|
339
|
+
export function livenessVerdict(piPidAlive, deadFor) {
|
|
340
|
+
if (piPidAlive !== false)
|
|
341
|
+
return 'leave';
|
|
342
|
+
if (deadFor === null || deadFor < REVIVE_GRACE_MS)
|
|
343
|
+
return 'pending';
|
|
344
|
+
return 'revive';
|
|
307
345
|
}
|
|
308
|
-
/**
|
|
309
|
-
*
|
|
310
|
-
*
|
|
311
|
-
*
|
|
312
|
-
*
|
|
313
|
-
* unhealthySince + revivedThisTick — NO new machinery (decision §1.7 / R3).
|
|
346
|
+
/** The ONE liveness path for EVERY node. Every node runs on a detached headless
|
|
347
|
+
* broker; a tmux pane is only a viewer and is never consulted here — liveness is
|
|
348
|
+
* the recorded engine pid (signal-0) alone. Reuses the existing supervision
|
|
349
|
+
* primitives, now universal: pid signal-0 + REVIVE_GRACE_MS grace +
|
|
350
|
+
* unhealthySince + revivedThisTick. A viewer window closing is NOT a node death.
|
|
314
351
|
* Cases:
|
|
315
|
-
* • pid alive → leave; clear any stale grace timer.
|
|
352
|
+
* • pid alive → leave; clear any stale grace timer. THEN enforce a stuck
|
|
353
|
+
* refresh (§H): handleYieldStall self-guards (no-op unless intent='refresh'
|
|
354
|
+
* with the turn over) and, once the engine has sat alive past the grace,
|
|
355
|
+
* kills it — so the daemon, not the (possibly stale) in-process stophook, is
|
|
356
|
+
* the authority on refresh, and the dead-pid refresh branch below then
|
|
357
|
+
* revives it fresh. AND enforce §I: handleErrorStall self-guards (no-op
|
|
358
|
+
* unless intent is null with the turn over) and kills an engine parked on a
|
|
359
|
+
* trailing-error turn whose session has been quiet past the grace, so the
|
|
360
|
+
* crash branch below grace-revives it RESUME.
|
|
316
361
|
* • pid null + pi_session_id set (a relaunch in flight — reviveNode clears
|
|
317
|
-
* pi_pid right after launch) → leave; the fresh
|
|
362
|
+
* pi_pid right after launch) → leave; the fresh engine re-records its pid.
|
|
318
363
|
* • pid null + pi_session_id null (NEVER booted) → normally the sub-second SDK
|
|
319
364
|
* boot gap, but a broker that throws BEFORE session_start records no pid and
|
|
320
365
|
* no session ever — so after a boot grace with STILL nothing, crash +
|
|
321
|
-
* surfaceBootFailure
|
|
322
|
-
* • pid dead + intent==='refresh' → clean yield
|
|
323
|
-
*
|
|
366
|
+
* surfaceBootFailure up the spine (M-1).
|
|
367
|
+
* • pid dead + intent==='refresh' → clean yield (or the §H force-kill above
|
|
368
|
+
* just landed): respawn FRESH immediately, no grace wait.
|
|
324
369
|
* • pid dead + intent==='idle-release' → dormant by choice; leave, the second
|
|
325
370
|
* pass revives (resume) on the next unseen inbox entry.
|
|
326
|
-
* • pid dead + any other intent (crash
|
|
327
|
-
*
|
|
328
|
-
|
|
329
|
-
* already handled above), so it resumes — no explicit resume flag needed. */
|
|
330
|
-
async function handleBrokerLiveness(row, now, revivedThisTick) {
|
|
371
|
+
* • pid dead + any other intent (a crash) → grace-revive RESUME on the saved
|
|
372
|
+
* session (livenessVerdict → REVIVE_GRACE_MS → unhealthySince). */
|
|
373
|
+
async function handleNodeLiveness(row, now, revivedThisTick) {
|
|
331
374
|
const id = row.node_id;
|
|
332
375
|
const pid = row.pi_pid;
|
|
333
|
-
// The
|
|
376
|
+
// The engine is live → nothing pending; clear any boot/grace timer. THEN
|
|
377
|
+
// enforce §H: handleYieldStall no-ops unless this node yielded (intent=refresh),
|
|
378
|
+
// its turn is over, and the engine never exited — past the grace it kills the
|
|
379
|
+
// engine so the dead-pid refresh branch below revives it fresh.
|
|
334
380
|
if (pid != null && isPidAlive(pid)) {
|
|
335
381
|
unhealthySince.delete(id);
|
|
382
|
+
handleYieldStall(row, pid, now);
|
|
383
|
+
handleErrorStall(row, pid, now);
|
|
336
384
|
return;
|
|
337
385
|
}
|
|
338
386
|
if (pid == null) {
|
|
339
387
|
// No supervised pid recorded. Two very different cases turn on pi_session_id:
|
|
340
388
|
// • a relaunch in flight — reviveNode clears pi_pid right after launch, but
|
|
341
389
|
// the node ALREADY booted once (pi_session_id captured), so the fresh
|
|
342
|
-
//
|
|
390
|
+
// engine re-records its pid within a tick or two; leave it.
|
|
343
391
|
// • a NEVER-BOOTED broker (pi_session_id null) — normally the sub-second SDK
|
|
344
392
|
// boot gap, BUT a broker that THROWS before session_start (malformed
|
|
345
393
|
// broker-launch.json, SessionManager.open on a missing .jsonl, a loader/
|
|
346
|
-
// registry/createAgentSession failure, the
|
|
394
|
+
// registry/createAgentSession failure, the fork / bare-id guards, or
|
|
347
395
|
// broker-cli's own fatal catch) records NO pid and NO session — EVER.
|
|
348
396
|
// With pid==null read unconditionally as "still booting" that strands the
|
|
349
397
|
// node 'active' with no engine forever and its parent waits on a dead
|
|
350
|
-
// child.
|
|
351
|
-
//
|
|
398
|
+
// child. After a boot grace with STILL no pid AND no session, crash +
|
|
399
|
+
// surfaceBootFailure up the spine (M-1).
|
|
352
400
|
const meta = getNode(id);
|
|
353
401
|
if (meta === null || meta.pi_session_id != null) {
|
|
354
402
|
unhealthySince.delete(id); // relaunch in flight (or identity already bound)
|
|
@@ -373,10 +421,16 @@ async function handleBrokerLiveness(row, now, revivedThisTick) {
|
|
|
373
421
|
}
|
|
374
422
|
return;
|
|
375
423
|
}
|
|
376
|
-
// The
|
|
424
|
+
// The engine pid is DEAD — any pending refresh/error-stall clock is moot (the
|
|
425
|
+
// dead-pid paths below own recovery now). Branch on intent.
|
|
426
|
+
yieldStallSince.delete(id);
|
|
427
|
+
yieldTermAt.delete(id);
|
|
428
|
+
errorTermAt.delete(id);
|
|
429
|
+
errorTailMemo.delete(id);
|
|
377
430
|
if (row.intent === 'refresh') {
|
|
378
|
-
// Clean yield → respawn FRESH,
|
|
379
|
-
|
|
431
|
+
// Clean yield — or the §H force-kill above just landed — → respawn FRESH,
|
|
432
|
+
// immediately (no grace wait).
|
|
433
|
+
process.stderr.write(`[crtrd] revive ${id} (refresh-yield)\n`);
|
|
380
434
|
reviveNode(id, { resume: false });
|
|
381
435
|
revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
|
|
382
436
|
return;
|
|
@@ -387,10 +441,20 @@ async function handleBrokerLiveness(row, now, revivedThisTick) {
|
|
|
387
441
|
unhealthySince.delete(id);
|
|
388
442
|
return;
|
|
389
443
|
}
|
|
390
|
-
// Any other intent → grace-revive RESUME
|
|
391
|
-
//
|
|
392
|
-
//
|
|
393
|
-
|
|
444
|
+
// Any other intent → a crash: grace-revive RESUME on the saved session.
|
|
445
|
+
// reviveNode clears pi_pid until the fresh engine re-records it, so the next
|
|
446
|
+
// tick won't re-fire on this stale pid.
|
|
447
|
+
const since = unhealthySince.get(id);
|
|
448
|
+
const verdict = livenessVerdict(false, since === undefined ? null : now - since);
|
|
449
|
+
if (verdict === 'pending') {
|
|
450
|
+
if (since === undefined)
|
|
451
|
+
unhealthySince.set(id, now);
|
|
452
|
+
return;
|
|
453
|
+
}
|
|
454
|
+
unhealthySince.delete(id);
|
|
455
|
+
process.stderr.write(`[crtrd] revive ${id} (engine dead, intent=${String(row.intent)})\n`);
|
|
456
|
+
reviveNode(id, { resume: true });
|
|
457
|
+
revivedThisTick.add(id); // third-pass bare double-spawn guard (Maj-4)
|
|
394
458
|
}
|
|
395
459
|
/** Fail loud for a drifted/broken wake (design §6.6/Q5): wake the ARMER DIRECTLY
|
|
396
460
|
* via appendInbox — NOT pushUrgent, which fans to subscribersOf(owner), and a
|
|
@@ -560,162 +624,27 @@ export async function superviseTick(now = Date.now()) {
|
|
|
560
624
|
process.stderr.write(`[crtrd] listNodes error: ${err.message}\n`);
|
|
561
625
|
return;
|
|
562
626
|
}
|
|
627
|
+
// kind:'human' rows are NOT broker-hosted agent nodes — they are bookkeeping
|
|
628
|
+
// rows for the `crtr human` bridge. The decision/review TUI runs in a detached
|
|
629
|
+
// `crtr human _run` pane (NOT a pi engine), and the row's lifecycle is driven
|
|
630
|
+
// entirely by that worker's `pushFinal` (or `human cancel`). They never record
|
|
631
|
+
// a `pi_pid` or `pi_session_id`, so the universal broker-liveness path would
|
|
632
|
+
// (after the boot grace) read every one as a never-booted broker and crash it
|
|
633
|
+
// with surfaceBootFailure — exactly the regression where a headless
|
|
634
|
+
// orchestrator's `human ask` nodes all died at boot. The daemon must never
|
|
635
|
+
// supervise them: drop them from ALL three passes here.
|
|
636
|
+
rows = rows.filter((r) => r.kind !== 'human');
|
|
563
637
|
// Node-ids revived in pass 1 + pass 2 THIS tick — the third pass's bare branch
|
|
564
638
|
// skips them so it never launches a second pi on the same .jsonl (Maj-4).
|
|
565
639
|
const revivedThisTick = new Set();
|
|
566
640
|
for (const row of rows) {
|
|
567
641
|
try {
|
|
568
|
-
//
|
|
569
|
-
//
|
|
570
|
-
//
|
|
571
|
-
//
|
|
572
|
-
// pane-gone
|
|
573
|
-
|
|
574
|
-
// (decision §1.6) — no edit to the carve-out is needed.
|
|
575
|
-
if (row.host_kind === 'broker') {
|
|
576
|
-
await handleBrokerLiveness(row, now, revivedThisTick);
|
|
577
|
-
continue;
|
|
578
|
-
}
|
|
579
|
-
// Runtime (tmux_session, window, intent, pi_pid) is now authoritative IN
|
|
580
|
-
// the row — no per-node getNode re-read. Only the boot-failure split below
|
|
581
|
-
// still needs identity (pi_session_id), read on demand there.
|
|
582
|
-
// Nodes with no tmux placement at all are inline roots — not daemon-
|
|
583
|
-
// managed. Pane-anchored: a node still counts as placed if it has a pane
|
|
584
|
-
// even when its derived window/session cache is null.
|
|
585
|
-
if (row.tmux_session == null && row.window == null && row.pane == null)
|
|
586
|
-
continue;
|
|
587
|
-
if (hostFor(row).isAlive(row)) {
|
|
588
|
-
// The pane is up — but that alone doesn't mean pi is. Reconcile first
|
|
589
|
-
// (follow any manual pane move, and lazy-backfill a legacy row's pane
|
|
590
|
-
// from its live window), then judge pi liveness off the fresh row. The
|
|
591
|
-
// alive-gate means reconcile here only ever FOLLOWS/backfills — never
|
|
592
|
-
// nulls the LOCATION out from under the gone-branches below.
|
|
593
|
-
reconcile(row.node_id);
|
|
594
|
-
handleLiveWindow(getRow(row.node_id) ?? row, now, revivedThisTick);
|
|
595
|
-
continue;
|
|
596
|
-
}
|
|
597
|
-
// The pane is gone. Branch on why.
|
|
598
|
-
unhealthySince.delete(row.node_id); // pane-gone path owns it now
|
|
599
|
-
yieldStallSince.delete(row.node_id);
|
|
600
|
-
yieldTermAt.delete(row.node_id);
|
|
601
|
-
// Zombie vehicle (audit 2026-06-09, Bug 2 — node mq4ms6n4-fbc455cd): the
|
|
602
|
-
// pane died but its pi SURVIVED (pi can outlive the pane's SIGHUP). Every
|
|
603
|
-
// branch below assumes "pane gone ⇒ pi gone"; with a live orphan that
|
|
604
|
-
// assumption corrupts — the headless pi keeps re-arming its inbox watcher
|
|
605
|
-
// with no pane a human can reach, the second pass's pi-liveness gate skips
|
|
606
|
-
// a released node forever, and a refresh-revive would run a second pi
|
|
607
|
-
// beside it. There is no legitimate pane-gone+pi-alive state (in-place
|
|
608
|
-
// respawns keep the pane id; teardown paths null presence before reaping),
|
|
609
|
-
// so kill the orphan first, then route normally. SIGKILL, not SIGTERM: an
|
|
610
|
-
// orphaned pi's shutdown hooks mutating canvas state mid-routing is
|
|
611
|
-
// exactly what we're fencing out.
|
|
612
|
-
if (row.pi_pid != null && isPidAlive(row.pi_pid)) {
|
|
613
|
-
process.stderr.write(`[crtrd] kill ${row.node_id} (zombie pi ${row.pi_pid}: pane gone, pi alive)\n`);
|
|
614
|
-
try {
|
|
615
|
-
process.kill(row.pi_pid, 'SIGKILL');
|
|
616
|
-
}
|
|
617
|
-
catch {
|
|
618
|
-
/* raced its own death — fine */
|
|
619
|
-
}
|
|
620
|
-
}
|
|
621
|
-
if (row.intent === 'refresh') {
|
|
622
|
-
// The node set intent=refresh before stopping — a clean yield. Respawn
|
|
623
|
-
// fresh so it re-reads its roadmap/context dir.
|
|
624
|
-
process.stderr.write(`[crtrd] revive ${row.node_id} (refresh-yield)\n`);
|
|
625
|
-
reviveNode(row.node_id, { resume: false });
|
|
626
|
-
revivedThisTick.add(row.node_id); // third-pass bare double-spawn guard (Maj-4)
|
|
627
|
-
}
|
|
628
|
-
else if (row.intent === 'idle-release') {
|
|
629
|
-
// The node freed its own window on purpose while dormant. Drop the stale
|
|
630
|
-
// window ref and keep it 'idle'; the inbox-poll pass below revives it
|
|
631
|
-
// (resume) the moment a subscribed worker delivers.
|
|
632
|
-
setPresence(row.node_id, { tmux_session: row.tmux_session, window: null });
|
|
633
|
-
}
|
|
634
|
-
else {
|
|
635
|
-
// The pane vanished without the node yielding or releasing — most often
|
|
636
|
-
// the user CLOSED it (kill-pane/kill-window), which crtr cannot tell apart
|
|
637
|
-
// from a window death. Closing a pane is a benign "get it off my screen",
|
|
638
|
-
// NOT an orphan-the-work kill, so route on what the node was DOING and keep
|
|
639
|
-
// anything with outstanding work REVIVABLE:
|
|
640
|
-
// • never-booted (pi_session_id null) → crash + surface boot failure.
|
|
641
|
-
// A spawn failure the parent was never told about — it had no turn to
|
|
642
|
-
// finish, so it can never be a finalize. (Boot-failed vs crashed turns
|
|
643
|
-
// on pi_session_id, an IDENTITY field — the one place this pass still
|
|
644
|
-
// reads meta; surfaceBootFailure also wants name/kind for its message.)
|
|
645
|
-
// • MID-GENERATION (busy marker present) → crash (→dead). agent_start
|
|
646
|
-
// touched the marker and agent_end never cleared it ⇒ the pane was
|
|
647
|
-
// killed inside a turn: a genuine mid-run death. (The pane is gone, so
|
|
648
|
-
// pi is dead; we read isBusy WITHOUT the usual AND-pidAlive guard on
|
|
649
|
-
// purpose — here a stale marker IS the proof it died mid-turn.)
|
|
650
|
-
// • finished its turn (busy ABSENT) but STILL WAITING — awaiting a LIVE
|
|
651
|
-
// child OR holding a pending self-wake → RELEASE (→idle + idle-release),
|
|
652
|
-
// NOT dead, and NO parent wake. This is HEALTHY DORMANCY (the same
|
|
653
|
-
// boundary the stop-guard draws): real orchestration / a scheduled
|
|
654
|
-
// clock is outstanding; killing it would orphan in-flight work, and
|
|
655
|
-
// waking its parent here would re-create the spurious-wake storm the
|
|
656
|
-
// wake doctrine exists to kill. Drop the stale window; the second /
|
|
657
|
-
// wakeups pass revives it on the next child push or clock fire.
|
|
658
|
-
// • finished its turn AND nothing live to wait for AND no pending clock →
|
|
659
|
-
// finalize (→done): it did its own work and the pane was closed to
|
|
660
|
-
// dismiss it. GENUINE death → wake the inbox-waiting parent.
|
|
661
|
-
const meta = getNode(row.node_id);
|
|
662
|
-
const neverBooted = meta !== null && meta.pi_session_id == null;
|
|
663
|
-
const finishedTurn = !neverBooted && !isBusy(row.node_id);
|
|
664
|
-
// The death-vs-dormancy boundary, drawn EXACTLY where the stop-guard draws
|
|
665
|
-
// it (stop-guard.ts): a live active subscription OR a pending self-wake is
|
|
666
|
-
// a legitimate wait. Only a finished node with NEITHER is genuinely done.
|
|
667
|
-
const stillWaiting = hasActiveLiveSubscription(row.node_id) || hasPendingSelfWake(row.node_id);
|
|
668
|
-
if (finishedTurn && !stillWaiting) {
|
|
669
|
-
transition(row.node_id, 'finalize');
|
|
670
|
-
process.stderr.write(`[crtrd] done ${row.node_id} (pane gone after turn end, nothing live to wait for)\n`);
|
|
671
|
-
// Wake the inbox-waiting parent: the child ended without a `push final`
|
|
672
|
-
// and was dismissed; without this the daemon-finalize reaches no one
|
|
673
|
-
// (D-1) and a purely-inbox-waiting parent hangs forever.
|
|
674
|
-
if (meta !== null) {
|
|
675
|
-
try {
|
|
676
|
-
await surfaceChildDeath(meta, 'finalize');
|
|
677
|
-
}
|
|
678
|
-
catch (err) {
|
|
679
|
-
process.stderr.write(`[crtrd] surfaceChildDeath(finalize) ${row.node_id} error: ${err.message}\n`);
|
|
680
|
-
}
|
|
681
|
-
}
|
|
682
|
-
}
|
|
683
|
-
else if (finishedTurn) {
|
|
684
|
-
// Still legitimately waiting (a live child OR a pending self-wake) →
|
|
685
|
-
// revivable, NOT a death: closing the pane must not orphan in-flight
|
|
686
|
-
// work, and healthy dormancy must NOT wake the parent. Clear the stale
|
|
687
|
-
// window; the second / wakeups pass revives it.
|
|
688
|
-
setPresence(row.node_id, { tmux_session: row.tmux_session, window: null, pane: null });
|
|
689
|
-
transition(row.node_id, 'release');
|
|
690
|
-
process.stderr.write(`[crtrd] release ${row.node_id} (pane gone while still waiting → revivable, no parent wake)\n`);
|
|
691
|
-
}
|
|
692
|
-
else {
|
|
693
|
-
transition(row.node_id, 'crash');
|
|
694
|
-
if (neverBooted) {
|
|
695
|
-
process.stderr.write(`[crtrd] boot-failed ${row.node_id} (pane gone before pi ever started)\n`);
|
|
696
|
-
try {
|
|
697
|
-
await surfaceBootFailure(meta);
|
|
698
|
-
}
|
|
699
|
-
catch (err) {
|
|
700
|
-
process.stderr.write(`[crtrd] surfaceBootFailure ${row.node_id} error: ${err.message}\n`);
|
|
701
|
-
}
|
|
702
|
-
}
|
|
703
|
-
else {
|
|
704
|
-
process.stderr.write(`[crtrd] dead ${row.node_id} (pane gone mid-generation, intent=${String(row.intent)})\n`);
|
|
705
|
-
// Wake the inbox-waiting parent on a GENUINE mid-run death (D-1): a
|
|
706
|
-
// booted child whose pane died inside a turn is unambiguously dead
|
|
707
|
-
// (busy marker present), never healthy dormancy.
|
|
708
|
-
if (meta !== null) {
|
|
709
|
-
try {
|
|
710
|
-
await surfaceChildDeath(meta, 'crash');
|
|
711
|
-
}
|
|
712
|
-
catch (err) {
|
|
713
|
-
process.stderr.write(`[crtrd] surfaceChildDeath(crash) ${row.node_id} error: ${err.message}\n`);
|
|
714
|
-
}
|
|
715
|
-
}
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
}
|
|
642
|
+
// Every node is broker-hosted (host_kind always 'broker'; pane/window/
|
|
643
|
+
// tmux_session always NULL) and supervised by its engine pid alone. A tmux
|
|
644
|
+
// pane, if any, is just a viewer — the daemon never consults it and a viewer
|
|
645
|
+
// closing is not a node death. There is no inline-root carve-out and no
|
|
646
|
+
// pane-gone reaping: one liveness path for all.
|
|
647
|
+
await handleNodeLiveness(row, now, revivedThisTick);
|
|
719
648
|
}
|
|
720
649
|
catch (err) {
|
|
721
650
|
// One bad node must never kill the loop.
|
|
@@ -729,18 +658,16 @@ export async function superviseTick(now = Date.now()) {
|
|
|
729
658
|
for (const row of rows) {
|
|
730
659
|
try {
|
|
731
660
|
// Re-read the ROW for fresh runtime (the first pass may have mutated it);
|
|
732
|
-
// no meta needed — status/intent
|
|
661
|
+
// no meta needed — status/intent live in the row.
|
|
733
662
|
const r = getRow(row.node_id);
|
|
734
663
|
if (r === null)
|
|
735
664
|
continue;
|
|
736
665
|
if (r.status !== 'idle' || r.intent !== 'idle-release')
|
|
737
666
|
continue;
|
|
738
|
-
// The in-process inbox-watcher only owns delivery while
|
|
739
|
-
// A released node is
|
|
740
|
-
//
|
|
741
|
-
//
|
|
742
|
-
// on pi liveness, NOT pane presence (which would skip a frozen pane
|
|
743
|
-
// forever, §3c).
|
|
667
|
+
// The in-process inbox-watcher only owns delivery while the engine is LIVE.
|
|
668
|
+
// A released node's broker is DEAD (it freed itself while dormant), so it
|
|
669
|
+
// has no watcher and the daemon must wake it. Gate the skip on engine
|
|
670
|
+
// liveness alone — viewer panes are irrelevant here.
|
|
744
671
|
if (r.pi_pid != null && isPidAlive(r.pi_pid))
|
|
745
672
|
continue;
|
|
746
673
|
const entries = readInboxSince(row.node_id, readCursor(row.node_id));
|