borgmcp 1.1.15 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +15 -0
- package/LICENSE +201 -33
- package/NOTICE +2 -0
- package/README.md +85 -36
- package/SECURITY.md +7 -0
- package/dist/agent-runtime.d.ts.map +1 -0
- package/dist/agent-runtime.js +63 -1
- package/dist/agent-runtime.js.map +1 -0
- package/dist/assimilate-cmd.d.ts +96 -17
- package/dist/assimilate-cmd.d.ts.map +1 -0
- package/dist/assimilate-cmd.js +1396 -48
- package/dist/assimilate-cmd.js.map +1 -0
- package/dist/assimilate-deps.d.ts.map +1 -0
- package/dist/assimilate-deps.js +384 -3
- package/dist/assimilate-deps.js.map +1 -0
- package/dist/assimilate-guard.d.ts +5 -7
- package/dist/assimilate-guard.d.ts.map +1 -0
- package/dist/assimilate-guard.js +60 -2
- package/dist/assimilate-guard.js.map +1 -0
- package/dist/assimilate-welcome.d.ts +1 -1
- package/dist/assimilate-welcome.d.ts.map +1 -0
- package/dist/assimilate-welcome.js +32 -2
- package/dist/assimilate-welcome.js.map +1 -0
- package/dist/auth-env.d.ts +7 -50
- package/dist/auth-env.d.ts.map +1 -0
- package/dist/auth-env.js +22 -1
- package/dist/auth-env.js.map +1 -0
- package/dist/backends/launch-all-pastelist.d.ts.map +1 -0
- package/dist/backends/launch-all-pastelist.js +11 -4
- package/dist/backends/launch-all-pastelist.js.map +1 -0
- package/dist/backends/launch-all-tmux.d.ts.map +1 -0
- package/dist/backends/launch-all-tmux.js +49 -1
- package/dist/backends/launch-all-tmux.js.map +1 -0
- package/dist/backends/launch-all-windows.d.ts.map +1 -0
- package/dist/backends/launch-all-windows.js +86 -12
- package/dist/backends/launch-all-windows.js.map +1 -0
- package/dist/bare-launch-menu.d.ts.map +1 -0
- package/dist/bare-launch-menu.js +83 -6
- package/dist/bare-launch-menu.js.map +1 -0
- package/dist/claude-launch-args.d.ts.map +1 -0
- package/dist/claude-launch-args.js +34 -1
- package/dist/claude-launch-args.js.map +1 -0
- package/dist/claude.d.ts.map +1 -0
- package/dist/claude.js +486 -14
- package/dist/claude.js.map +1 -0
- package/dist/cleanup-cmd.d.ts +5 -5
- package/dist/cleanup-cmd.d.ts.map +1 -0
- package/dist/cleanup-cmd.js +453 -14
- package/dist/cleanup-cmd.js.map +1 -0
- package/dist/clear-rewake-core.d.ts.map +1 -0
- package/dist/clear-rewake-core.js +23 -2
- package/dist/clear-rewake-core.js.map +1 -0
- package/dist/clear-rewake.d.ts.map +1 -0
- package/dist/clear-rewake.js +19 -1
- package/dist/clear-rewake.js.map +1 -0
- package/dist/cli-help.d.ts +5 -0
- package/dist/cli-help.d.ts.map +1 -0
- package/dist/cli-help.js +104 -56
- package/dist/cli-help.js.map +1 -0
- package/dist/cli-platform.d.ts.map +1 -0
- package/dist/cli-platform.js +106 -4
- package/dist/cli-platform.js.map +1 -0
- package/dist/cli-tool-approval.d.ts +84 -0
- package/dist/cli-tool-approval.d.ts.map +1 -0
- package/dist/cli-tool-approval.js +583 -0
- package/dist/cli-tool-approval.js.map +1 -0
- package/dist/codex-app-server.d.ts.map +1 -0
- package/dist/codex-app-server.js +228 -4
- package/dist/codex-app-server.js.map +1 -0
- package/dist/codex-app-wake.d.ts.map +1 -0
- package/dist/codex-app-wake.js +432 -2
- package/dist/codex-app-wake.js.map +1 -0
- package/dist/codex-launch.d.ts.map +1 -0
- package/dist/codex-launch.js +102 -2
- package/dist/codex-launch.js.map +1 -0
- package/dist/codex-remote.d.ts +6 -1
- package/dist/codex-remote.d.ts.map +1 -0
- package/dist/codex-remote.js +394 -1
- package/dist/codex-remote.js.map +1 -0
- package/dist/codex-wake-resolve.d.ts.map +1 -0
- package/dist/codex-wake-resolve.js +127 -1
- package/dist/codex-wake-resolve.js.map +1 -0
- package/dist/config-utils.d.ts +2 -2
- package/dist/config-utils.d.ts.map +1 -0
- package/dist/config-utils.js +766 -3
- package/dist/config-utils.js.map +1 -0
- package/dist/config.d.ts +98 -61
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +426 -1
- package/dist/config.js.map +1 -0
- package/dist/console-prefix.d.ts.map +1 -0
- package/dist/console-prefix.js +96 -1
- package/dist/console-prefix.js.map +1 -0
- package/dist/cube-name.d.ts.map +1 -0
- package/dist/cube-name.js +67 -1
- package/dist/cube-name.js.map +1 -0
- package/dist/cubes.d.ts +126 -18
- package/dist/cubes.d.ts.map +1 -0
- package/dist/cubes.js +412 -4
- package/dist/cubes.js.map +1 -0
- package/dist/debug.d.ts.map +1 -0
- package/dist/debug.js +71 -1
- package/dist/debug.js.map +1 -0
- package/dist/direct-log.d.ts.map +1 -0
- package/dist/direct-log.js +11 -1
- package/dist/direct-log.js.map +1 -0
- package/dist/docs-sections.d.ts +11 -10
- package/dist/docs-sections.d.ts.map +1 -0
- package/dist/docs-sections.js +113 -3
- package/dist/docs-sections.js.map +1 -0
- package/dist/drone-lifecycle.d.ts +5 -21
- package/dist/drone-lifecycle.d.ts.map +1 -0
- package/dist/drone-lifecycle.js +57 -8
- package/dist/drone-lifecycle.js.map +1 -0
- package/dist/drone-management.d.ts +29 -0
- package/dist/drone-management.d.ts.map +1 -0
- package/dist/drone-management.js +109 -0
- package/dist/drone-management.js.map +1 -0
- package/dist/ensure-mcp-config.d.ts.map +1 -0
- package/dist/ensure-mcp-config.js +36 -1
- package/dist/ensure-mcp-config.js.map +1 -0
- package/dist/evict-drone.d.ts +10 -2
- package/dist/evict-drone.d.ts.map +1 -0
- package/dist/evict-drone.js +60 -1
- package/dist/evict-drone.js.map +1 -0
- package/dist/gc-orphan-inboxes.d.ts.map +1 -0
- package/dist/gc-orphan-inboxes.js +177 -1
- package/dist/gc-orphan-inboxes.js.map +1 -0
- package/dist/inbox-monitor.d.ts +1 -1
- package/dist/inbox-monitor.d.ts.map +1 -0
- package/dist/inbox-monitor.js +856 -3
- package/dist/inbox-monitor.js.map +1 -0
- package/dist/index.d.ts +6 -4
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1034 -37
- package/dist/index.js.map +1 -0
- package/dist/launch-all-cmd.d.ts.map +1 -0
- package/dist/launch-all-cmd.js +377 -37
- package/dist/launch-all-cmd.js.map +1 -0
- package/dist/launch-all-command.d.ts.map +1 -0
- package/dist/launch-all-command.js +24 -1
- package/dist/launch-all-command.js.map +1 -0
- package/dist/launch-all-deps.d.ts +3 -8
- package/dist/launch-all-deps.d.ts.map +1 -0
- package/dist/launch-all-deps.js +99 -1
- package/dist/launch-all-deps.js.map +1 -0
- package/dist/launch-all-discovery.d.ts +1 -0
- package/dist/launch-all-discovery.d.ts.map +1 -0
- package/dist/launch-all-discovery.js +90 -4
- package/dist/launch-all-discovery.js.map +1 -0
- package/dist/launch-all-locks.d.ts.map +1 -0
- package/dist/launch-all-locks.js +54 -1
- package/dist/launch-all-locks.js.map +1 -0
- package/dist/launch-gate.d.ts +2 -2
- package/dist/launch-gate.d.ts.map +1 -0
- package/dist/launch-gate.js +73 -1
- package/dist/launch-gate.js.map +1 -0
- package/dist/lifecycle-log-guard.d.ts.map +1 -0
- package/dist/lifecycle-log-guard.js +93 -2
- package/dist/lifecycle-log-guard.js.map +1 -0
- package/dist/list-roles-render.d.ts.map +1 -0
- package/dist/list-roles-render.js +40 -6
- package/dist/list-roles-render.js.map +1 -0
- package/dist/local-log-routing.d.ts +19 -0
- package/dist/local-log-routing.d.ts.map +1 -0
- package/dist/local-log-routing.js +67 -0
- package/dist/local-log-routing.js.map +1 -0
- package/dist/local-manage-tool-result.d.ts +8 -0
- package/dist/local-manage-tool-result.d.ts.map +1 -0
- package/dist/local-manage-tool-result.js +10 -0
- package/dist/local-manage-tool-result.js.map +1 -0
- package/dist/local-server-cursor.d.ts +34 -0
- package/dist/local-server-cursor.d.ts.map +1 -0
- package/dist/local-server-cursor.js +151 -0
- package/dist/local-server-cursor.js.map +1 -0
- package/dist/log-audit.d.ts.map +1 -0
- package/dist/log-audit.js +194 -3
- package/dist/log-audit.js.map +1 -0
- package/dist/log-stream.d.ts +31 -18
- package/dist/log-stream.d.ts.map +1 -0
- package/dist/log-stream.js +1214 -12
- package/dist/log-stream.js.map +1 -0
- package/dist/model-presets.d.ts.map +1 -0
- package/dist/model-presets.js +19 -1
- package/dist/model-presets.js.map +1 -0
- package/dist/name-validator.d.ts.map +1 -0
- package/dist/name-validator.js +22 -1
- package/dist/name-validator.js.map +1 -0
- package/dist/opencode-drone.d.ts.map +1 -0
- package/dist/opencode-drone.js +438 -4
- package/dist/opencode-drone.js.map +1 -0
- package/dist/opencode-plugin.d.ts +1 -1
- package/dist/opencode-plugin.d.ts.map +1 -0
- package/dist/opencode-plugin.js +29 -4
- package/dist/opencode-plugin.js.map +1 -0
- package/dist/parse-assimilate-args.d.ts +1 -1
- package/dist/parse-assimilate-args.d.ts.map +1 -0
- package/dist/parse-assimilate-args.js +125 -1
- package/dist/parse-assimilate-args.js.map +1 -0
- package/dist/parse-launch-all-args.d.ts.map +1 -0
- package/dist/parse-launch-all-args.js +70 -1
- package/dist/parse-launch-all-args.js.map +1 -0
- package/dist/postinstall-banner.d.ts.map +1 -0
- package/dist/postinstall-banner.js +27 -2
- package/dist/postinstall-banner.js.map +1 -0
- package/dist/postinstall.d.ts.map +1 -0
- package/dist/postinstall.js +32 -5
- package/dist/postinstall.js.map +1 -0
- package/dist/readiness-probe.d.ts.map +1 -0
- package/dist/readiness-probe.js +9 -1
- package/dist/readiness-probe.js.map +1 -0
- package/dist/regen-format.d.ts.map +1 -0
- package/dist/regen-format.js +545 -13
- package/dist/regen-format.js.map +1 -0
- package/dist/regen.d.ts.map +1 -0
- package/dist/regen.js +141 -5
- package/dist/regen.js.map +1 -0
- package/dist/remote-client.d.ts +65 -175
- package/dist/remote-client.d.ts.map +1 -0
- package/dist/remote-client.js +942 -1
- package/dist/remote-client.js.map +1 -0
- package/dist/reset-local-seat-cmd.d.ts +50 -0
- package/dist/reset-local-seat-cmd.d.ts.map +1 -0
- package/dist/reset-local-seat-cmd.js +169 -0
- package/dist/reset-local-seat-cmd.js.map +1 -0
- package/dist/role-match.d.ts.map +1 -0
- package/dist/role-match.js +23 -1
- package/dist/role-match.js.map +1 -0
- package/dist/role-resolver.d.ts.map +1 -0
- package/dist/role-resolver.js +60 -1
- package/dist/role-resolver.js.map +1 -0
- package/dist/roster-render.d.ts.map +1 -0
- package/dist/roster-render.js +121 -3
- package/dist/roster-render.js.map +1 -0
- package/dist/seat-probe.d.ts +32 -13
- package/dist/seat-probe.d.ts.map +1 -0
- package/dist/seat-probe.js +78 -1
- package/dist/seat-probe.js.map +1 -0
- package/dist/seat-store.d.ts +76 -0
- package/dist/seat-store.d.ts.map +1 -0
- package/dist/seat-store.js +309 -0
- package/dist/seat-store.js.map +1 -0
- package/dist/seats.d.ts +303 -0
- package/dist/seats.d.ts.map +1 -0
- package/dist/seats.js +551 -0
- package/dist/seats.js.map +1 -0
- package/dist/self-path.d.ts +27 -0
- package/dist/self-path.d.ts.map +1 -0
- package/dist/self-path.js +43 -0
- package/dist/self-path.js.map +1 -0
- package/dist/server-endpoint.d.ts +9 -0
- package/dist/server-endpoint.d.ts.map +1 -0
- package/dist/server-endpoint.js +38 -0
- package/dist/server-endpoint.js.map +1 -0
- package/dist/server-errors.d.ts +50 -0
- package/dist/server-errors.d.ts.map +1 -0
- package/dist/server-errors.js +79 -0
- package/dist/server-errors.js.map +1 -0
- package/dist/server-handshake.d.ts +197 -0
- package/dist/server-handshake.d.ts.map +1 -0
- package/dist/server-handshake.js +505 -0
- package/dist/server-handshake.js.map +1 -0
- package/dist/server-response.d.ts +3 -0
- package/dist/server-response.d.ts.map +1 -0
- package/dist/server-response.js +67 -0
- package/dist/server-response.js.map +1 -0
- package/dist/server-trust.d.ts +14 -0
- package/dist/server-trust.d.ts.map +1 -0
- package/dist/server-trust.js +225 -0
- package/dist/server-trust.js.map +1 -0
- package/dist/setup-confirm.d.ts +3 -3
- package/dist/setup-confirm.d.ts.map +1 -0
- package/dist/setup-confirm.js +126 -2
- package/dist/setup-confirm.js.map +1 -0
- package/dist/setup.d.ts +5 -3
- package/dist/setup.d.ts.map +1 -0
- package/dist/setup.js +169 -46
- package/dist/setup.js.map +1 -0
- package/dist/shell-escape.d.ts.map +1 -0
- package/dist/shell-escape.js +22 -1
- package/dist/shell-escape.js.map +1 -0
- package/dist/spawn.d.ts +2 -2
- package/dist/spawn.d.ts.map +1 -0
- package/dist/spawn.js +29 -10
- package/dist/spawn.js.map +1 -0
- package/dist/startup-services.d.ts +0 -1
- package/dist/startup-services.d.ts.map +1 -0
- package/dist/startup-services.js +26 -1
- package/dist/startup-services.js.map +1 -0
- package/dist/stream-owner.d.ts +6 -2
- package/dist/stream-owner.d.ts.map +1 -0
- package/dist/stream-owner.js +547 -2
- package/dist/stream-owner.js.map +1 -0
- package/dist/stream-status.d.ts +1 -1
- package/dist/stream-status.d.ts.map +1 -0
- package/dist/stream-status.js +268 -3
- package/dist/stream-status.js.map +1 -0
- package/dist/sync-roles-render.d.ts.map +1 -0
- package/dist/sync-roles-render.js +118 -3
- package/dist/sync-roles-render.js.map +1 -0
- package/dist/sync.d.ts +1 -1
- package/dist/sync.d.ts.map +1 -0
- package/dist/sync.js +286 -22
- package/dist/sync.js.map +1 -0
- package/dist/terminal-title.d.ts.map +1 -0
- package/dist/terminal-title.js +68 -1
- package/dist/terminal-title.js.map +1 -0
- package/dist/token-store.d.ts +17 -111
- package/dist/token-store.d.ts.map +1 -0
- package/dist/token-store.js +81 -1
- package/dist/token-store.js.map +1 -0
- package/dist/tool-manifest.d.ts +4 -8
- package/dist/tool-manifest.d.ts.map +1 -0
- package/dist/tool-manifest.js +554 -1
- package/dist/tool-manifest.js.map +1 -0
- package/dist/tool-scope.d.ts +8 -12
- package/dist/tool-scope.d.ts.map +1 -0
- package/dist/tool-scope.js +97 -1
- package/dist/tool-scope.js.map +1 -0
- package/dist/types.d.ts +1 -5
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +5 -0
- package/dist/types.js.map +1 -0
- package/dist/unknown-subcommand.d.ts +1 -1
- package/dist/unknown-subcommand.d.ts.map +1 -0
- package/dist/unknown-subcommand.js +37 -1
- package/dist/unknown-subcommand.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +78 -2
- package/dist/version.js.map +1 -0
- package/dist/working-repo.d.ts.map +1 -0
- package/dist/working-repo.js +81 -1
- package/dist/working-repo.js.map +1 -0
- package/dist/worktree-lifecycle.d.ts.map +1 -0
- package/dist/worktree-lifecycle.js +205 -2
- package/dist/worktree-lifecycle.js.map +1 -0
- package/docs/EXTRACTION_PROVENANCE.md +40 -0
- package/docs/LOCAL_SERVER.md +103 -0
- package/docs/RELEASING.md +120 -0
- package/package.json +52 -34
- package/src/agent-runtime.ts +69 -0
- package/src/assimilate-cmd.ts +1864 -0
- package/src/assimilate-deps.ts +444 -0
- package/src/assimilate-guard.ts +84 -0
- package/src/assimilate-welcome.ts +39 -0
- package/src/auth-env.ts +21 -0
- package/src/backends/launch-all-pastelist.ts +18 -0
- package/src/backends/launch-all-tmux.ts +75 -0
- package/src/backends/launch-all-windows.ts +110 -0
- package/src/bare-launch-menu.ts +148 -0
- package/src/claude-launch-args.ts +35 -0
- package/src/claude.ts +561 -0
- package/src/cleanup-cmd.ts +581 -0
- package/src/clear-rewake-core.ts +34 -0
- package/src/clear-rewake.ts +18 -0
- package/src/cli-help.ts +116 -0
- package/src/cli-platform.ts +127 -0
- package/src/cli-tool-approval.ts +716 -0
- package/src/codex-app-server.ts +238 -0
- package/src/codex-app-wake.ts +512 -0
- package/src/codex-launch.ts +140 -0
- package/src/codex-remote.ts +500 -0
- package/src/codex-wake-resolve.ts +155 -0
- package/src/config-utils.ts +811 -0
- package/src/config.ts +595 -0
- package/src/console-prefix.ts +99 -0
- package/src/cube-name.ts +69 -0
- package/src/cubes.ts +583 -0
- package/src/debug.ts +73 -0
- package/src/direct-log.ts +9 -0
- package/src/docs-sections.ts +129 -0
- package/src/drone-lifecycle.ts +62 -0
- package/src/drone-management.ts +162 -0
- package/src/ensure-mcp-config.ts +54 -0
- package/src/evict-drone.ts +82 -0
- package/src/gc-orphan-inboxes.ts +229 -0
- package/src/inbox-monitor.ts +1005 -0
- package/src/index.ts +1201 -0
- package/src/launch-all-cmd.ts +450 -0
- package/src/launch-all-command.ts +30 -0
- package/src/launch-all-deps.ts +188 -0
- package/src/launch-all-discovery.ts +118 -0
- package/src/launch-all-locks.ts +74 -0
- package/src/launch-gate.ts +80 -0
- package/src/lifecycle-log-guard.ts +144 -0
- package/src/list-roles-render.ts +52 -0
- package/src/local-log-routing.ts +106 -0
- package/src/local-manage-tool-result.ts +10 -0
- package/src/local-server-cursor.ts +188 -0
- package/src/log-audit.ts +207 -0
- package/src/log-stream.ts +1505 -0
- package/src/model-presets.ts +22 -0
- package/src/name-validator.ts +23 -0
- package/src/opencode-drone.ts +528 -0
- package/src/opencode-plugin.ts +38 -0
- package/src/parse-assimilate-args.ts +125 -0
- package/src/parse-launch-all-args.ts +100 -0
- package/src/postinstall-banner.ts +37 -0
- package/src/postinstall.ts +36 -0
- package/src/readiness-probe.ts +12 -0
- package/src/regen-format.ts +651 -0
- package/src/regen.ts +150 -0
- package/src/remote-client.ts +1379 -0
- package/src/reset-local-seat-cmd.ts +227 -0
- package/src/role-match.ts +29 -0
- package/src/role-resolver.ts +93 -0
- package/src/roster-render.ts +196 -0
- package/src/seat-probe.ts +115 -0
- package/src/seat-store.ts +333 -0
- package/src/seats.ts +741 -0
- package/src/self-path.ts +51 -0
- package/src/server-endpoint.ts +38 -0
- package/src/server-errors.ts +93 -0
- package/src/server-handshake.ts +783 -0
- package/src/server-response.ts +76 -0
- package/src/server-trust.ts +245 -0
- package/src/setup-confirm.ts +171 -0
- package/src/setup.ts +186 -0
- package/src/shell-escape.ts +22 -0
- package/src/spawn.ts +31 -0
- package/src/startup-services.ts +36 -0
- package/src/stream-owner.ts +686 -0
- package/src/stream-status.ts +337 -0
- package/src/sync-roles-render.ts +163 -0
- package/src/sync.ts +363 -0
- package/src/terminal-title.ts +76 -0
- package/src/token-store.ts +102 -0
- package/src/tool-manifest.ts +610 -0
- package/src/tool-scope.ts +110 -0
- package/src/types.ts +5 -0
- package/src/unknown-subcommand.ts +35 -0
- package/src/version.ts +82 -0
- package/src/working-repo.ts +99 -0
- package/src/worktree-lifecycle.ts +254 -0
- package/dist/auth-recovery.d.ts +0 -33
- package/dist/auth-recovery.js +0 -1
- package/dist/auth.d.ts +0 -103
- package/dist/auth.js +0 -39
- package/dist/device-auth.d.ts +0 -75
- package/dist/device-auth.js +0 -1
- package/dist/drone-address.d.ts +0 -26
- package/dist/drone-address.js +0 -1
- package/dist/get-started.d.ts +0 -20
- package/dist/get-started.js +0 -2
- package/dist/health-beat.d.ts +0 -134
- package/dist/health-beat.js +0 -1
- package/dist/role-section.d.ts +0 -110
- package/dist/role-section.js +0 -8
- package/dist/setup-action.d.ts +0 -20
- package/dist/setup-action.js +0 -1
- package/dist/stale-version-check.d.ts +0 -40
- package/dist/stale-version-check.js +0 -1
- package/dist/subscription-retry.d.ts +0 -40
- package/dist/subscription-retry.js +0 -1
- package/dist/templates.d.ts +0 -107
- package/dist/templates.js +0 -359
- package/dist/token-crypto.d.ts +0 -50
- package/dist/token-crypto.js +0 -1
|
@@ -0,0 +1,1505 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Real-time drone wakeup via Server-Sent Events.
|
|
3
|
+
*
|
|
4
|
+
* Replaces the former long-poll + inbox-file-shim wire path
|
|
5
|
+
* for the wire layer; the local inbox file is preserved as the
|
|
6
|
+
* Claude-side wake primitive (Monitor on tail -F) because Claude Code
|
|
7
|
+
* does not currently wake idle agent loops on MCP-protocol
|
|
8
|
+
* notifications. See spec:
|
|
9
|
+
* docs/superpowers/specs/2026-05-11-server-push-log-subscription.md
|
|
10
|
+
*
|
|
11
|
+
* Lifetimes:
|
|
12
|
+
* - One persistent fetch-streaming connection to the cube-scoped stream
|
|
13
|
+
* per active cube.
|
|
14
|
+
* - On every received `event: log`, a single line is appended to the
|
|
15
|
+
* per-drone inbox file (same format the old poller wrote).
|
|
16
|
+
* - On every `event: heartbeat`, any carried hwm is compared against
|
|
17
|
+
* `lastPersistedEventId`; divergence starts a short grace timer
|
|
18
|
+
* before reconnect so an in-flight live broadcast can arrive first.
|
|
19
|
+
* - Outer `runLoop` reconnects with exponential backoff on any
|
|
20
|
+
* stream-level error, including heartbeat watchdog firing.
|
|
21
|
+
*
|
|
22
|
+
* State exposed via `getStreamStatus()` so the `borg_stream-status`
|
|
23
|
+
* MCP tool can probe without perturbing the stream (no second
|
|
24
|
+
* connection, no second auth — just an in-process state snapshot).
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { Buffer } from 'node:buffer';
|
|
28
|
+
import { promises as fs } from 'node:fs';
|
|
29
|
+
import path from 'node:path';
|
|
30
|
+
import { compareBroadcastHwm, type BroadcastHwm } from 'borgmcp-shared/log-stream-hwm';
|
|
31
|
+
import { getActiveCube, inboxPathForDrone } from './cubes.js';
|
|
32
|
+
import { assertUuidShape } from './evict-drone.js';
|
|
33
|
+
import { loadBorgServerTrust } from './server-trust.js';
|
|
34
|
+
import {
|
|
35
|
+
advanceLocalServerCursor,
|
|
36
|
+
clearLocalServerCursor,
|
|
37
|
+
encodeLocalServerCursor,
|
|
38
|
+
getLocalServerCursor,
|
|
39
|
+
type LocalServerCursor,
|
|
40
|
+
} from './local-server-cursor.js';
|
|
41
|
+
import {
|
|
42
|
+
DroneEvictedError,
|
|
43
|
+
DRONE_EVICTED_CODE,
|
|
44
|
+
EVICTED_RESULT_MARKER,
|
|
45
|
+
errorCodeFromBody,
|
|
46
|
+
} from './drone-lifecycle.js';
|
|
47
|
+
import {
|
|
48
|
+
CODEX_HEARTBEAT_CADENCE_MS,
|
|
49
|
+
fireCodexHeartbeatTick,
|
|
50
|
+
formatCodexWakePrompt,
|
|
51
|
+
resolveSessionAgentKind,
|
|
52
|
+
startCodexHeartbeat,
|
|
53
|
+
wakeCodexViaAppServer,
|
|
54
|
+
} from './codex-app-wake.js';
|
|
55
|
+
import { readBoundedResponseBody } from './server-response.js';
|
|
56
|
+
import {
|
|
57
|
+
acquireStreamLease,
|
|
58
|
+
readOwnershipSnapshot,
|
|
59
|
+
STREAM_OWNER_STALE_MS,
|
|
60
|
+
type StreamLease,
|
|
61
|
+
type StreamOwnershipSnapshot,
|
|
62
|
+
} from './stream-owner.js';
|
|
63
|
+
|
|
64
|
+
// ------------------------------------------------------------------
|
|
65
|
+
// Tuning constants
|
|
66
|
+
// ------------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
/** Server emits heartbeats every 20s; allow several misses before reconnecting. */
|
|
69
|
+
const HEARTBEAT_TIMEOUT_MS = 90_000;
|
|
70
|
+
/** Grace window for a heartbeat HWM that is ahead of the local cursor. */
|
|
71
|
+
const HWM_DIVERGENCE_GRACE_MS = 2_000;
|
|
72
|
+
const RECONNECT_MIN_MS = 500;
|
|
73
|
+
const RECONNECT_MAX_MS = 30_000;
|
|
74
|
+
export const LOCAL_SERVER_SSE_FRAME_LIMIT_BYTES = 64 * 1024;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* client#42: the server's structured code for an expired stream resume cursor
|
|
78
|
+
* (server 410 CURSOR_EXPIRED — the pointed-at entry was pruned). Recoverable,
|
|
79
|
+
* NOT terminal: distinct from DRONE_EVICTED.
|
|
80
|
+
*/
|
|
81
|
+
export const CURSOR_EXPIRED_CODE = 'CURSOR_EXPIRED';
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* client#42: recoverable stream error thrown when the server rejects the resume
|
|
85
|
+
* cursor as expired. The resume cursor is reset BEFORE this is thrown, so the
|
|
86
|
+
* reconnect loop's next connect re-establishes from a fresh valid point instead
|
|
87
|
+
* of looping forever on the dead cursor. Deliberately NOT a DroneEvictedError —
|
|
88
|
+
* the reconnect loop treats only DroneEvictedError as terminal, so this falls
|
|
89
|
+
* through to the ordinary backoff-reconnect (which now recovers).
|
|
90
|
+
*/
|
|
91
|
+
export class StreamCursorExpiredError extends Error {
|
|
92
|
+
constructor(
|
|
93
|
+
message = 'stream resume cursor expired — reset for catchup, reconnecting'
|
|
94
|
+
) {
|
|
95
|
+
super(message);
|
|
96
|
+
this.name = 'StreamCursorExpiredError';
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function isProcessAlive(pid: number): boolean {
|
|
101
|
+
try {
|
|
102
|
+
process.kill(pid, 0);
|
|
103
|
+
return true;
|
|
104
|
+
} catch (error) {
|
|
105
|
+
return (error as NodeJS.ErrnoException).code === 'EPERM';
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* gh#opencode: module-level mutable opencode entry injector. Set after
|
|
110
|
+
* startup (from index.ts) once the opencode drone module is initialized.
|
|
111
|
+
* Used by defaultDeps when no explicit injectOpenCode is supplied.
|
|
112
|
+
*/
|
|
113
|
+
let _moduleInjectOpenCode: ((text: string) => Promise<boolean>) | undefined;
|
|
114
|
+
|
|
115
|
+
export function setModuleInjectOpenCode(fn: (text: string) => Promise<boolean>): void {
|
|
116
|
+
_moduleInjectOpenCode = fn;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Bounded recent-id set sized for the SSE replay window per spec §(3).
|
|
121
|
+
* The catchup query returns up to 200 entries on reconnect; 50 is
|
|
122
|
+
* comfortable headroom for the typical reorder/dedup case (1–3
|
|
123
|
+
* entries) without the memory cost of the old long-poll 500-entry
|
|
124
|
+
* ring buffer.
|
|
125
|
+
*/
|
|
126
|
+
const RECENT_IDS_CAP = 50;
|
|
127
|
+
export const INBOX_TAIL_LINES_CAP = 512;
|
|
128
|
+
export const INBOX_TAIL_TRIM_THRESHOLD_LINES = INBOX_TAIL_LINES_CAP * 2;
|
|
129
|
+
|
|
130
|
+
// ------------------------------------------------------------------
|
|
131
|
+
// Probeable status (read by the `borg_stream-status` MCP tool)
|
|
132
|
+
// ------------------------------------------------------------------
|
|
133
|
+
|
|
134
|
+
export type RunLoopHealth = 'connected' | 'reconnecting' | 'silent-inert' | 'never-started';
|
|
135
|
+
|
|
136
|
+
export interface StreamStatus {
|
|
137
|
+
connected: boolean;
|
|
138
|
+
lastWireActivityAt: string | null;
|
|
139
|
+
lastContentEventAt: string | null;
|
|
140
|
+
lastHeartbeatAt: string | null;
|
|
141
|
+
lastPersistedEventId: string | null;
|
|
142
|
+
reconnectAttempts: number;
|
|
143
|
+
runLoopRestartCount: number;
|
|
144
|
+
ownership?: StreamOwnershipSnapshot;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const processStartMs = Date.now();
|
|
148
|
+
|
|
149
|
+
const streamState: StreamStatus = {
|
|
150
|
+
connected: false,
|
|
151
|
+
lastWireActivityAt: null,
|
|
152
|
+
lastContentEventAt: null,
|
|
153
|
+
lastHeartbeatAt: null,
|
|
154
|
+
lastPersistedEventId: null,
|
|
155
|
+
reconnectAttempts: 0,
|
|
156
|
+
runLoopRestartCount: 0,
|
|
157
|
+
ownership: { state: 'unowned' },
|
|
158
|
+
};
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Snapshot of the current stream status. Safe to call at any time
|
|
162
|
+
* (returns a copy, does not interact with the running connection).
|
|
163
|
+
*/
|
|
164
|
+
export function classifyRunLoopHealth(state: StreamStatus, uptimeMs: number = Date.now() - processStartMs): RunLoopHealth {
|
|
165
|
+
if (state.connected && state.lastWireActivityAt) return 'connected';
|
|
166
|
+
if (!state.connected && state.reconnectAttempts > 0) return 'reconnecting';
|
|
167
|
+
if (!state.connected && !state.lastWireActivityAt && state.reconnectAttempts === 0 && uptimeMs > 10_000) return 'silent-inert';
|
|
168
|
+
return 'never-started';
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function getStreamStatus(): StreamStatus & { runLoopHealth: RunLoopHealth } {
|
|
172
|
+
return { ...streamState, runLoopHealth: classifyRunLoopHealth(streamState) };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Reset the module-level stream state. EXPORTED FOR TESTS ONLY — the
|
|
177
|
+
* stream state is a singleton that accumulates across the process
|
|
178
|
+
* lifetime in normal operation. Tests asserting on the new
|
|
179
|
+
* content-vs-wire field semantics need a clean slate between
|
|
180
|
+
* scenarios; nothing in production code should call this.
|
|
181
|
+
*
|
|
182
|
+
* @internal — test-only surface; not part of the public client API. The
|
|
183
|
+
* `__`-prefix + `ForTest` suffix mark it as such at every call site.
|
|
184
|
+
*/
|
|
185
|
+
export function __resetStreamStateForTest(): void {
|
|
186
|
+
streamState.connected = false;
|
|
187
|
+
streamState.lastWireActivityAt = null;
|
|
188
|
+
streamState.lastContentEventAt = null;
|
|
189
|
+
streamState.lastHeartbeatAt = null;
|
|
190
|
+
streamState.lastPersistedEventId = null;
|
|
191
|
+
streamState.reconnectAttempts = 0;
|
|
192
|
+
streamState.runLoopRestartCount = 0;
|
|
193
|
+
streamState.ownership = { state: 'unowned' };
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// ------------------------------------------------------------------
|
|
197
|
+
// Entry point
|
|
198
|
+
// ------------------------------------------------------------------
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* Spawn the background SSE consumer loop. Fire-and-forget; the loop
|
|
202
|
+
* runs until process exit. Errors are written to stderr (so they don't
|
|
203
|
+
* pollute the MCP stdio channel) and the loop continues.
|
|
204
|
+
*
|
|
205
|
+
* Idempotent in the sense that calling twice would create two parallel
|
|
206
|
+
* loops; the caller (`index.ts`) wires this once at startup.
|
|
207
|
+
*/
|
|
208
|
+
let codexHeartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
|
209
|
+
// gh#866 item 3: a one-shot deferred re-arm scheduled by the ENOENT teardown
|
|
210
|
+
// path so a mid-session app-server restart self-heals within the session (see
|
|
211
|
+
// onCodexAppServerSocketDead). null when no re-arm is pending.
|
|
212
|
+
let codexReArmTimer: ReturnType<typeof setTimeout> | null = null;
|
|
213
|
+
// Re-arm delay = one tick cadence: closes the mid-session permanent-deaf gap to
|
|
214
|
+
// a single-cadence window (== the no-teardown baseline) without busy-probing a
|
|
215
|
+
// dead socket. Overridable for tests.
|
|
216
|
+
let codexReArmDelayMs = CODEX_HEARTBEAT_CADENCE_MS;
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* gh#857 WI-2: start the codex /loop-equivalent heartbeat AT MOST ONCE — a
|
|
220
|
+
* re-entrant startLogStream must not leak a second interval (idempotent start).
|
|
221
|
+
* No-op for claude (startCodexHeartbeat returns null on a non-codex session, so
|
|
222
|
+
* no timer is ever stored). Extracted + injectable (`start`) so the production
|
|
223
|
+
* wiring is unit-testable without running the real stream loop (QA e75339e7).
|
|
224
|
+
*/
|
|
225
|
+
export function ensureCodexHeartbeatStarted(
|
|
226
|
+
start: () => ReturnType<typeof setInterval> | null = defaultStartCodexHeartbeat
|
|
227
|
+
): void {
|
|
228
|
+
if (codexHeartbeatTimer) return; // already started → don't leak a second timer
|
|
229
|
+
codexHeartbeatTimer = start();
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/**
|
|
233
|
+
* Production heartbeat start: schedule the codex tick with the two gh#861 gates
|
|
234
|
+
* wired from the stream loop's own state —
|
|
235
|
+
* - finding 2: `isStreamOwner` mirrors the per-entry path's lease-silencing so a
|
|
236
|
+
* lease-LOSING duplicate child never injects;
|
|
237
|
+
* - finding 3: `onAppServerSocketDead` tears the timer down when the tick finds
|
|
238
|
+
* the app-server socket gone, then schedules a one-shot deferred re-arm
|
|
239
|
+
* (gh#866 item 3) so a mid-session restart self-heals — see
|
|
240
|
+
* onCodexAppServerSocketDead.
|
|
241
|
+
*/
|
|
242
|
+
function defaultStartCodexHeartbeat(): ReturnType<typeof setInterval> | null {
|
|
243
|
+
return startCodexHeartbeat({
|
|
244
|
+
tick: () =>
|
|
245
|
+
void fireCodexHeartbeatTick({
|
|
246
|
+
isStreamOwner: () => streamState.ownership?.state === 'owner',
|
|
247
|
+
onAppServerSocketDead: onCodexAppServerSocketDead,
|
|
248
|
+
}),
|
|
249
|
+
});
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* gh#866 item 3: the tick's ENOENT (positively-dead app-server socket) teardown
|
|
254
|
+
* handler. The recurring timer is torn down (gh#861 finding 3 — don't keep
|
|
255
|
+
* ticking against a dead socket), BUT because the only other re-arm site is the
|
|
256
|
+
* runLoop top — unreachable while `streamOnce` holds the connected SSE session,
|
|
257
|
+
* which can outlive an app-server restart — we ALSO schedule a one-shot deferred
|
|
258
|
+
* re-arm. A transient ENOENT (codex app-server restart) therefore self-heals
|
|
259
|
+
* within the session instead of leaving the seat permanently deaf until the next
|
|
260
|
+
* reconnect. The re-armed timer's first tick re-probes: still dead → tears down +
|
|
261
|
+
* reschedules (a gentle one-cadence probe loop); alive → resumes normal cadence.
|
|
262
|
+
* Distinct from the runLoop cube-cleared teardown, which calls stopCodexHeartbeat
|
|
263
|
+
* directly and is correctly re-armed by runLoop when an active cube returns.
|
|
264
|
+
*/
|
|
265
|
+
function onCodexAppServerSocketDead(): void {
|
|
266
|
+
stopCodexHeartbeat();
|
|
267
|
+
scheduleCodexHeartbeatReArm();
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* Schedule a single deferred re-arm of the codex heartbeat (gh#866 item 3).
|
|
272
|
+
* Idempotent: a re-arm already pending is not stacked. The timer is unref'd so
|
|
273
|
+
* it never alone keeps the process alive.
|
|
274
|
+
*/
|
|
275
|
+
function scheduleCodexHeartbeatReArm(
|
|
276
|
+
delayMs: number = codexReArmDelayMs
|
|
277
|
+
): void {
|
|
278
|
+
if (codexReArmTimer) return; // a re-arm is already pending → don't stack
|
|
279
|
+
codexReArmTimer = setTimeout(() => {
|
|
280
|
+
codexReArmTimer = null;
|
|
281
|
+
ensureCodexHeartbeatStarted();
|
|
282
|
+
}, delayMs);
|
|
283
|
+
codexReArmTimer.unref?.();
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* gh#861 finding 3: tear down the codex heartbeat timer — the teardown seam for
|
|
288
|
+
* the periodic interval. Called when the active cube is cleared (nothing to inject
|
|
289
|
+
* into) or the tick detects a dead app-server socket. Also clears any pending
|
|
290
|
+
* deferred re-arm (gh#866 item 3) so a cube-cleared teardown doesn't leave a
|
|
291
|
+
* stray re-arm queued. Re-armable: ensureCodexHeartbeatStarted starts a fresh
|
|
292
|
+
* timer once an active cube returns.
|
|
293
|
+
*/
|
|
294
|
+
export function stopCodexHeartbeat(): void {
|
|
295
|
+
if (codexHeartbeatTimer) clearInterval(codexHeartbeatTimer);
|
|
296
|
+
codexHeartbeatTimer = null;
|
|
297
|
+
if (codexReArmTimer) clearTimeout(codexReArmTimer);
|
|
298
|
+
codexReArmTimer = null;
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/** Test-only alias of the production teardown seam (re-testable idempotence). */
|
|
302
|
+
export function __resetCodexHeartbeatForTest(): void {
|
|
303
|
+
stopCodexHeartbeat();
|
|
304
|
+
codexReArmDelayMs = CODEX_HEARTBEAT_CADENCE_MS;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Test-only override of the gh#866-item3 deferred re-arm delay so the
|
|
309
|
+
* mid-session re-arm lifecycle is drivable with a tiny real delay instead of the
|
|
310
|
+
* 20-minute cadence. Reset by __resetCodexHeartbeatForTest.
|
|
311
|
+
* @internal
|
|
312
|
+
*/
|
|
313
|
+
export function __setCodexReArmDelayForTest(ms: number): void {
|
|
314
|
+
codexReArmDelayMs = ms;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** The forever stream loop (extracted so startLogStream's wiring is testable). */
|
|
318
|
+
function runStreamLoopForever(): void {
|
|
319
|
+
void (async () => {
|
|
320
|
+
while (true) {
|
|
321
|
+
try {
|
|
322
|
+
await runLoop();
|
|
323
|
+
process.stderr.write('[borg-mcp log stream] runLoop returned unexpectedly; restarting in 5s\n');
|
|
324
|
+
} catch (err: any) {
|
|
325
|
+
process.stderr.write(`[borg-mcp log stream] runLoop threw: ${err?.message ?? err}; restarting in 5s\n`);
|
|
326
|
+
}
|
|
327
|
+
streamState.runLoopRestartCount += 1;
|
|
328
|
+
await sleep(5000);
|
|
329
|
+
}
|
|
330
|
+
})();
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
export function startLogStream(opts: { runForever?: () => void } = {}): void {
|
|
334
|
+
// gh#857 WI-2: start the codex heartbeat (independent periodic drain backstop),
|
|
335
|
+
// guarded so a re-entrant call can't leak a second interval. Lives for the
|
|
336
|
+
// child's life, runs regardless of SSE connection state.
|
|
337
|
+
ensureCodexHeartbeatStarted();
|
|
338
|
+
// The forever loop is injectable so tests can pin THIS wiring (the heartbeat
|
|
339
|
+
// start above) without spawning the real network/keychain loop (QA 75f18e8f).
|
|
340
|
+
(opts.runForever ?? runStreamLoopForever)();
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// ------------------------------------------------------------------
|
|
344
|
+
// Dependency injection seams (for tests)
|
|
345
|
+
// ------------------------------------------------------------------
|
|
346
|
+
|
|
347
|
+
export interface StreamDeps {
|
|
348
|
+
/** Override the global fetch (tests inject a controlled Response). */
|
|
349
|
+
fetchImpl?: typeof fetch;
|
|
350
|
+
/** Override persisted trust loading to verify pre-network confinement. */
|
|
351
|
+
loadTrust?: typeof loadBorgServerTrust;
|
|
352
|
+
/** Override stream cursor loading to verify pre-storage confinement. */
|
|
353
|
+
getCursor?: typeof getLocalServerCursor;
|
|
354
|
+
/** Override the inbox-line append (tests assert against the calls). */
|
|
355
|
+
appendLine?: (
|
|
356
|
+
cubeId: string,
|
|
357
|
+
droneId: string,
|
|
358
|
+
line: string
|
|
359
|
+
) => Promise<void>;
|
|
360
|
+
/** Override durable inbox-entry dedup (tests avoid real filesystem). */
|
|
361
|
+
hasInboxEntryId?: (
|
|
362
|
+
cubeId: string,
|
|
363
|
+
droneId: string,
|
|
364
|
+
entryId: string,
|
|
365
|
+
renderedLine: string
|
|
366
|
+
) => Promise<boolean>;
|
|
367
|
+
/** Optional Codex app-server wake sink; tests inject a spy. */
|
|
368
|
+
wakeCodex?: (reason: string) => void;
|
|
369
|
+
/** Override the heartbeat watchdog timeout. */
|
|
370
|
+
heartbeatTimeoutMs?: number;
|
|
371
|
+
/** Override HWM divergence grace for focused tests. */
|
|
372
|
+
hwmDivergenceGraceMs?: number;
|
|
373
|
+
/** Abort the active stream when ownership is lost by another process. */
|
|
374
|
+
abortSignal?: AbortSignal;
|
|
375
|
+
/** Override stream-owner lock state in focused duplicate-process tests. */
|
|
376
|
+
ownerDeps?: import('./stream-owner.js').StreamOwnerDeps;
|
|
377
|
+
/** Override owner stale threshold in focused duplicate-process tests. */
|
|
378
|
+
ownerStaleMs?: number;
|
|
379
|
+
/**
|
|
380
|
+
* Optional opencode entry injector for autonomous drone processing.
|
|
381
|
+
* When provided AND the injection succeeds, the inbox file write is skipped
|
|
382
|
+
* (the drone processes the entry autonomously). On failure, falls through
|
|
383
|
+
* to the inbox write for backup.
|
|
384
|
+
*/
|
|
385
|
+
injectOpenCode?: (text: string) => Promise<boolean>;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
const defaultDeps: Required<StreamDeps> = {
|
|
389
|
+
fetchImpl: globalThis.fetch.bind(globalThis),
|
|
390
|
+
loadTrust: loadBorgServerTrust,
|
|
391
|
+
getCursor: getLocalServerCursor,
|
|
392
|
+
appendLine: defaultAppendLine,
|
|
393
|
+
hasInboxEntryId: defaultHasInboxEntryId,
|
|
394
|
+
wakeCodex: wakeCodexViaAppServer,
|
|
395
|
+
heartbeatTimeoutMs: HEARTBEAT_TIMEOUT_MS,
|
|
396
|
+
hwmDivergenceGraceMs: HWM_DIVERGENCE_GRACE_MS,
|
|
397
|
+
abortSignal: new AbortController().signal,
|
|
398
|
+
ownerDeps: {},
|
|
399
|
+
ownerStaleMs: 70_000,
|
|
400
|
+
injectOpenCode: (text) => _moduleInjectOpenCode ? _moduleInjectOpenCode(text) : Promise.resolve(false),
|
|
401
|
+
};
|
|
402
|
+
|
|
403
|
+
// ------------------------------------------------------------------
|
|
404
|
+
// Outer reconnect loop
|
|
405
|
+
// ------------------------------------------------------------------
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Test-only injection seam for runLoop (gh#866 item 2). Production calls
|
|
409
|
+
* `runLoop()` with no args → every dep falls back to the real import and the
|
|
410
|
+
* loop runs forever, exactly as before (zero behavior change). Tests pass stubs
|
|
411
|
+
* plus a `maxIterations` bound to drive a fixed number of iterations through the
|
|
412
|
+
* heartbeat-lifecycle seams — teardown (`stopCodexHeartbeat` on a cleared cube)
|
|
413
|
+
* and re-arm (`ensureCodexHeartbeatStarted` on an active cube) — without real
|
|
414
|
+
* network/keychain IO.
|
|
415
|
+
*/
|
|
416
|
+
export interface RunLoopTestDeps {
|
|
417
|
+
getActiveCube?: typeof getActiveCube;
|
|
418
|
+
acquireStreamLease?: typeof acquireStreamLease;
|
|
419
|
+
streamOnce?: typeof streamOnce;
|
|
420
|
+
sleep?: (ms: number) => Promise<void>;
|
|
421
|
+
maxIterations?: number;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
async function runLoop(testDeps: RunLoopTestDeps = {}): Promise<void> {
|
|
425
|
+
const _getActiveCube = testDeps.getActiveCube ?? getActiveCube;
|
|
426
|
+
const _acquireStreamLease = testDeps.acquireStreamLease ?? acquireStreamLease;
|
|
427
|
+
const _streamOnce = testDeps.streamOnce ?? streamOnce;
|
|
428
|
+
const _sleep = testDeps.sleep ?? sleep;
|
|
429
|
+
const _maxIterations = testDeps.maxIterations ?? Infinity;
|
|
430
|
+
let _iterations = 0;
|
|
431
|
+
|
|
432
|
+
let attempt = 0;
|
|
433
|
+
let lastEventId: string | null = null;
|
|
434
|
+
let currentCubeId: string | null = null;
|
|
435
|
+
let lease: StreamLease | null = null;
|
|
436
|
+
let leaseKey: string | null = null;
|
|
437
|
+
|
|
438
|
+
try {
|
|
439
|
+
while (_iterations < _maxIterations) {
|
|
440
|
+
_iterations += 1;
|
|
441
|
+
const active = await _getActiveCube();
|
|
442
|
+
if (!active) {
|
|
443
|
+
if (lease) {
|
|
444
|
+
await lease.release();
|
|
445
|
+
lease = null;
|
|
446
|
+
leaseKey = null;
|
|
447
|
+
}
|
|
448
|
+
streamState.connected = false;
|
|
449
|
+
streamState.ownership = { state: 'unowned' };
|
|
450
|
+
// gh#861 finding 3: active cube cleared → tear down the codex heartbeat
|
|
451
|
+
// (nothing to inject into); re-armed below once an active cube returns.
|
|
452
|
+
stopCodexHeartbeat();
|
|
453
|
+
await _sleep(5000);
|
|
454
|
+
continue;
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// gh#861 finding 3: an active cube is present → ensure the codex heartbeat is
|
|
458
|
+
// running. Idempotent (no-op when already armed); re-arms after a prior
|
|
459
|
+
// teardown (cube-cleared / dead app-server socket).
|
|
460
|
+
ensureCodexHeartbeatStarted();
|
|
461
|
+
|
|
462
|
+
// Reset resume cursor on cube switch — entries from a prior cube
|
|
463
|
+
// mean nothing for the new cube's stream.
|
|
464
|
+
if (active.cubeId !== currentCubeId) {
|
|
465
|
+
currentCubeId = active.cubeId;
|
|
466
|
+
lastEventId = null;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
const nextLeaseKey = `${active.cubeId}:${active.droneId}`;
|
|
470
|
+
if (lease && leaseKey !== nextLeaseKey) {
|
|
471
|
+
await lease.release();
|
|
472
|
+
lease = null;
|
|
473
|
+
leaseKey = null;
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
if (!lease) {
|
|
477
|
+
lease = await _acquireStreamLease(
|
|
478
|
+
active.cubeId,
|
|
479
|
+
active.droneId,
|
|
480
|
+
STREAM_OWNER_STALE_MS,
|
|
481
|
+
{ isPidAlive: isProcessAlive },
|
|
482
|
+
);
|
|
483
|
+
leaseKey = lease ? nextLeaseKey : null;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
if (!lease) {
|
|
487
|
+
streamState.connected = false;
|
|
488
|
+
streamState.ownership = await readOwnershipSnapshot(active.cubeId, active.droneId);
|
|
489
|
+
await _sleep(5000);
|
|
490
|
+
continue;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
streamState.ownership = await readOwnershipSnapshot(active.cubeId, active.droneId);
|
|
494
|
+
|
|
495
|
+
let ownerLost = false;
|
|
496
|
+
try {
|
|
497
|
+
const ownerAbort = new AbortController();
|
|
498
|
+
const refresh = async () => {
|
|
499
|
+
try {
|
|
500
|
+
if (!(await lease!.refresh())) {
|
|
501
|
+
ownerLost = true;
|
|
502
|
+
ownerAbort.abort(new Error('stream ownership lost'));
|
|
503
|
+
}
|
|
504
|
+
} catch (err) {
|
|
505
|
+
ownerLost = true;
|
|
506
|
+
ownerAbort.abort(err instanceof Error ? err : new Error(String(err)));
|
|
507
|
+
}
|
|
508
|
+
};
|
|
509
|
+
const refreshTimer = setInterval(
|
|
510
|
+
() => {
|
|
511
|
+
void refresh();
|
|
512
|
+
},
|
|
513
|
+
Math.max(1000, Math.floor(HEARTBEAT_TIMEOUT_MS / 2))
|
|
514
|
+
);
|
|
515
|
+
try {
|
|
516
|
+
await _streamOnce(
|
|
517
|
+
active,
|
|
518
|
+
lastEventId,
|
|
519
|
+
(id) => {
|
|
520
|
+
lastEventId = id;
|
|
521
|
+
},
|
|
522
|
+
{ abortSignal: ownerAbort.signal }
|
|
523
|
+
);
|
|
524
|
+
} finally {
|
|
525
|
+
clearInterval(refreshTimer);
|
|
526
|
+
}
|
|
527
|
+
if (ownerLost) {
|
|
528
|
+
lease = null;
|
|
529
|
+
leaseKey = null;
|
|
530
|
+
streamState.connected = false;
|
|
531
|
+
streamState.ownership = await readOwnershipSnapshot(active.cubeId, active.droneId);
|
|
532
|
+
await _sleep(5000);
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
// Clean disconnect (e.g. server-side rollout). Reset backoff.
|
|
536
|
+
attempt = 0;
|
|
537
|
+
streamState.reconnectAttempts = 0;
|
|
538
|
+
} catch (err: any) {
|
|
539
|
+
if (ownerLost) {
|
|
540
|
+
lease = null;
|
|
541
|
+
leaseKey = null;
|
|
542
|
+
streamState.connected = false;
|
|
543
|
+
streamState.ownership = await readOwnershipSnapshot(active.cubeId, active.droneId);
|
|
544
|
+
await _sleep(5000);
|
|
545
|
+
continue;
|
|
546
|
+
}
|
|
547
|
+
// gh#877 Path-B (B25): an authoritative DRONE_EVICTED is TERMINAL — the
|
|
548
|
+
// seat is gone. Stop reconnecting (do NOT back off and retry forever
|
|
549
|
+
// against a dead seat); release the lease and return so the child's SSE
|
|
550
|
+
// loop quiesces cleanly. The agent's graceful shutdown (TaskStop Monitor,
|
|
551
|
+
// no /loop reschedule) is driven separately by the EVICTED tool-result it
|
|
552
|
+
// already received on the authed call that produced this verdict.
|
|
553
|
+
if (err instanceof DroneEvictedError) {
|
|
554
|
+
if (lease) await lease.release().catch(() => {});
|
|
555
|
+
lease = null;
|
|
556
|
+
leaseKey = null;
|
|
557
|
+
streamState.connected = false;
|
|
558
|
+
streamState.ownership = await readOwnershipSnapshot(active.cubeId, active.droneId);
|
|
559
|
+
process.stderr.write(
|
|
560
|
+
`[borg-mcp log stream] drone evicted — stream terminated (no reconnect).\n`
|
|
561
|
+
);
|
|
562
|
+
return;
|
|
563
|
+
}
|
|
564
|
+
streamState.connected = false;
|
|
565
|
+
const delay =
|
|
566
|
+
Math.min(RECONNECT_MIN_MS * 2 ** attempt, RECONNECT_MAX_MS) +
|
|
567
|
+
Math.random() * 500;
|
|
568
|
+
process.stderr.write(
|
|
569
|
+
`[borg-mcp log stream] reconnect in ${Math.round(delay)}ms: ${err?.message ?? err}\n`
|
|
570
|
+
);
|
|
571
|
+
attempt += 1;
|
|
572
|
+
streamState.reconnectAttempts = attempt;
|
|
573
|
+
await _sleep(delay);
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
} finally {
|
|
577
|
+
if (lease) await lease.release().catch(() => {});
|
|
578
|
+
streamState.connected = false;
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* Test-only entry to the bounded form of runLoop (gh#866 item 2). Drives a
|
|
584
|
+
* fixed number of iterations with injected deps so the heartbeat teardown
|
|
585
|
+
* (cleared cube → `stopCodexHeartbeat`) and re-arm (active cube →
|
|
586
|
+
* `ensureCodexHeartbeatStarted`) seams are exercisable without real
|
|
587
|
+
* network/keychain IO. Never called in production.
|
|
588
|
+
*
|
|
589
|
+
* @internal — test-only surface; mirrors the `__…ForTest` convention used by
|
|
590
|
+
* `__resetStreamStateForTest` / `__resetCodexHeartbeatForTest`.
|
|
591
|
+
*/
|
|
592
|
+
export function __runLoopForTest(testDeps: RunLoopTestDeps): Promise<void> {
|
|
593
|
+
return runLoop(testDeps);
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
// ------------------------------------------------------------------
|
|
597
|
+
// Single SSE session (exported for tests)
|
|
598
|
+
// ------------------------------------------------------------------
|
|
599
|
+
|
|
600
|
+
export interface ActiveCube {
|
|
601
|
+
cubeId: string;
|
|
602
|
+
droneId: string;
|
|
603
|
+
sessionToken: string;
|
|
604
|
+
apiUrl: string;
|
|
605
|
+
serverTrustIdentity?: string;
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
export async function streamOnce(
|
|
609
|
+
active: ActiveCube,
|
|
610
|
+
lastEventId: string | null,
|
|
611
|
+
onEventId: (id: string) => void,
|
|
612
|
+
deps: StreamDeps = {}
|
|
613
|
+
): Promise<void> {
|
|
614
|
+
const {
|
|
615
|
+
fetchImpl,
|
|
616
|
+
loadTrust,
|
|
617
|
+
getCursor,
|
|
618
|
+
appendLine,
|
|
619
|
+
hasInboxEntryId,
|
|
620
|
+
wakeCodex,
|
|
621
|
+
heartbeatTimeoutMs,
|
|
622
|
+
hwmDivergenceGraceMs,
|
|
623
|
+
abortSignal,
|
|
624
|
+
injectOpenCode,
|
|
625
|
+
} = { ...defaultDeps, ...deps };
|
|
626
|
+
|
|
627
|
+
assertUuidShape(active.cubeId, 'cube_id');
|
|
628
|
+
assertUuidShape(active.droneId, 'drone_id');
|
|
629
|
+
if (active.serverTrustIdentity === undefined) {
|
|
630
|
+
throw new Error('Selected Borg server authority state is missing or unreadable');
|
|
631
|
+
}
|
|
632
|
+
const token = active.sessionToken;
|
|
633
|
+
const headers: Record<string, string> = {
|
|
634
|
+
Authorization: `Bearer ${token}`,
|
|
635
|
+
Accept: 'text/event-stream',
|
|
636
|
+
};
|
|
637
|
+
let requestFetch = fetchImpl;
|
|
638
|
+
if (deps.fetchImpl === undefined) {
|
|
639
|
+
const trust = await loadTrust(active.apiUrl);
|
|
640
|
+
if (trust.identity !== active.serverTrustIdentity) {
|
|
641
|
+
throw new Error('Borg server trust identity changed; refusing the stream');
|
|
642
|
+
}
|
|
643
|
+
requestFetch = trust.fetchImpl;
|
|
644
|
+
}
|
|
645
|
+
const cursor = await getCursor({
|
|
646
|
+
origin: active.apiUrl,
|
|
647
|
+
trustIdentity: active.serverTrustIdentity,
|
|
648
|
+
cubeId: active.cubeId,
|
|
649
|
+
droneId: active.droneId,
|
|
650
|
+
purpose: 'stream',
|
|
651
|
+
});
|
|
652
|
+
const query = cursor ? `?cursor=${encodeURIComponent(encodeLocalServerCursor(cursor))}` : '';
|
|
653
|
+
const streamPath = `/api/cubes/${active.cubeId}/stream${query}`;
|
|
654
|
+
|
|
655
|
+
const ac = new AbortController();
|
|
656
|
+
const abortFromExternal = () => {
|
|
657
|
+
try {
|
|
658
|
+
ac.abort(abortSignal.reason ?? new Error('external abort'));
|
|
659
|
+
} catch {
|
|
660
|
+
// ignore
|
|
661
|
+
}
|
|
662
|
+
};
|
|
663
|
+
if (abortSignal.aborted) abortFromExternal();
|
|
664
|
+
abortSignal.addEventListener('abort', abortFromExternal, { once: true });
|
|
665
|
+
// Heartbeat watchdog: if no event of any type arrives within
|
|
666
|
+
// heartbeatTimeoutMs, abort the request so the outer loop reconnects.
|
|
667
|
+
let watchdog: NodeJS.Timeout | null = null;
|
|
668
|
+
const bumpWatchdog = () => {
|
|
669
|
+
if (watchdog) clearTimeout(watchdog);
|
|
670
|
+
watchdog = setTimeout(() => {
|
|
671
|
+
try {
|
|
672
|
+
ac.abort(new Error('heartbeat watchdog timeout'));
|
|
673
|
+
} catch {
|
|
674
|
+
// ignore
|
|
675
|
+
}
|
|
676
|
+
}, heartbeatTimeoutMs);
|
|
677
|
+
};
|
|
678
|
+
bumpWatchdog();
|
|
679
|
+
|
|
680
|
+
// Local mirror of the resume cursor, updated AFTER each successful
|
|
681
|
+
// disk write (or dedup-recognized replay). Heartbeat-hwm comparison
|
|
682
|
+
// reads this value.
|
|
683
|
+
let lastPersistedEventId: string | null = lastEventId;
|
|
684
|
+
let lastBroadcastHwm: BroadcastHwm | null = null;
|
|
685
|
+
let pendingHwmDivergence:
|
|
686
|
+
| { hwm: BroadcastHwm; timer: NodeJS.Timeout }
|
|
687
|
+
| null = null;
|
|
688
|
+
|
|
689
|
+
const clearPendingHwmDivergence = () => {
|
|
690
|
+
if (!pendingHwmDivergence) return;
|
|
691
|
+
clearTimeout(pendingHwmDivergence.timer);
|
|
692
|
+
pendingHwmDivergence = null;
|
|
693
|
+
};
|
|
694
|
+
|
|
695
|
+
// gh#402 replay-storm amplifier fix (c80b1aaa #1): advance the resume cursor
|
|
696
|
+
// MONOTONICALLY by (created_at, id). An out-of-order older broadcast (or a
|
|
697
|
+
// catchup replay of an older entry) must NOT regress lastPersistedEventId —
|
|
698
|
+
// a regressed resume cursor widens the next reconnect's catchup window and
|
|
699
|
+
// re-replays entries (the tail -F storm). Reuses compareBroadcastHwm
|
|
700
|
+
// ((created_at,id) tiebreak), the SAME key the server orders broadcasts by.
|
|
701
|
+
//
|
|
702
|
+
// FAIL-OPEN on a missing created_at: every real SSE payload carries a
|
|
703
|
+
// non-empty created_at (EnrichedEntry / ack / heartbeat-hwm all guarantee
|
|
704
|
+
// it server-side), so the guard engages for every production event. But an
|
|
705
|
+
// absent/empty created_at is NOT ordinally comparable (UUID ids aren't
|
|
706
|
+
// either — see the dedup branch), so we only BLOCK a regression we can
|
|
707
|
+
// PROVE: both the incoming entry and the current cursor carry a real
|
|
708
|
+
// created_at AND the incoming is older-or-equal. Otherwise ADVANCE —
|
|
709
|
+
// freezing the resume cursor on a fresh forward entry that happens to lack
|
|
710
|
+
// a created_at would itself widen the next reconnect's window (the very
|
|
711
|
+
// storm this fixes). Proven older-or-equal events are still recorded in
|
|
712
|
+
// recentIds for dedup but do not move the cursor or re-fire onEventId.
|
|
713
|
+
let lastPersistedHwm: BroadcastHwm | null = null;
|
|
714
|
+
const markEventPersisted = (id: string, createdAt: string) => {
|
|
715
|
+
const next: BroadcastHwm = { id, created_at: createdAt };
|
|
716
|
+
if (
|
|
717
|
+
lastPersistedHwm &&
|
|
718
|
+
createdAt &&
|
|
719
|
+
lastPersistedHwm.created_at &&
|
|
720
|
+
compareBroadcastHwm(next, lastPersistedHwm) <= 0
|
|
721
|
+
) {
|
|
722
|
+
return;
|
|
723
|
+
}
|
|
724
|
+
lastPersistedHwm = next;
|
|
725
|
+
lastPersistedEventId = id;
|
|
726
|
+
streamState.lastPersistedEventId = id;
|
|
727
|
+
onEventId(id);
|
|
728
|
+
};
|
|
729
|
+
|
|
730
|
+
const markBroadcastPersisted = (hwm: BroadcastHwm | null) => {
|
|
731
|
+
if (!hwm) return;
|
|
732
|
+
lastBroadcastHwm =
|
|
733
|
+
!lastBroadcastHwm || compareBroadcastHwm(hwm, lastBroadcastHwm) > 0
|
|
734
|
+
? hwm
|
|
735
|
+
: lastBroadcastHwm;
|
|
736
|
+
if (
|
|
737
|
+
pendingHwmDivergence &&
|
|
738
|
+
compareBroadcastHwm(lastBroadcastHwm, pendingHwmDivergence.hwm) >= 0
|
|
739
|
+
) {
|
|
740
|
+
clearPendingHwmDivergence();
|
|
741
|
+
}
|
|
742
|
+
};
|
|
743
|
+
|
|
744
|
+
const scheduleHwmDivergenceReconnect = (hwm: BroadcastHwm) => {
|
|
745
|
+
if (pendingHwmDivergence?.hwm.id === hwm.id) return;
|
|
746
|
+
clearPendingHwmDivergence();
|
|
747
|
+
const timer = setTimeout(() => {
|
|
748
|
+
if (lastBroadcastHwm && compareBroadcastHwm(lastBroadcastHwm, hwm) >= 0) {
|
|
749
|
+
clearPendingHwmDivergence();
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
try {
|
|
753
|
+
ac.abort(new Error('hwm divergence — reconnect for catchup'));
|
|
754
|
+
} catch {
|
|
755
|
+
// ignore
|
|
756
|
+
}
|
|
757
|
+
}, hwmDivergenceGraceMs);
|
|
758
|
+
pendingHwmDivergence = { hwm, timer };
|
|
759
|
+
};
|
|
760
|
+
|
|
761
|
+
// Bounded recent-id set for replay-on-reconnect dedup per spec §(3).
|
|
762
|
+
// Set + FIFO array for O(1) membership + bounded memory.
|
|
763
|
+
const recentIds = new Set<string>();
|
|
764
|
+
const recentIdsOrder: string[] = [];
|
|
765
|
+
let isCatchingUp = lastEventId !== null;
|
|
766
|
+
|
|
767
|
+
// gh#29 quality-stream (#5): shared inbox-write + cursor-advance helpers,
|
|
768
|
+
// extracted from the previously-duplicated ack / regular-log branches in the
|
|
769
|
+
// event loop below. Behavior-preserving — the per-branch comments document
|
|
770
|
+
// the load-bearing semantics each path relies on.
|
|
771
|
+
|
|
772
|
+
// Format + (catchup-dedup-aware) append the entry's inbox line.
|
|
773
|
+
// LOAD-BEARING ORDER: the disk write must complete before the cursor
|
|
774
|
+
// advances (recordSeen), so an append failure (disk full, EACCES, path race
|
|
775
|
+
// during cube switch) replays the entry on reconnect rather than being
|
|
776
|
+
// skipped past by an already-advanced Last-Event-ID — the §(3) durability
|
|
777
|
+
// contract. Returns 'persisted-skip' when the entry is already on disk from
|
|
778
|
+
// an earlier catchup receive (caller advances the cursor via markEventPersisted
|
|
779
|
+
// here, then continues WITHOUT re-recording); 'written' after a fresh append.
|
|
780
|
+
const writeInboxLine = async (
|
|
781
|
+
ev: Extract<ParsedEvent, { type: 'log' }>
|
|
782
|
+
): Promise<'persisted-skip' | 'written'> => {
|
|
783
|
+
const line = formatInboxLine(withSseEventId(ev.data, ev.id));
|
|
784
|
+
if (
|
|
785
|
+
isCatchingUp &&
|
|
786
|
+
// gh#441: pass the rendered line so the dedup can also recognize LEGACY
|
|
787
|
+
// (no-entry_id-prefix) on-disk lines, not just the [entry_id:] marker.
|
|
788
|
+
(await hasInboxEntryId(active.cubeId, active.droneId, ev.id, line))
|
|
789
|
+
) {
|
|
790
|
+
markEventPersisted(ev.id, ev.data?.created_at ?? '');
|
|
791
|
+
return 'persisted-skip';
|
|
792
|
+
}
|
|
793
|
+
// gh#opencode: try autonomous opencode injection first. When the drone's
|
|
794
|
+
// child session processes entries directly, skip the inbox write (the
|
|
795
|
+
// Monitor/tail-F path is unused for opencode). Falls through to inbox on
|
|
796
|
+
// failure so the entry is never lost.
|
|
797
|
+
if (await injectOpenCode(line)) {
|
|
798
|
+
return 'written';
|
|
799
|
+
}
|
|
800
|
+
await appendLine(active.cubeId, active.droneId, line);
|
|
801
|
+
wakeCodex(formatCodexWakePrompt(line));
|
|
802
|
+
return 'written';
|
|
803
|
+
};
|
|
804
|
+
|
|
805
|
+
// Record the event in the bounded recent-ids dedup set (FIFO-capped) and
|
|
806
|
+
// advance both cursors. The broadcast-HWM cursor advances via
|
|
807
|
+
// broadcastHwmFromLogEvent, which returns null for visibility==='direct' and
|
|
808
|
+
// kind==='ack' (those do NOT advance the server's DO broadcast HWM either —
|
|
809
|
+
// D6), so it advances ONLY for broadcast entries. gh#402 replay-storm fix
|
|
810
|
+
// (583aed7e): this runs for OWN-POST broadcasts too — the author's own
|
|
811
|
+
// broadcast IS counted by the server HWM, so the client broadcast cursor
|
|
812
|
+
// must advance to match; otherwise the next heartbeat reads server-hwm >
|
|
813
|
+
// client-cursor and fires a spurious divergence-reconnect (the storm
|
|
814
|
+
// trigger). The null-for-direct/ack guard keeps own-direct echoes correct.
|
|
815
|
+
const recordSeen = async (ev: Extract<ParsedEvent, { type: 'log' }>): Promise<void> => {
|
|
816
|
+
recentIds.add(ev.id);
|
|
817
|
+
recentIdsOrder.push(ev.id);
|
|
818
|
+
while (recentIdsOrder.length > RECENT_IDS_CAP) {
|
|
819
|
+
const oldId = recentIdsOrder.shift();
|
|
820
|
+
if (oldId) recentIds.delete(oldId);
|
|
821
|
+
}
|
|
822
|
+
markEventPersisted(ev.id, ev.data?.created_at ?? '');
|
|
823
|
+
markBroadcastPersisted(broadcastHwmFromLogEvent(ev));
|
|
824
|
+
if (ev.cursor) {
|
|
825
|
+
// client#41: delivery advances the STREAM cursor only. The unread
|
|
826
|
+
// watermark (read-log unread_only) is deliberately NOT touched here, so a
|
|
827
|
+
// wake-triggering entry stays unread until the agent drains it — SSE
|
|
828
|
+
// delivery must not consume the unread state (silent missed wake).
|
|
829
|
+
await advanceLocalServerCursor({
|
|
830
|
+
origin: active.apiUrl,
|
|
831
|
+
trustIdentity: active.serverTrustIdentity!,
|
|
832
|
+
cubeId: active.cubeId,
|
|
833
|
+
droneId: active.droneId,
|
|
834
|
+
purpose: 'stream',
|
|
835
|
+
}, ev.cursor);
|
|
836
|
+
}
|
|
837
|
+
};
|
|
838
|
+
|
|
839
|
+
let response: Response;
|
|
840
|
+
try {
|
|
841
|
+
response = await requestFetch(`${active.apiUrl}${streamPath}`, {
|
|
842
|
+
method: 'GET',
|
|
843
|
+
headers,
|
|
844
|
+
signal: ac.signal,
|
|
845
|
+
});
|
|
846
|
+
} catch (err) {
|
|
847
|
+
if (watchdog) clearTimeout(watchdog);
|
|
848
|
+
throw err;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
if (!response.ok || !response.body) {
|
|
852
|
+
if (watchdog) clearTimeout(watchdog);
|
|
853
|
+
// gh#877 Path-B (stream bootstrap): an evicted drone's stream re-subscribe
|
|
854
|
+
// returns the authoritative 410 DRONE_EVICTED. Surface it as the terminal
|
|
855
|
+
// typed error so the reconnect loop stops retrying (B25) instead of backing
|
|
856
|
+
// off forever against a dead seat. Keyed on the structured code, not the
|
|
857
|
+
// bare status (SEC R2). Any other status falls through to the generic throw
|
|
858
|
+
// so the reconnect loop keeps retrying transient failures.
|
|
859
|
+
if (response.status === 410) {
|
|
860
|
+
const body = await readBoundedResponseBody(
|
|
861
|
+
response,
|
|
862
|
+
LOCAL_SERVER_SSE_FRAME_LIMIT_BYTES,
|
|
863
|
+
'Local Borg server SSE response exceeded the response limit',
|
|
864
|
+
ac.signal,
|
|
865
|
+
).catch(() => '');
|
|
866
|
+
const code = errorCodeFromBody(body);
|
|
867
|
+
if (code === DRONE_EVICTED_CODE) {
|
|
868
|
+
throw new DroneEvictedError();
|
|
869
|
+
}
|
|
870
|
+
// client#42: an expired resume cursor is RECOVERABLE, not terminal. The
|
|
871
|
+
// pointed-at entry was pruned server-side, so retrying the SAME cursor
|
|
872
|
+
// 410s forever (a wedged, silently-dead wake path). Reset the stream's
|
|
873
|
+
// resume cursor to a fresh valid point (clear it → the next connect omits
|
|
874
|
+
// ?cursor= and the server re-establishes from the current tail), then
|
|
875
|
+
// throw the distinct recoverable error so the reconnect loop backs off and
|
|
876
|
+
// resumes — NOT the terminal DRONE_EVICTED path, NOT an infinite retry of
|
|
877
|
+
// the dead cursor. The unread watermark (client#41) is untouched, so no
|
|
878
|
+
// undrained wake is lost across the reset.
|
|
879
|
+
if (code === CURSOR_EXPIRED_CODE) {
|
|
880
|
+
await clearLocalServerCursor({
|
|
881
|
+
origin: active.apiUrl,
|
|
882
|
+
trustIdentity: active.serverTrustIdentity,
|
|
883
|
+
cubeId: active.cubeId,
|
|
884
|
+
droneId: active.droneId,
|
|
885
|
+
purpose: 'stream',
|
|
886
|
+
});
|
|
887
|
+
throw new StreamCursorExpiredError();
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
throw new Error(`stream HTTP ${response.status}`);
|
|
891
|
+
}
|
|
892
|
+
|
|
893
|
+
streamState.connected = true;
|
|
894
|
+
|
|
895
|
+
try {
|
|
896
|
+
for await (const event of parseSSE(
|
|
897
|
+
response.body,
|
|
898
|
+
LOCAL_SERVER_SSE_FRAME_LIMIT_BYTES,
|
|
899
|
+
)) {
|
|
900
|
+
bumpWatchdog();
|
|
901
|
+
const nowIso = new Date().toISOString();
|
|
902
|
+
streamState.lastWireActivityAt = nowIso;
|
|
903
|
+
// Content vs wire split (T1.2): content freshness is what a reader
|
|
904
|
+
// skimming the top-line verdict actually cares about. Heartbeats
|
|
905
|
+
// bump wire-activity only; log and bookmark events bump both.
|
|
906
|
+
if (event.type === 'log' || event.type === 'bookmark') {
|
|
907
|
+
streamState.lastContentEventAt = nowIso;
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
// gh#877 Path-A: terminal eviction control frame. Handled EARLY (before
|
|
911
|
+
// log/heartbeat) so a replayed frame on reconnect still fires. This is a
|
|
912
|
+
// WAKE HINT with ZERO authority (SEC R2): write a sentinel line to the
|
|
913
|
+
// inbox file so the (possibly asleep) agent's Monitor wakes it, then close
|
|
914
|
+
// this stream. The agent CONFIRMS via an authed call that returns 410
|
|
915
|
+
// DRONE_EVICTED before tearing down — a spurious frame on a live seat is
|
|
916
|
+
// inert (the confirm returns normally). We do NOT terminate the agent here
|
|
917
|
+
// (the client process cannot reach the agent loop); we only deliver the
|
|
918
|
+
// wake. The reconnect's stream-bootstrap 410 (authoritative) is what flips
|
|
919
|
+
// this loop terminal below.
|
|
920
|
+
if (event.type === 'eviction') {
|
|
921
|
+
streamState.lastContentEventAt = nowIso;
|
|
922
|
+
try {
|
|
923
|
+
await appendLine(
|
|
924
|
+
active.cubeId,
|
|
925
|
+
active.droneId,
|
|
926
|
+
formatEvictionSentinelLine(event.reason)
|
|
927
|
+
);
|
|
928
|
+
} catch {
|
|
929
|
+
// Inbox write failed — the Path-B 410 backstop still tears the drone
|
|
930
|
+
// down on its next authed call. Best-effort wake only.
|
|
931
|
+
}
|
|
932
|
+
// Close this SSE session; the reconnect will hit the authoritative 410.
|
|
933
|
+
break;
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
if (event.type === 'heartbeat') {
|
|
937
|
+
streamState.lastHeartbeatAt = nowIso;
|
|
938
|
+
// First/baseline heartbeat absorb: until this session has seen
|
|
939
|
+
// a broadcast entry, the server's broadcast HWM is our baseline.
|
|
940
|
+
// Direct messages may advance the persistence cursor past this
|
|
941
|
+
// value; directional comparison must not treat that mirror case
|
|
942
|
+
// as missed-broadcast evidence.
|
|
943
|
+
if (event.hwm && lastBroadcastHwm === null) {
|
|
944
|
+
markBroadcastPersisted(event.hwm);
|
|
945
|
+
if (lastPersistedEventId === null) {
|
|
946
|
+
markEventPersisted(event.hwm.id, event.hwm.created_at);
|
|
947
|
+
}
|
|
948
|
+
continue;
|
|
949
|
+
}
|
|
950
|
+
if (
|
|
951
|
+
event.hwm &&
|
|
952
|
+
lastBroadcastHwm &&
|
|
953
|
+
compareBroadcastHwm(event.hwm, lastBroadcastHwm) <= 0
|
|
954
|
+
) {
|
|
955
|
+
clearPendingHwmDivergence();
|
|
956
|
+
continue;
|
|
957
|
+
}
|
|
958
|
+
// §(5) divergence-detection: reconnect only when the server's
|
|
959
|
+
// broadcast HWM is AHEAD of the client's broadcast cursor. A
|
|
960
|
+
// recipient persisting a direct entry legitimately advances
|
|
961
|
+
// lastPersistedEventId beyond the broadcast HWM; strict
|
|
962
|
+
// inequality would reintroduce gh#402 churn for that mirror case.
|
|
963
|
+
if (
|
|
964
|
+
event.hwm &&
|
|
965
|
+
lastBroadcastHwm &&
|
|
966
|
+
compareBroadcastHwm(event.hwm, lastBroadcastHwm) > 0
|
|
967
|
+
) {
|
|
968
|
+
scheduleHwmDivergenceReconnect(event.hwm);
|
|
969
|
+
}
|
|
970
|
+
continue;
|
|
971
|
+
}
|
|
972
|
+
if (event.type === 'bookmark') {
|
|
973
|
+
isCatchingUp = false;
|
|
974
|
+
continue;
|
|
975
|
+
}
|
|
976
|
+
if (event.type === 'log') {
|
|
977
|
+
// DEDUP per §(3) recent-ids contract: an out-of-order DO
|
|
978
|
+
// broadcast followed by reconnect+catchup can replay an entry
|
|
979
|
+
// we already persisted. The entry IS on disk from an earlier
|
|
980
|
+
// receive in this session — just not from THIS iteration's
|
|
981
|
+
// appendLine. So we skip the duplicate write AND advance the
|
|
982
|
+
// cursor (so the heartbeat-hwm comparison converges and so
|
|
983
|
+
// Last-Event-ID on the next reconnect reflects the highest
|
|
984
|
+
// id we've actually got persisted). UUIDs are not ordinally
|
|
985
|
+
// comparable with created_at, so set membership is the check.
|
|
986
|
+
if (recentIds.has(event.id)) {
|
|
987
|
+
markEventPersisted(event.id, event.data?.created_at ?? '');
|
|
988
|
+
markBroadcastPersisted(broadcastHwmFromLogEvent(event));
|
|
989
|
+
continue;
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
// OWN-DRONE FILTER: restore the silent-self property — parity
|
|
993
|
+
// with pre-cutover inbox.ts:87-88. The DO broadcasts every
|
|
994
|
+
// entry to every connected drone INCLUDING the originator;
|
|
995
|
+
// without this skip, posting a log entry would wake the
|
|
996
|
+
// posting drone on its own message (visible via Monitor on
|
|
997
|
+
// the inbox file). The entry IS on disk (we wrote it via
|
|
998
|
+
// appendLog), so skip the inbox echo but still advance the
|
|
999
|
+
// cursor + record in recentIds so heartbeat-hwm comparison
|
|
1000
|
+
// converges and the next reconnect's Last-Event-ID reflects
|
|
1001
|
+
// the highest id we've actually got persisted. Structurally
|
|
1002
|
+
// identical to the dedup branch above — same "skip write,
|
|
1003
|
+
// advance state" shape, different trigger condition.
|
|
1004
|
+
//
|
|
1005
|
+
// HEARTBEAT-PING CARVE-OUT (gh#71): the gh#39 cron watchdog
|
|
1006
|
+
// authors heartbeat-pings WITH the silent target as drone_id
|
|
1007
|
+
// so each ping is attributed to the drone it intends to wake.
|
|
1008
|
+
// Without this carve-out, the own-drone filter would silently
|
|
1009
|
+
// skip the target's own ping → inbox file never written →
|
|
1010
|
+
// Monitor never fires → the platform-level wake guarantee is
|
|
1011
|
+
// broken for the cube-wide-silent class gh#39 was designed to
|
|
1012
|
+
// prevent. We let heartbeat-pings authored "by" the target
|
|
1013
|
+
// drone through the disk-write path; the existing rate-limit
|
|
1014
|
+
// in the server heartbeat contract (max 1 ping per drone per ~1h)
|
|
1015
|
+
// bounds the silent-self property's relaxation.
|
|
1016
|
+
const isHeartbeatPing =
|
|
1017
|
+
typeof event.data?.message === 'string' &&
|
|
1018
|
+
event.data.message.startsWith('[HEARTBEAT-PING]');
|
|
1019
|
+
|
|
1020
|
+
// Sprint 26 ack-fan-out: ack events have `kind: 'ack'` plus an
|
|
1021
|
+
// `author_drone_id` field naming the recipient (the author of
|
|
1022
|
+
// the entry that got acked). Only the author writes the ack
|
|
1023
|
+
// line to their inbox — all other subscribers drop the event
|
|
1024
|
+
// but still advance the cursor. The legacy entry-shaped fields
|
|
1025
|
+
// on the ack payload exist for pre-Sprint-26 clients that
|
|
1026
|
+
// don't recognize the `kind` discriminator; new clients route
|
|
1027
|
+
// here BEFORE the legacy own-drone filter so the ack-specific
|
|
1028
|
+
// semantic takes precedence.
|
|
1029
|
+
if (event.data?.kind === 'ack') {
|
|
1030
|
+
if (event.data?.author_drone_id === active.droneId) {
|
|
1031
|
+
if ((await writeInboxLine(event)) === 'persisted-skip') continue;
|
|
1032
|
+
}
|
|
1033
|
+
await recordSeen(event);
|
|
1034
|
+
continue;
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
// Local-server ActivityHub publishes the transparent shared log to
|
|
1038
|
+
// every cube stream. `visibility:direct` is routing/wake metadata, not
|
|
1039
|
+
// a read ACL: borg_read-log still returns the entry to every member,
|
|
1040
|
+
// while only an addressed drone writes it to its inbox and wakes. A
|
|
1041
|
+
// malformed direct frame with no matching recipient fails closed in
|
|
1042
|
+
// the wake direction (cursor advances; no cube-wide wake storm).
|
|
1043
|
+
if (event.data?.visibility === 'direct') {
|
|
1044
|
+
const recipients = Array.isArray(event.data?.recipient_drone_ids)
|
|
1045
|
+
? event.data.recipient_drone_ids.filter(
|
|
1046
|
+
(recipient: unknown): recipient is string => typeof recipient === 'string',
|
|
1047
|
+
)
|
|
1048
|
+
: [];
|
|
1049
|
+
if (!recipients.includes(active.droneId)) {
|
|
1050
|
+
await recordSeen(event);
|
|
1051
|
+
continue;
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
if (event.data?.drone_id === active.droneId && !isHeartbeatPing) {
|
|
1056
|
+
// Own post: silent-self (no inbox echo — already on disk via
|
|
1057
|
+
// appendLog). recordSeen still advances BOTH cursors, including the
|
|
1058
|
+
// broadcast cursor for an own broadcast — see recordSeen's gh#402
|
|
1059
|
+
// (583aed7e) note for why skipping it here would storm.
|
|
1060
|
+
await recordSeen(event);
|
|
1061
|
+
continue;
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
// Regular inbound entry: write the inbox line (catchup-dedup aware),
|
|
1065
|
+
// then advance both cursors.
|
|
1066
|
+
if ((await writeInboxLine(event)) === 'persisted-skip') continue;
|
|
1067
|
+
await recordSeen(event);
|
|
1068
|
+
}
|
|
1069
|
+
}
|
|
1070
|
+
} finally {
|
|
1071
|
+
abortSignal.removeEventListener('abort', abortFromExternal);
|
|
1072
|
+
if (watchdog) clearTimeout(watchdog);
|
|
1073
|
+
clearPendingHwmDivergence();
|
|
1074
|
+
streamState.connected = false;
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
export async function streamOnceIfOwner(
|
|
1079
|
+
active: ActiveCube,
|
|
1080
|
+
lastEventId: string | null,
|
|
1081
|
+
onEventId: (id: string) => void,
|
|
1082
|
+
deps: StreamDeps = {}
|
|
1083
|
+
): Promise<'streamed' | 'skipped'> {
|
|
1084
|
+
const { ownerDeps, ownerStaleMs } = { ...defaultDeps, ...deps };
|
|
1085
|
+
const lease = await acquireStreamLease(
|
|
1086
|
+
active.cubeId,
|
|
1087
|
+
active.droneId,
|
|
1088
|
+
ownerStaleMs,
|
|
1089
|
+
ownerDeps
|
|
1090
|
+
);
|
|
1091
|
+
if (!lease) {
|
|
1092
|
+
streamState.connected = false;
|
|
1093
|
+
streamState.ownership = await readOwnershipSnapshot(
|
|
1094
|
+
active.cubeId,
|
|
1095
|
+
active.droneId,
|
|
1096
|
+
ownerDeps
|
|
1097
|
+
);
|
|
1098
|
+
return 'skipped';
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
streamState.ownership = await readOwnershipSnapshot(
|
|
1102
|
+
active.cubeId,
|
|
1103
|
+
active.droneId,
|
|
1104
|
+
ownerDeps
|
|
1105
|
+
);
|
|
1106
|
+
try {
|
|
1107
|
+
await streamOnce(active, lastEventId, onEventId, deps);
|
|
1108
|
+
return 'streamed';
|
|
1109
|
+
} finally {
|
|
1110
|
+
await lease.release();
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
// ------------------------------------------------------------------
|
|
1115
|
+
// SSE event parser
|
|
1116
|
+
// ------------------------------------------------------------------
|
|
1117
|
+
|
|
1118
|
+
export type ParsedEvent =
|
|
1119
|
+
| { type: 'log'; id: string; data: any; cursor?: LocalServerCursor }
|
|
1120
|
+
| { type: 'heartbeat'; ts: string | null; hwm: BroadcastHwm | null }
|
|
1121
|
+
| { type: 'bookmark'; as_of: string | null }
|
|
1122
|
+
// gh#877 Path-A: terminal eviction control frame (wake hint, zero authority).
|
|
1123
|
+
| { type: 'eviction'; cube_id: string | null; reason: string | null }
|
|
1124
|
+
| { type: 'unknown'; raw: string };
|
|
1125
|
+
|
|
1126
|
+
/**
|
|
1127
|
+
* Async generator over an SSE response body. Yields one ParsedEvent
|
|
1128
|
+
* per "event:/data:" block (separated by blank lines per RFC 5234).
|
|
1129
|
+
*
|
|
1130
|
+
* Exported so tests can pump a synthetic ReadableStream through it.
|
|
1131
|
+
*/
|
|
1132
|
+
export async function* parseSSE(
|
|
1133
|
+
body: ReadableStream<Uint8Array>,
|
|
1134
|
+
maxFrameBytes?: number,
|
|
1135
|
+
): AsyncGenerator<ParsedEvent> {
|
|
1136
|
+
const reader = body.getReader();
|
|
1137
|
+
const decoder = new TextDecoder();
|
|
1138
|
+
let buffer = '';
|
|
1139
|
+
let bufferBytes = 0;
|
|
1140
|
+
|
|
1141
|
+
const appendFragment = (fragment: string) => {
|
|
1142
|
+
const nextBytes = bufferBytes + Buffer.byteLength(fragment, 'utf8');
|
|
1143
|
+
if (maxFrameBytes !== undefined && nextBytes > maxFrameBytes) {
|
|
1144
|
+
throw new Error('Local Borg server SSE frame exceeded the response limit');
|
|
1145
|
+
}
|
|
1146
|
+
buffer += fragment;
|
|
1147
|
+
bufferBytes = nextBytes;
|
|
1148
|
+
};
|
|
1149
|
+
|
|
1150
|
+
const drainText = function* (text: string): Generator<ParsedEvent> {
|
|
1151
|
+
let remaining = text;
|
|
1152
|
+
while (remaining.length > 0) {
|
|
1153
|
+
// Preserve a terminator split across two stream chunks without ever
|
|
1154
|
+
// allowing the retained partial frame itself to grow past the cap.
|
|
1155
|
+
if (buffer.endsWith('\n') && remaining.startsWith('\n')) {
|
|
1156
|
+
buffer = buffer.slice(0, -1);
|
|
1157
|
+
bufferBytes = Buffer.byteLength(buffer, 'utf8');
|
|
1158
|
+
remaining = remaining.slice(1);
|
|
1159
|
+
const parsed = parseEventBlock(buffer);
|
|
1160
|
+
buffer = '';
|
|
1161
|
+
bufferBytes = 0;
|
|
1162
|
+
if (parsed) yield parsed;
|
|
1163
|
+
continue;
|
|
1164
|
+
}
|
|
1165
|
+
|
|
1166
|
+
const idx = remaining.indexOf('\n\n');
|
|
1167
|
+
if (idx === -1) {
|
|
1168
|
+
appendFragment(remaining);
|
|
1169
|
+
return;
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
appendFragment(remaining.slice(0, idx));
|
|
1173
|
+
remaining = remaining.slice(idx + 2);
|
|
1174
|
+
const parsed = parseEventBlock(buffer);
|
|
1175
|
+
buffer = '';
|
|
1176
|
+
bufferBytes = 0;
|
|
1177
|
+
if (parsed) yield parsed;
|
|
1178
|
+
}
|
|
1179
|
+
};
|
|
1180
|
+
|
|
1181
|
+
try {
|
|
1182
|
+
while (true) {
|
|
1183
|
+
const { value, done } = await reader.read();
|
|
1184
|
+
if (done) {
|
|
1185
|
+
const tail = decoder.decode();
|
|
1186
|
+
if (tail) {
|
|
1187
|
+
for (const parsed of drainText(tail)) yield parsed;
|
|
1188
|
+
}
|
|
1189
|
+
if (buffer.trim()) {
|
|
1190
|
+
const parsed = parseEventBlock(buffer);
|
|
1191
|
+
if (parsed) yield parsed;
|
|
1192
|
+
}
|
|
1193
|
+
return;
|
|
1194
|
+
}
|
|
1195
|
+
for (const parsed of drainText(decoder.decode(value, { stream: true }))) {
|
|
1196
|
+
yield parsed;
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
} catch (error) {
|
|
1200
|
+
await reader.cancel(error).catch(() => {});
|
|
1201
|
+
throw error;
|
|
1202
|
+
} finally {
|
|
1203
|
+
try {
|
|
1204
|
+
reader.releaseLock();
|
|
1205
|
+
} catch {
|
|
1206
|
+
// ignore — stream may already be closed
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
function parseEventBlock(block: string): ParsedEvent | null {
|
|
1212
|
+
let eventName: string | null = null;
|
|
1213
|
+
let id: string | null = null;
|
|
1214
|
+
let dataLines: string[] = [];
|
|
1215
|
+
for (const line of block.split('\n')) {
|
|
1216
|
+
if (line.startsWith('event:')) {
|
|
1217
|
+
eventName = line.slice(6).trim();
|
|
1218
|
+
} else if (line.startsWith('id:')) {
|
|
1219
|
+
id = line.slice(3).trim();
|
|
1220
|
+
} else if (line.startsWith('data:')) {
|
|
1221
|
+
dataLines.push(line.slice(5).trim());
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
const dataStr = dataLines.join('\n');
|
|
1225
|
+
if (!eventName) return null;
|
|
1226
|
+
|
|
1227
|
+
if (eventName === 'log') {
|
|
1228
|
+
if (!id) return null;
|
|
1229
|
+
let parsed: any;
|
|
1230
|
+
try {
|
|
1231
|
+
parsed = JSON.parse(dataStr);
|
|
1232
|
+
} catch {
|
|
1233
|
+
return null;
|
|
1234
|
+
}
|
|
1235
|
+
const cursor = parsed?.cursor;
|
|
1236
|
+
const validCursor = cursor &&
|
|
1237
|
+
typeof cursor.id === 'string' &&
|
|
1238
|
+
typeof cursor.created_at === 'string';
|
|
1239
|
+
return {
|
|
1240
|
+
type: 'log',
|
|
1241
|
+
id,
|
|
1242
|
+
data: parsed?.entry ?? parsed,
|
|
1243
|
+
...(validCursor ? { cursor } : {}),
|
|
1244
|
+
};
|
|
1245
|
+
}
|
|
1246
|
+
if (eventName === 'heartbeat') {
|
|
1247
|
+
let ts: string | null = null;
|
|
1248
|
+
let hwm: BroadcastHwm | null = null;
|
|
1249
|
+
try {
|
|
1250
|
+
const parsed = JSON.parse(dataStr);
|
|
1251
|
+
ts = typeof parsed.ts === 'string' ? parsed.ts : null;
|
|
1252
|
+
hwm = parseHeartbeatHwm(parsed.hwm);
|
|
1253
|
+
} catch {
|
|
1254
|
+
// fall through with nulls
|
|
1255
|
+
}
|
|
1256
|
+
return { type: 'heartbeat', ts, hwm };
|
|
1257
|
+
}
|
|
1258
|
+
if (eventName === 'bookmark') {
|
|
1259
|
+
let as_of: string | null = null;
|
|
1260
|
+
try {
|
|
1261
|
+
const parsed = JSON.parse(dataStr);
|
|
1262
|
+
as_of = typeof parsed.as_of === 'string' ? parsed.as_of : null;
|
|
1263
|
+
} catch {
|
|
1264
|
+
// fall through with null
|
|
1265
|
+
}
|
|
1266
|
+
return { type: 'bookmark', as_of };
|
|
1267
|
+
}
|
|
1268
|
+
// gh#877 Path-A: terminal eviction control frame. Recognized explicitly so a
|
|
1269
|
+
// forged/garbage event type still falls through to `unknown` and no-ops (the
|
|
1270
|
+
// SEC R5 parser-default-ignore property is preserved).
|
|
1271
|
+
if (eventName === 'eviction') {
|
|
1272
|
+
let cube_id: string | null = null;
|
|
1273
|
+
let reason: string | null = null;
|
|
1274
|
+
try {
|
|
1275
|
+
const parsed = JSON.parse(dataStr);
|
|
1276
|
+
cube_id = typeof parsed.cube_id === 'string' ? parsed.cube_id : null;
|
|
1277
|
+
reason = typeof parsed.reason === 'string' ? parsed.reason : null;
|
|
1278
|
+
} catch {
|
|
1279
|
+
// fall through with nulls
|
|
1280
|
+
}
|
|
1281
|
+
return { type: 'eviction', cube_id, reason };
|
|
1282
|
+
}
|
|
1283
|
+
return { type: 'unknown', raw: block };
|
|
1284
|
+
}
|
|
1285
|
+
|
|
1286
|
+
// ------------------------------------------------------------------
|
|
1287
|
+
// Helpers
|
|
1288
|
+
// ------------------------------------------------------------------
|
|
1289
|
+
|
|
1290
|
+
export interface EnrichedEntry {
|
|
1291
|
+
id?: string;
|
|
1292
|
+
entry_id?: string;
|
|
1293
|
+
created_at?: string;
|
|
1294
|
+
drone_label?: string | null;
|
|
1295
|
+
role_name?: string | null;
|
|
1296
|
+
message?: string;
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
function parseHeartbeatHwm(value: unknown): BroadcastHwm | null {
|
|
1300
|
+
if (!value || typeof value !== 'object') return null;
|
|
1301
|
+
const candidate = value as { id?: unknown; created_at?: unknown };
|
|
1302
|
+
return typeof candidate.id === 'string' &&
|
|
1303
|
+
candidate.id.length > 0 &&
|
|
1304
|
+
typeof candidate.created_at === 'string' &&
|
|
1305
|
+
candidate.created_at.length > 0
|
|
1306
|
+
? { id: candidate.id, created_at: candidate.created_at }
|
|
1307
|
+
: null;
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
function broadcastHwmFromLogEvent(
|
|
1311
|
+
event: Extract<ParsedEvent, { type: 'log' }>
|
|
1312
|
+
): BroadcastHwm | null {
|
|
1313
|
+
if (event.data?.visibility === 'direct' || event.data?.kind === 'ack') {
|
|
1314
|
+
return null;
|
|
1315
|
+
}
|
|
1316
|
+
const createdAt = event.data?.created_at;
|
|
1317
|
+
return typeof createdAt === 'string' && createdAt.length > 0
|
|
1318
|
+
? { id: event.id, created_at: createdAt }
|
|
1319
|
+
: null;
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
function withSseEventId(entry: any, eventId: string): EnrichedEntry {
|
|
1323
|
+
if (!entry || typeof entry !== 'object') return { id: eventId };
|
|
1324
|
+
if (typeof entry.id === 'string' && entry.id.length > 0) return entry;
|
|
1325
|
+
return { ...entry, id: eventId };
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
/**
|
|
1329
|
+
* First argument that is a non-empty string, else ''. Used to pick the
|
|
1330
|
+
* inbox entry id from the server `id` field, falling back to the legacy
|
|
1331
|
+
* `entry_id` field. Flattens what was a nested ternary at the call site.
|
|
1332
|
+
*/
|
|
1333
|
+
function firstNonEmptyString(...candidates: unknown[]): string {
|
|
1334
|
+
for (const candidate of candidates) {
|
|
1335
|
+
if (typeof candidate === 'string' && candidate.length > 0) return candidate;
|
|
1336
|
+
}
|
|
1337
|
+
return '';
|
|
1338
|
+
}
|
|
1339
|
+
|
|
1340
|
+
/**
|
|
1341
|
+
* Format one inbox-file line. Preserves the long-poll inbox.ts prefix
|
|
1342
|
+
* shape (`<iso-ts> <drone-label> (<role-name>): ...`) so existing
|
|
1343
|
+
* Monitor tail consumers and audit tooling keep parsing it.
|
|
1344
|
+
*
|
|
1345
|
+
* Newlines in the message body are joined to ` ⏎ ` (U+23CE Return Symbol,
|
|
1346
|
+
* space-padded) so the entire entry is one physical line in the inbox
|
|
1347
|
+
* file. Rationale: Claude Code's Monitor primitive consumes stdout via
|
|
1348
|
+
* Node `readline` — every `\n` creates a NEW notification event, so a
|
|
1349
|
+
* multi-line cube-log entry would fire multiple notifications with only
|
|
1350
|
+
* the first line being a recognized entry-start (the rest dropped by
|
|
1351
|
+
* `formatEventLine`'s regex). Joining at write time delivers the full
|
|
1352
|
+
* entry content in a single notification body. Drones recognize ⏎ as
|
|
1353
|
+
* "newline was here" — convention noted in the regen-format playbook.
|
|
1354
|
+
*/
|
|
1355
|
+
export function formatInboxLine(entry: EnrichedEntry): string {
|
|
1356
|
+
const ts =
|
|
1357
|
+
typeof entry.created_at === 'string'
|
|
1358
|
+
? new Date(entry.created_at).toISOString()
|
|
1359
|
+
: new Date().toISOString();
|
|
1360
|
+
const label = entry.drone_label ?? '?';
|
|
1361
|
+
const role = entry.role_name ?? '?';
|
|
1362
|
+
const rawMessage =
|
|
1363
|
+
typeof entry.message === 'string' ? entry.message : '';
|
|
1364
|
+
// Server `id`, else legacy `entry_id`, else '' (behavior-identical to the
|
|
1365
|
+
// prior nested ternary).
|
|
1366
|
+
const entryId = firstNonEmptyString(entry.id, entry.entry_id);
|
|
1367
|
+
const idPrefix = entryId ? `[entry_id: ${entryId}] ` : '';
|
|
1368
|
+
// Normalize \r\n, \r, and \n all to ` ⏎ ` so the entry fits on one
|
|
1369
|
+
// physical line regardless of line-ending convention in the source.
|
|
1370
|
+
const message = rawMessage.replace(/\r\n|\r|\n/g, ' ⏎ ');
|
|
1371
|
+
return `${ts} ${label} (${role}): ${idPrefix}${message}`;
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
/**
|
|
1375
|
+
* gh#877 Path-A: format the terminal-eviction WAKE SENTINEL line written to the
|
|
1376
|
+
* inbox file. Same one-physical-line shape as formatInboxLine so the existing
|
|
1377
|
+
* `borg-inbox-monitor` tail recognizes it and fires a notification that wakes
|
|
1378
|
+
* the agent. The `[CUBE-EVICTED]` marker is a WAKE HINT only — the agent must
|
|
1379
|
+
* CONFIRM via an authed borg_* call returning 410 DRONE_EVICTED before tearing
|
|
1380
|
+
* down (so a peer posting sentinel-shaped log text can't force a false
|
|
1381
|
+
* shutdown of a live seat — SEC R2 / QA log-content-plane forgery guard).
|
|
1382
|
+
*/
|
|
1383
|
+
export function formatEvictionSentinelLine(reason: string | null): string {
|
|
1384
|
+
const ts = new Date().toISOString();
|
|
1385
|
+
const detail = reason && reason.trim().length > 0 ? reason : 'evicted from cube';
|
|
1386
|
+
return (
|
|
1387
|
+
`${ts} SYSTEM (eviction): ${EVICTED_RESULT_MARKER} ${detail} — ` +
|
|
1388
|
+
'confirm with any borg_* call; on typed DRONE_EVICTED, stop this session without retrying or restarting the loop.'
|
|
1389
|
+
);
|
|
1390
|
+
}
|
|
1391
|
+
|
|
1392
|
+
async function defaultAppendLine(
|
|
1393
|
+
cubeId: string,
|
|
1394
|
+
droneId: string,
|
|
1395
|
+
line: string
|
|
1396
|
+
): Promise<void> {
|
|
1397
|
+
const p = inboxPathForDrone(cubeId, droneId);
|
|
1398
|
+
await appendCappedInboxLine(p, line, INBOX_TAIL_LINES_CAP);
|
|
1399
|
+
}
|
|
1400
|
+
|
|
1401
|
+
export async function appendCappedInboxLine(
|
|
1402
|
+
inboxPath: string,
|
|
1403
|
+
line: string,
|
|
1404
|
+
maxLines = INBOX_TAIL_LINES_CAP,
|
|
1405
|
+
trimThresholdLines = maxLines * 2
|
|
1406
|
+
): Promise<void> {
|
|
1407
|
+
await fs.mkdir(path.dirname(inboxPath), { recursive: true });
|
|
1408
|
+
await fs.appendFile(inboxPath, line + '\n', 'utf-8');
|
|
1409
|
+
await trimInboxFileToRecentLines(inboxPath, maxLines, trimThresholdLines);
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
export async function trimInboxFileToRecentLines(
|
|
1413
|
+
inboxPath: string,
|
|
1414
|
+
maxLines: number,
|
|
1415
|
+
trimThresholdLines = maxLines
|
|
1416
|
+
): Promise<void> {
|
|
1417
|
+
if (!Number.isInteger(maxLines) || maxLines < 1) {
|
|
1418
|
+
throw new Error('maxLines must be a positive integer');
|
|
1419
|
+
}
|
|
1420
|
+
if (
|
|
1421
|
+
!Number.isInteger(trimThresholdLines) ||
|
|
1422
|
+
trimThresholdLines < maxLines
|
|
1423
|
+
) {
|
|
1424
|
+
throw new Error('trimThresholdLines must be an integer >= maxLines');
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
const raw = await fs.readFile(inboxPath, 'utf-8');
|
|
1428
|
+
const lines = raw.split(/\r?\n/);
|
|
1429
|
+
if (lines.at(-1) === '') lines.pop();
|
|
1430
|
+
if (lines.length <= trimThresholdLines) return;
|
|
1431
|
+
|
|
1432
|
+
const kept = lines.slice(-maxLines);
|
|
1433
|
+
const tmpPath = path.join(
|
|
1434
|
+
path.dirname(inboxPath),
|
|
1435
|
+
`.${path.basename(inboxPath)}.${process.pid}.${Date.now()}.tmp`
|
|
1436
|
+
);
|
|
1437
|
+
try {
|
|
1438
|
+
await fs.writeFile(tmpPath, kept.join('\n') + '\n', 'utf-8');
|
|
1439
|
+
await fs.rename(tmpPath, inboxPath);
|
|
1440
|
+
} catch (err) {
|
|
1441
|
+
await fs.rm(tmpPath, { force: true });
|
|
1442
|
+
throw err;
|
|
1443
|
+
}
|
|
1444
|
+
}
|
|
1445
|
+
|
|
1446
|
+
/**
|
|
1447
|
+
* @internal Exported for unit tests.
|
|
1448
|
+
*
|
|
1449
|
+
* Decide whether `raw` (the full inbox-file contents) already contains the
|
|
1450
|
+
* entry identified by `entryId` / rendered as `renderedLine`. Robust to BOTH
|
|
1451
|
+
* on-disk line formats (gh#441 — catchup-replay-flood fix):
|
|
1452
|
+
* - NEW format (0.9.39+): `…: [entry_id: <id>] <message>` — matched by the
|
|
1453
|
+
* `[entry_id: <id>]` marker (the #412 dedup, preserved verbatim).
|
|
1454
|
+
* - LEGACY format (pre-0.9.39 / old ACK lines): `…: <message>` with NO
|
|
1455
|
+
* entry_id prefix — matched by an EXACT, line-anchored comparison against
|
|
1456
|
+
* the entry's legacy rendering (`renderedLine` with the id-prefix stripped).
|
|
1457
|
+
*
|
|
1458
|
+
* Why this exists: a worker DEPLOY evicts the LogBroadcaster DO → every drone's
|
|
1459
|
+
* SSE drops → fleet-wide simultaneous reconnect → catchup. The old substring
|
|
1460
|
+
* check (`raw.includes('[entry_id: <id>]')`) missed legacy lines, so catchup
|
|
1461
|
+
* re-appended them → tail -F replay flood.
|
|
1462
|
+
*
|
|
1463
|
+
* The legacy match is LINE-ANCHORED (exact line equality), NOT a substring
|
|
1464
|
+
* match, on purpose: a substring match would false-positive-DROP a genuinely
|
|
1465
|
+
* new entry whose message merely EXTENDS an existing line (e.g. `…: hello` vs
|
|
1466
|
+
* `…: hello world`). A dropped entry is worse than a re-append — the drone
|
|
1467
|
+
* misses a wake — so when in doubt this fails toward re-append, never drop.
|
|
1468
|
+
*/
|
|
1469
|
+
export function inboxRawHasEntry(
|
|
1470
|
+
raw: string,
|
|
1471
|
+
entryId: string,
|
|
1472
|
+
renderedLine: string
|
|
1473
|
+
): boolean {
|
|
1474
|
+
// New-format: entry_id marker (preserves the #412 dedup).
|
|
1475
|
+
if (entryId && raw.includes(`[entry_id: ${entryId}]`)) return true;
|
|
1476
|
+
// Legacy-format: exact, line-anchored match against the id-prefix-stripped
|
|
1477
|
+
// rendering. `String.prototype.replace` with a string pattern strips only the
|
|
1478
|
+
// first (sole) prefix occurrence; entryId is treated literally (not regex).
|
|
1479
|
+
const legacyForm = entryId
|
|
1480
|
+
? renderedLine.replace(`[entry_id: ${entryId}] `, '')
|
|
1481
|
+
: renderedLine;
|
|
1482
|
+
if (legacyForm && raw.split(/\r?\n/).includes(legacyForm)) return true;
|
|
1483
|
+
return false;
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
async function defaultHasInboxEntryId(
|
|
1487
|
+
cubeId: string,
|
|
1488
|
+
droneId: string,
|
|
1489
|
+
entryId: string,
|
|
1490
|
+
renderedLine: string
|
|
1491
|
+
): Promise<boolean> {
|
|
1492
|
+
const p = inboxPathForDrone(cubeId, droneId);
|
|
1493
|
+
let raw: string;
|
|
1494
|
+
try {
|
|
1495
|
+
raw = await fs.readFile(p, 'utf-8');
|
|
1496
|
+
} catch (err: any) {
|
|
1497
|
+
if (err?.code === 'ENOENT') return false;
|
|
1498
|
+
throw err;
|
|
1499
|
+
}
|
|
1500
|
+
return inboxRawHasEntry(raw, entryId, renderedLine);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
function sleep(ms: number): Promise<void> {
|
|
1504
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
1505
|
+
}
|