@apralabs/apra-fleet 0.4.3 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -30
- package/dist/agents/backlog-groomer.md +75 -27
- package/dist/agents/ci-watcher.md +37 -22
- package/dist/agents/deployer.md +39 -22
- package/dist/agents/doer.md +74 -40
- package/dist/agents/harvester.md +49 -40
- package/dist/agents/integ-test-runner.md +40 -23
- package/dist/agents/kb-reconciler.md +28 -7
- package/dist/agents/plan-reviewer.md +49 -24
- package/dist/agents/planner.md +66 -43
- package/dist/agents/regression-test-runner.md +101 -78
- package/dist/agents/reviewer.md +117 -56
- package/dist/agents/schemas/backlog-groomer-output.json +1 -1
- package/dist/agents/schemas/doer-output.json +32 -0
- package/dist/agents/schemas/harvester-input.json +1 -5
- package/dist/agents/schemas/planner-output.json +1 -1
- package/dist/agents/schemas/regression-test-runner-output.json +41 -7
- package/dist/agents/schemas/reviewer-output.json +62 -3
- package/dist/cli/agent-transform.d.ts +7 -0
- package/dist/cli/agent-transform.js +46 -1
- package/dist/cli/auth.d.ts +1 -0
- package/dist/cli/auth.js +38 -8
- package/dist/cli/beads-install.d.ts +35 -0
- package/dist/cli/beads-install.js +125 -0
- package/dist/cli/beads-pin.d.ts +5 -0
- package/dist/cli/beads-pin.js +27 -0
- package/dist/cli/call.d.ts +27 -0
- package/dist/cli/call.js +177 -0
- package/dist/cli/config.d.ts +24 -1
- package/dist/cli/config.js +69 -2
- package/dist/cli/dolt-install.d.ts +18 -2
- package/dist/cli/dolt-install.js +46 -15
- package/dist/cli/fleet-se-guard.d.ts +51 -0
- package/dist/cli/fleet-se-guard.js +215 -0
- package/dist/cli/install-guard.d.ts +54 -0
- package/dist/cli/install-guard.js +132 -4
- package/dist/cli/install.d.ts +77 -21
- package/dist/cli/install.js +610 -152
- package/dist/cli/kb-commit.d.ts +2 -1
- package/dist/cli/kb-commit.js +1 -1
- package/dist/cli/kb-import.d.ts +2 -1
- package/dist/cli/kb-import.js +1 -1
- package/dist/cli/register-member.js +20 -3
- package/dist/cli/remove-member.d.ts +4 -0
- package/dist/cli/remove-member.js +63 -0
- package/dist/cli/secret.js +9 -0
- package/dist/cli/start.js +59 -4
- package/dist/cli/status.d.ts +3 -0
- package/dist/cli/status.js +45 -12
- package/dist/cli/stop.js +64 -9
- package/dist/cli/uninstall.js +101 -15
- package/dist/cli/update.js +33 -2
- package/dist/cli/watch.js +1 -1
- package/dist/cli/workflow.js +3 -2
- package/dist/commands/kb-server.js +9 -2
- package/dist/hub-service/http-server.d.ts +5 -1
- package/dist/hub-service/http-server.js +55 -5
- package/dist/index.js +155 -8
- package/dist/os/child-env-bound.d.ts +77 -0
- package/dist/os/child-env-bound.js +171 -0
- package/dist/os/linux.d.ts +7 -2
- package/dist/os/linux.js +40 -14
- package/dist/os/macos.d.ts +1 -1
- package/dist/os/macos.js +2 -11
- package/dist/os/os-commands.d.ts +17 -2
- package/dist/os/windows-gitbash.d.ts +1 -0
- package/dist/os/windows-gitbash.js +17 -3
- package/dist/os/windows-wrapper.d.ts +0 -4
- package/dist/os/windows-wrapper.js +2 -1
- package/dist/os/windows.d.ts +5 -2
- package/dist/os/windows.js +75 -29
- package/dist/paths.d.ts +21 -0
- package/dist/paths.js +35 -0
- package/dist/providers/agy.d.ts +6 -2
- package/dist/providers/agy.js +245 -106
- package/dist/providers/claude-permission-denial.d.ts +7 -0
- package/dist/providers/claude-permission-denial.js +137 -0
- package/dist/providers/claude.d.ts +86 -38
- package/dist/providers/claude.js +426 -164
- package/dist/providers/codex.js +5 -1
- package/dist/providers/copilot.js +5 -1
- package/dist/providers/opencode.d.ts +10 -2
- package/dist/providers/opencode.js +114 -53
- package/dist/providers/provider.d.ts +171 -33
- package/dist/providers/provider.js +164 -3
- package/dist/services/agent-provisioner.d.ts +29 -0
- package/dist/services/agent-provisioner.js +107 -0
- package/dist/services/agent-shadow.d.ts +96 -0
- package/dist/services/agent-shadow.js +479 -0
- package/dist/services/agy-project.d.ts +2 -1
- package/dist/services/agy-project.js +15 -17
- package/dist/services/auth-socket.d.ts +25 -0
- package/dist/services/auth-socket.js +68 -23
- package/dist/services/auth-web.js +1 -1
- package/dist/services/budget-awareness.d.ts +11 -11
- package/dist/services/budget-awareness.js +15 -3
- package/dist/services/claude-local-scope-mcp.d.ts +19 -0
- package/dist/services/claude-local-scope-mcp.js +91 -0
- package/dist/services/cloud/aws.js +2 -1
- package/dist/services/cloud/lifecycle.js +19 -2
- package/dist/services/cloud/task-wrapper.d.ts +0 -9
- package/dist/services/cloud/task-wrapper.js +15 -6
- package/dist/services/event-loop-watchdog.d.ts +32 -0
- package/dist/services/event-loop-watchdog.js +140 -0
- package/dist/services/http-transport.d.ts +49 -1
- package/dist/services/http-transport.js +190 -24
- package/dist/services/in-flight-agents.d.ts +2 -0
- package/dist/services/in-flight-agents.js +4 -0
- package/dist/services/knowledge/bible-basis-filter.d.ts +53 -0
- package/dist/services/knowledge/bible-basis-filter.js +134 -0
- package/dist/services/knowledge/bible-blob-id.d.ts +44 -0
- package/dist/services/knowledge/bible-blob-id.js +113 -0
- package/dist/services/knowledge/bible-import.d.ts +65 -0
- package/dist/services/knowledge/bible-import.js +211 -0
- package/dist/services/knowledge/file-hash.d.ts +31 -0
- package/dist/services/knowledge/file-hash.js +153 -23
- package/dist/services/knowledge/http-provider.d.ts +4 -3
- package/dist/services/knowledge/http-provider.js +17 -5
- package/dist/services/knowledge/kb-maintainer-grant.d.ts +15 -0
- package/dist/services/knowledge/kb-maintainer-grant.js +34 -0
- package/dist/services/knowledge/kb-providers.d.ts +4 -0
- package/dist/services/knowledge/kb-providers.js +8 -0
- package/dist/services/knowledge/kb-removed-scope-keys.d.ts +28 -0
- package/dist/services/knowledge/kb-removed-scope-keys.js +59 -0
- package/dist/services/knowledge/kb-scopes.d.ts +14 -0
- package/dist/services/knowledge/kb-scopes.js +47 -0
- package/dist/services/knowledge/kb-self.d.ts +91 -0
- package/dist/services/knowledge/kb-self.js +198 -0
- package/dist/services/knowledge/member-bible-view.d.ts +21 -0
- package/dist/services/knowledge/member-bible-view.js +194 -0
- package/dist/services/knowledge/pre-init.js +1 -0
- package/dist/services/knowledge/project-slug.js +2 -2
- package/dist/services/knowledge/sqlite-provider.d.ts +64 -11
- package/dist/services/knowledge/sqlite-provider.js +303 -41
- package/dist/services/knowledge/types.d.ts +45 -2
- package/dist/services/member-access-secret.d.ts +70 -0
- package/dist/services/member-access-secret.js +195 -0
- package/dist/services/member-call-counts.d.ts +24 -0
- package/dist/services/member-call-counts.js +58 -0
- package/dist/services/member-config-io.d.ts +192 -0
- package/dist/services/member-config-io.js +443 -0
- package/dist/services/member-fleet-install.d.ts +710 -0
- package/dist/services/member-fleet-install.js +1895 -0
- package/dist/services/member-secret-env.d.ts +42 -0
- package/dist/services/member-secret-env.js +68 -0
- package/dist/services/member-tool-allowlist.d.ts +49 -0
- package/dist/services/member-tool-allowlist.js +111 -0
- package/dist/services/model-pricing.d.ts +19 -0
- package/dist/services/model-pricing.js +54 -8
- package/dist/services/port-holder.d.ts +86 -0
- package/dist/services/port-holder.js +351 -0
- package/dist/services/preflight-check.js +2 -0
- package/dist/services/registry.d.ts +13 -2
- package/dist/services/registry.js +27 -1
- package/dist/services/relay-strategy.d.ts +2 -0
- package/dist/services/relay-strategy.js +8 -0
- package/dist/services/server-lifecycle.d.ts +26 -0
- package/dist/services/server-lifecycle.js +85 -0
- package/dist/services/service-manager/index.d.ts +7 -1
- package/dist/services/service-manager/index.js +17 -4
- package/dist/services/service-manager/linux.d.ts +1 -1
- package/dist/services/service-manager/linux.js +15 -7
- package/dist/services/service-manager/macos.d.ts +11 -1
- package/dist/services/service-manager/macos.js +45 -7
- package/dist/services/service-manager/service-path.d.ts +24 -0
- package/dist/services/service-manager/service-path.js +55 -0
- package/dist/services/service-manager/types.d.ts +38 -3
- package/dist/services/service-manager/types.js +29 -0
- package/dist/services/service-manager/windows.d.ts +181 -2
- package/dist/services/service-manager/windows.js +666 -49
- package/dist/services/service-notice.d.ts +14 -0
- package/dist/services/service-notice.js +62 -0
- package/dist/services/service-start-guard.d.ts +20 -0
- package/dist/services/service-start-guard.js +86 -0
- package/dist/services/session-cost.d.ts +24 -0
- package/dist/services/session-cost.js +105 -0
- package/dist/services/session-mcp-config.d.ts +83 -0
- package/dist/services/session-mcp-config.js +211 -0
- package/dist/services/sftp.d.ts +34 -0
- package/dist/services/sftp.js +304 -76
- package/dist/services/shell-probe.d.ts +0 -24
- package/dist/services/shell-probe.js +28 -3
- package/dist/services/singleton.d.ts +68 -2
- package/dist/services/singleton.js +168 -11
- package/dist/services/sprint-coordination.js +9 -3
- package/dist/services/ssh.d.ts +27 -0
- package/dist/services/ssh.js +229 -38
- package/dist/services/stall/find-log-file.js +12 -5
- package/dist/services/stall/index.d.ts +1 -1
- package/dist/services/stall/read-log-tail.d.ts +23 -0
- package/dist/services/stall/read-log-tail.js +33 -2
- package/dist/services/stall/stall-detector.d.ts +25 -1
- package/dist/services/stall/stall-detector.js +85 -6
- package/dist/services/stall/stall-poller.d.ts +16 -0
- package/dist/services/stall/stall-poller.js +50 -6
- package/dist/services/stopped-marker.d.ts +16 -0
- package/dist/services/stopped-marker.js +55 -0
- package/dist/services/strategy.d.ts +33 -1
- package/dist/services/strategy.js +86 -17
- package/dist/services/supervisor-service-cleanup.d.ts +52 -0
- package/dist/services/supervisor-service-cleanup.js +332 -0
- package/dist/services/tool-registry.d.ts +8 -1
- package/dist/services/tool-registry.js +160 -59
- package/dist/services/tool-scope.d.ts +58 -0
- package/dist/services/tool-scope.js +99 -0
- package/dist/services/watch/project-resolver.js +1 -0
- package/dist/skills/auto-sprint-args/SKILL.md +2 -2
- package/dist/skills/fleet-sprint-cli/SKILL.md +39 -15
- package/dist/skills/pm/SKILL.md +3 -3
- package/dist/skills/pm/index.md +4 -2
- package/dist/skills/pm/kb-reconcile.md +29 -14
- package/dist/skills/pm/kb-review.md +1 -0
- package/dist/skills/pm/sprint.md +5 -5
- package/dist/tools/check-status.d.ts +53 -5
- package/dist/tools/check-status.js +210 -53
- package/dist/tools/child-id-allocator.d.ts +2 -2
- package/dist/tools/child-id-allocator.js +1 -1
- package/dist/tools/code-index-heal.d.ts +8 -0
- package/dist/tools/code-index-heal.js +21 -0
- package/dist/tools/code-index-state.d.ts +45 -0
- package/dist/tools/code-index-state.js +115 -0
- package/dist/tools/code-intelligence-codebase-memory.js +6 -33
- package/dist/tools/code-intelligence-freshness.js +1 -1
- package/dist/tools/code-intelligence-gitnexus.d.ts +30 -0
- package/dist/tools/code-intelligence-gitnexus.js +333 -57
- package/dist/tools/code-intelligence-kb-enrich.js +7 -6
- package/dist/tools/code-intelligence-readiness.d.ts +99 -0
- package/dist/tools/code-intelligence-readiness.js +206 -0
- package/dist/tools/code-intelligence-reindex.d.ts +195 -0
- package/dist/tools/code-intelligence-reindex.js +530 -50
- package/dist/tools/code-intelligence-telemetry.d.ts +6 -1
- package/dist/tools/code-intelligence-telemetry.js +12 -3
- package/dist/tools/code-intelligence.d.ts +81 -31
- package/dist/tools/code-intelligence.js +156 -52
- package/dist/tools/compose-permissions.d.ts +57 -2
- package/dist/tools/compose-permissions.js +497 -44
- package/dist/tools/execute-command.d.ts +11 -0
- package/dist/tools/execute-command.js +80 -12
- package/dist/tools/execute-prompt.d.ts +59 -15
- package/dist/tools/execute-prompt.js +544 -133
- package/dist/tools/kb-bible-commit.d.ts +65 -0
- package/dist/tools/kb-bible-commit.js +239 -0
- package/dist/tools/kb-capture.d.ts +13 -9
- package/dist/tools/kb-capture.js +42 -11
- package/dist/tools/kb-context.d.ts +27 -7
- package/dist/tools/kb-context.js +60 -9
- package/dist/tools/kb-export.d.ts +96 -7
- package/dist/tools/kb-export.js +228 -65
- package/dist/tools/kb-feedback.d.ts +13 -9
- package/dist/tools/kb-feedback.js +13 -7
- package/dist/tools/kb-freshness-sweep.d.ts +11 -7
- package/dist/tools/kb-freshness-sweep.js +7 -7
- package/dist/tools/kb-harvest.d.ts +11 -7
- package/dist/tools/kb-harvest.js +7 -7
- package/dist/tools/kb-import.d.ts +26 -10
- package/dist/tools/kb-import.js +169 -159
- package/dist/tools/kb-invalidate.d.ts +17 -10
- package/dist/tools/kb-invalidate.js +41 -8
- package/dist/tools/kb-list.d.ts +14 -10
- package/dist/tools/kb-list.js +30 -10
- package/dist/tools/kb-promote.d.ts +11 -7
- package/dist/tools/kb-promote.js +14 -8
- package/dist/tools/kb-query.d.ts +21 -11
- package/dist/tools/kb-query.js +76 -13
- package/dist/tools/kb-reconcile-prefilter.d.ts +11 -7
- package/dist/tools/kb-reconcile-prefilter.js +10 -8
- package/dist/tools/kb-resolve-contradiction.d.ts +11 -7
- package/dist/tools/kb-resolve-contradiction.js +7 -7
- package/dist/tools/kb-session-prime.d.ts +14 -7
- package/dist/tools/kb-session-prime.js +44 -44
- package/dist/tools/kb-setup.d.ts +11 -4
- package/dist/tools/kb-setup.js +25 -11
- package/dist/tools/kb-stats.d.ts +11 -10
- package/dist/tools/kb-stats.js +22 -41
- package/dist/tools/list-members.js +5 -2
- package/dist/tools/member-detail.d.ts +3 -0
- package/dist/tools/member-detail.js +88 -17
- package/dist/tools/member-reservation.d.ts +2 -2
- package/dist/tools/provision-auth.d.ts +39 -1
- package/dist/tools/provision-auth.js +269 -47
- package/dist/tools/provision-vcs-auth.d.ts +2 -2
- package/dist/tools/receive-files.js +3 -1
- package/dist/tools/register-member.d.ts +34 -8
- package/dist/tools/register-member.js +76 -58
- package/dist/tools/remove-member.js +110 -32
- package/dist/tools/send-email.d.ts +2 -2
- package/dist/tools/send-files.js +6 -4
- package/dist/tools/session-stats.d.ts +17 -0
- package/dist/tools/session-stats.js +27 -0
- package/dist/tools/shutdown-server.js +5 -0
- package/dist/tools/update-agent-cli.d.ts +19 -0
- package/dist/tools/update-agent-cli.js +59 -2
- package/dist/tools/update-member.d.ts +13 -10
- package/dist/tools/update-member.js +138 -7
- package/dist/tools/vcs-credential-exec.js +8 -2
- package/dist/types.d.ts +98 -1
- package/dist/utils/agent-helpers.js +2 -0
- package/dist/utils/auth-env.d.ts +15 -4
- package/dist/utils/auth-env.js +52 -23
- package/dist/utils/find-on-path.d.ts +37 -0
- package/dist/utils/find-on-path.js +85 -0
- package/dist/utils/log-helpers.d.ts +37 -0
- package/dist/utils/log-helpers.js +106 -20
- package/dist/utils/owner-only-fs.d.ts +56 -0
- package/dist/utils/owner-only-fs.js +98 -0
- package/dist/utils/process-utils.d.ts +28 -0
- package/dist/utils/process-utils.js +68 -0
- package/dist/utils/prompt-errors.js +4 -1
- package/dist/utils/session-caller.d.ts +4 -2
- package/dist/utils/session-caller.js +3 -2
- package/dist/utils/shell-escape.d.ts +42 -3
- package/dist/utils/shell-escape.js +70 -4
- package/dist/utils/ssh-error-messages.js +3 -0
- package/dist/utils/workspace-trust.d.ts +24 -1
- package/dist/utils/workspace-trust.js +21 -17
- package/dist/workflows/auto-sprint.js +223 -43
- package/package.json +11 -8
- package/packages/apra-fleet-client/package.json +2 -2
- package/packages/apra-fleet-client/src/client/api.mjs +558 -27
- package/packages/apra-fleet-client/src/client/auto-start.mjs +434 -0
- package/packages/apra-fleet-client/src/client/client.mjs +9 -0
- package/packages/apra-fleet-client/src/client/factory.mjs +23 -4
- package/packages/apra-fleet-client/src/client/reconnecting-transport.mjs +137 -0
- package/packages/apra-fleet-client/src/client/server-resolution.mjs +267 -30
- package/packages/apra-fleet-client/src/client/transport.mjs +37 -3
- package/packages/apra-fleet-se/apra-pm/agents/backlog-groomer.md +75 -27
- package/packages/apra-fleet-se/apra-pm/agents/ci-watcher.md +37 -22
- package/packages/apra-fleet-se/apra-pm/agents/deployer.md +39 -22
- package/packages/apra-fleet-se/apra-pm/agents/doer.md +74 -40
- package/packages/apra-fleet-se/apra-pm/agents/harvester.md +49 -40
- package/packages/apra-fleet-se/apra-pm/agents/integ-test-runner.md +40 -23
- package/packages/apra-fleet-se/apra-pm/agents/kb-reconciler.md +28 -7
- package/packages/apra-fleet-se/apra-pm/agents/plan-reviewer.md +49 -24
- package/packages/apra-fleet-se/apra-pm/agents/planner.md +66 -43
- package/packages/apra-fleet-se/apra-pm/agents/regression-test-runner.md +101 -78
- package/packages/apra-fleet-se/apra-pm/agents/reviewer.md +117 -56
- package/packages/apra-fleet-se/apra-pm/agents/schemas/backlog-groomer-output.json +1 -1
- package/packages/apra-fleet-se/apra-pm/agents/schemas/doer-output.json +32 -0
- package/packages/apra-fleet-se/apra-pm/agents/schemas/harvester-input.json +1 -5
- package/packages/apra-fleet-se/apra-pm/agents/schemas/planner-output.json +1 -1
- package/packages/apra-fleet-se/apra-pm/agents/schemas/regression-test-runner-output.json +41 -7
- package/packages/apra-fleet-se/apra-pm/agents/schemas/reviewer-output.json +62 -3
- package/packages/apra-fleet-se/apra-pm/skills/pm/SKILL.md +3 -3
- package/packages/apra-fleet-se/apra-pm/skills/pm/index.md +4 -2
- package/packages/apra-fleet-se/apra-pm/skills/pm/kb-reconcile.md +29 -14
- package/packages/apra-fleet-se/apra-pm/skills/pm/kb-review.md +1 -0
- package/packages/apra-fleet-se/apra-pm/skills/pm/sprint.md +5 -5
- package/packages/apra-fleet-se/bin/cli.mjs +184 -54
- package/packages/apra-fleet-se/bin/serve.mjs +143 -10
- package/packages/apra-fleet-se/fleet-sprint/abort.mjs +56 -20
- package/packages/apra-fleet-se/fleet-sprint/backlog-role.mjs +93 -0
- package/packages/apra-fleet-se/fleet-sprint/beads-children.mjs +208 -53
- package/packages/apra-fleet-se/fleet-sprint/beads-identity-check.mjs +576 -33
- package/packages/apra-fleet-se/fleet-sprint/beads-identity.mjs +9 -0
- package/packages/apra-fleet-se/fleet-sprint/beads-memory-hygiene.mjs +201 -0
- package/packages/apra-fleet-se/fleet-sprint/beads-scope.mjs +114 -7
- package/packages/apra-fleet-se/fleet-sprint/beads-transitions.mjs +154 -8
- package/packages/apra-fleet-se/fleet-sprint/ci-gate.mjs +353 -0
- package/packages/apra-fleet-se/fleet-sprint/conflict-ladder.mjs +12 -0
- package/packages/apra-fleet-se/fleet-sprint/contracts.mjs +60 -1
- package/packages/apra-fleet-se/fleet-sprint/coordination.mjs +25 -2
- package/packages/apra-fleet-se/fleet-sprint/dispatch-accounting.mjs +270 -0
- package/packages/apra-fleet-se/fleet-sprint/dispatch-failure.mjs +22 -2
- package/packages/apra-fleet-se/fleet-sprint/dispatch-role.mjs +205 -16
- package/packages/apra-fleet-se/fleet-sprint/docs/README.md +52 -14
- package/packages/apra-fleet-se/fleet-sprint/docs/dolt-sync-redesign.md +6 -1
- package/packages/apra-fleet-se/fleet-sprint/docs/fleet-sprint-diagram.md +3 -3
- package/packages/apra-fleet-se/fleet-sprint/dolt-sync.mjs +339 -20
- package/packages/apra-fleet-se/fleet-sprint/errors.mjs +210 -11
- package/packages/apra-fleet-se/fleet-sprint/explicit-id-create-guard.mjs +4 -4
- package/packages/apra-fleet-se/fleet-sprint/git-sync.mjs +200 -9
- package/packages/apra-fleet-se/fleet-sprint/git-topology.mjs +57 -9
- package/packages/apra-fleet-se/fleet-sprint/guarded-modules.mjs +53 -4
- package/packages/apra-fleet-se/fleet-sprint/kb-hints.mjs +181 -0
- package/packages/apra-fleet-se/fleet-sprint/kb-injection.mjs +64 -0
- package/packages/apra-fleet-se/fleet-sprint/kb-maintainer.mjs +298 -0
- package/packages/apra-fleet-se/fleet-sprint/kb.mjs +1257 -301
- package/packages/apra-fleet-se/fleet-sprint/lower-quality.mjs +39 -0
- package/packages/apra-fleet-se/fleet-sprint/member-align.mjs +391 -0
- package/packages/apra-fleet-se/fleet-sprint/member-call.mjs +374 -0
- package/packages/apra-fleet-se/fleet-sprint/member-init-probe.mjs +644 -0
- package/packages/apra-fleet-se/fleet-sprint/member-provisioning.mjs +447 -0
- package/packages/apra-fleet-se/fleet-sprint/member-sync.mjs +250 -29
- package/packages/apra-fleet-se/fleet-sprint/orphan-reclaim.mjs +153 -0
- package/packages/apra-fleet-se/fleet-sprint/phases/deploy.mjs +82 -0
- package/packages/apra-fleet-se/fleet-sprint/phases/develop.mjs +97 -50
- package/packages/apra-fleet-se/fleet-sprint/phases/ensure-sprint-branch.mjs +1 -0
- package/packages/apra-fleet-se/fleet-sprint/phases/final-review.mjs +81 -18
- package/packages/apra-fleet-se/fleet-sprint/phases/harvest.mjs +43 -11
- package/packages/apra-fleet-se/fleet-sprint/phases/integ-test.mjs +6 -6
- package/packages/apra-fleet-se/fleet-sprint/phases/plan.mjs +20 -15
- package/packages/apra-fleet-se/fleet-sprint/phases/publish-pr.mjs +60 -38
- package/packages/apra-fleet-se/fleet-sprint/phases/re-review.mjs +42 -10
- package/packages/apra-fleet-se/fleet-sprint/phases/regression-test.mjs +16 -11
- package/packages/apra-fleet-se/fleet-sprint/phases/replan.mjs +36 -3
- package/packages/apra-fleet-se/fleet-sprint/phases/review.mjs +72 -12
- package/packages/apra-fleet-se/fleet-sprint/pr-body.mjs +259 -0
- package/packages/apra-fleet-se/fleet-sprint/prompts.mjs +83 -25
- package/packages/apra-fleet-se/fleet-sprint/role-policies.mjs +19 -1
- package/packages/apra-fleet-se/fleet-sprint/runner.js +834 -142
- package/packages/apra-fleet-se/fleet-sprint/se-os-commands.mjs +24 -0
- package/packages/apra-fleet-se/fleet-sprint/se-posix.mjs +130 -0
- package/packages/apra-fleet-se/fleet-sprint/se-windows.mjs +111 -0
- package/packages/apra-fleet-se/fleet-sprint/shell-command-guard.mjs +44 -1
- package/packages/apra-fleet-se/fleet-sprint/skills/fleet-sprint-cli/SKILL.md +39 -15
- package/packages/apra-fleet-se/fleet-sprint/skills/fleet-supervisor/SKILL.md +84 -30
- package/packages/apra-fleet-se/fleet-sprint/sprint-args.mjs +138 -2
- package/packages/apra-fleet-se/fleet-sprint/sprint-report.mjs +111 -11
- package/packages/apra-fleet-se/fleet-sprint/sprint-state.mjs +22 -0
- package/packages/apra-fleet-se/fleet-sprint/usage-limit-controller.mjs +7 -0
- package/packages/apra-fleet-se/fleet-sprint/vcs-auth.mjs +180 -14
- package/packages/apra-fleet-se/fleet-sprint/vcs-module.mjs +134 -7
- package/packages/apra-fleet-se/fleet-sprint/vcs-providers/azure-devops.mjs +99 -0
- package/packages/apra-fleet-se/fleet-sprint/vcs-providers/github.mjs +223 -0
- package/packages/apra-fleet-se/fleet-sprint/vcs-providers/index.mjs +49 -0
- package/packages/apra-fleet-se/fleet-sprint/vcs-providers/shell-helpers.mjs +8 -4
- package/packages/apra-fleet-se/fleet-sprint/verify-block.mjs +152 -0
- package/packages/apra-fleet-se/fleet-sprint/viewer-extensions.mjs +235 -21
- package/packages/apra-fleet-se/package.json +2 -2
- package/packages/apra-fleet-se/src/supervisor/api.mjs +919 -0
- package/packages/apra-fleet-se/src/supervisor/backlog-member.mjs +332 -0
- package/packages/apra-fleet-se/src/supervisor/backlog.mjs +982 -0
- package/packages/apra-fleet-se/src/supervisor/beads-identity.mjs +246 -0
- package/packages/apra-fleet-se/src/supervisor/beads-view.mjs +438 -0
- package/packages/apra-fleet-se/src/supervisor/child-port.mjs +36 -0
- package/packages/apra-fleet-se/src/supervisor/dashboard.mjs +1659 -0
- package/packages/apra-fleet-se/src/supervisor/dolt-mutex.mjs +350 -0
- package/packages/apra-fleet-se/src/supervisor/dolt-orphan-sweep.mjs +338 -0
- package/packages/apra-fleet-se/src/supervisor/fleet-members.mjs +279 -0
- package/packages/apra-fleet-se/src/supervisor/history-view.mjs +276 -0
- package/packages/apra-fleet-se/src/supervisor/history.mjs +328 -0
- package/packages/apra-fleet-se/src/supervisor/id-allocator.mjs +521 -0
- package/packages/apra-fleet-se/src/supervisor/launch-form.mjs +444 -0
- package/packages/apra-fleet-se/src/supervisor/ledger.mjs +629 -0
- package/packages/apra-fleet-se/src/supervisor/lib/exec-bd.mjs +303 -0
- package/packages/apra-fleet-se/src/supervisor/log-timestamp.mjs +51 -0
- package/packages/apra-fleet-se/src/supervisor/log-view.mjs +159 -0
- package/packages/apra-fleet-se/src/supervisor/proxy.mjs +474 -0
- package/packages/apra-fleet-se/src/supervisor/readopt.mjs +136 -0
- package/packages/apra-fleet-se/src/supervisor/reconcile.mjs +263 -0
- package/packages/apra-fleet-se/src/supervisor/rename-with-retry.mjs +87 -0
- package/packages/apra-fleet-se/src/supervisor/scope-overlap.mjs +327 -0
- package/packages/apra-fleet-se/src/supervisor/self-log.mjs +133 -0
- package/packages/apra-fleet-se/src/supervisor/server.mjs +399 -0
- package/packages/apra-fleet-se/src/supervisor/spawner.mjs +542 -0
- package/packages/apra-fleet-se/src/supervisor/sprint-identity.mjs +96 -0
- package/packages/apra-fleet-se/src/supervisor/watchdog.mjs +987 -0
- package/packages/apra-fleet-workflow/package.json +1 -1
- package/packages/apra-fleet-workflow/src/viewer/index.mjs +64 -9
- package/packages/apra-fleet-workflow/src/viewer/run-summary.mjs +127 -0
- package/packages/apra-fleet-workflow/src/workflow/index.mjs +71 -5
- package/packages/apra-fleet-workflow/src/workflow/pricing.mjs +75 -25
- package/skills/fleet/permissions.md +9 -0
- package/skills/fleet/profiles/tag-kb-reconciler.json +2 -6
- package/version.json +1 -1
- package/dist/cli/agent-transform.d.ts.map +0 -1
- package/dist/cli/agent-transform.js.map +0 -1
- package/dist/cli/auth.d.ts.map +0 -1
- package/dist/cli/auth.js.map +0 -1
- package/dist/cli/config.d.ts.map +0 -1
- package/dist/cli/config.js.map +0 -1
- package/dist/cli/dolt-install.d.ts.map +0 -1
- package/dist/cli/dolt-install.js.map +0 -1
- package/dist/cli/install-guard.d.ts.map +0 -1
- package/dist/cli/install-guard.js.map +0 -1
- package/dist/cli/install.d.ts.map +0 -1
- package/dist/cli/install.js.map +0 -1
- package/dist/cli/join.d.ts.map +0 -1
- package/dist/cli/join.js.map +0 -1
- package/dist/cli/kb-commit.d.ts.map +0 -1
- package/dist/cli/kb-commit.js.map +0 -1
- package/dist/cli/kb-directives.d.ts.map +0 -1
- package/dist/cli/kb-directives.js.map +0 -1
- package/dist/cli/kb-import.d.ts.map +0 -1
- package/dist/cli/kb-import.js.map +0 -1
- package/dist/cli/register-member.d.ts.map +0 -1
- package/dist/cli/register-member.js.map +0 -1
- package/dist/cli/restart.d.ts.map +0 -1
- package/dist/cli/restart.js.map +0 -1
- package/dist/cli/secret.d.ts.map +0 -1
- package/dist/cli/secret.js.map +0 -1
- package/dist/cli/spoke.d.ts.map +0 -1
- package/dist/cli/spoke.js.map +0 -1
- package/dist/cli/start.d.ts.map +0 -1
- package/dist/cli/start.js.map +0 -1
- package/dist/cli/status.d.ts.map +0 -1
- package/dist/cli/status.js.map +0 -1
- package/dist/cli/stop.d.ts.map +0 -1
- package/dist/cli/stop.js.map +0 -1
- package/dist/cli/uninstall.d.ts.map +0 -1
- package/dist/cli/uninstall.js.map +0 -1
- package/dist/cli/update.d.ts.map +0 -1
- package/dist/cli/update.js.map +0 -1
- package/dist/cli/watch.d.ts.map +0 -1
- package/dist/cli/watch.js.map +0 -1
- package/dist/cli/workflow-assets.d.ts.map +0 -1
- package/dist/cli/workflow-assets.js.map +0 -1
- package/dist/cli/workflow.d.ts.map +0 -1
- package/dist/cli/workflow.js.map +0 -1
- package/dist/commands/kb-server.d.ts.map +0 -1
- package/dist/commands/kb-server.js.map +0 -1
- package/dist/delivery-mode.d.ts.map +0 -1
- package/dist/delivery-mode.js.map +0 -1
- package/dist/hub-service/activity.d.ts.map +0 -1
- package/dist/hub-service/activity.js.map +0 -1
- package/dist/hub-service/audit-log.d.ts.map +0 -1
- package/dist/hub-service/audit-log.js.map +0 -1
- package/dist/hub-service/db/pool.d.ts.map +0 -1
- package/dist/hub-service/db/pool.js.map +0 -1
- package/dist/hub-service/enrollment.d.ts.map +0 -1
- package/dist/hub-service/enrollment.js.map +0 -1
- package/dist/hub-service/envelope-routes.d.ts.map +0 -1
- package/dist/hub-service/envelope-routes.js.map +0 -1
- package/dist/hub-service/handlers/installers.d.ts.map +0 -1
- package/dist/hub-service/handlers/installers.js.map +0 -1
- package/dist/hub-service/http-server.d.ts.map +0 -1
- package/dist/hub-service/http-server.js.map +0 -1
- package/dist/hub-service/hub-jwt.d.ts.map +0 -1
- package/dist/hub-service/hub-jwt.js.map +0 -1
- package/dist/hub-service/jwt-revocation.d.ts.map +0 -1
- package/dist/hub-service/jwt-revocation.js.map +0 -1
- package/dist/hub-service/machines.d.ts.map +0 -1
- package/dist/hub-service/machines.js.map +0 -1
- package/dist/hub-service/main.d.ts.map +0 -1
- package/dist/hub-service/main.js.map +0 -1
- package/dist/hub-service/member-tokens.d.ts.map +0 -1
- package/dist/hub-service/member-tokens.js.map +0 -1
- package/dist/hub-service/member-view.d.ts.map +0 -1
- package/dist/hub-service/member-view.js.map +0 -1
- package/dist/hub-service/members.d.ts.map +0 -1
- package/dist/hub-service/members.js.map +0 -1
- package/dist/hub-service/presence.d.ts.map +0 -1
- package/dist/hub-service/presence.js.map +0 -1
- package/dist/hub-service/project-view.d.ts.map +0 -1
- package/dist/hub-service/project-view.js.map +0 -1
- package/dist/hub-service/projects.d.ts.map +0 -1
- package/dist/hub-service/projects.js.map +0 -1
- package/dist/hub-service/relay-queue.d.ts.map +0 -1
- package/dist/hub-service/relay-queue.js.map +0 -1
- package/dist/hub-service/session-jwt.d.ts.map +0 -1
- package/dist/hub-service/session-jwt.js.map +0 -1
- package/dist/hub-service/usage.d.ts.map +0 -1
- package/dist/hub-service/usage.js.map +0 -1
- package/dist/hub-service/users.d.ts.map +0 -1
- package/dist/hub-service/users.js.map +0 -1
- package/dist/hub-service/workspaces.d.ts.map +0 -1
- package/dist/hub-service/workspaces.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/onboarding/text.d.ts.map +0 -1
- package/dist/onboarding/text.js.map +0 -1
- package/dist/os/git-bash-candidates.d.ts.map +0 -1
- package/dist/os/git-bash-candidates.js.map +0 -1
- package/dist/os/index.d.ts.map +0 -1
- package/dist/os/index.js.map +0 -1
- package/dist/os/linux.d.ts.map +0 -1
- package/dist/os/linux.js.map +0 -1
- package/dist/os/macos.d.ts.map +0 -1
- package/dist/os/macos.js.map +0 -1
- package/dist/os/os-commands.d.ts.map +0 -1
- package/dist/os/os-commands.js.map +0 -1
- package/dist/os/windows-gitbash.d.ts.map +0 -1
- package/dist/os/windows-gitbash.js.map +0 -1
- package/dist/os/windows-wrapper.d.ts.map +0 -1
- package/dist/os/windows-wrapper.js.map +0 -1
- package/dist/os/windows.d.ts.map +0 -1
- package/dist/os/windows.js.map +0 -1
- package/dist/paths.d.ts.map +0 -1
- package/dist/paths.js.map +0 -1
- package/dist/providers/agy.d.ts.map +0 -1
- package/dist/providers/agy.js.map +0 -1
- package/dist/providers/claude.d.ts.map +0 -1
- package/dist/providers/claude.js.map +0 -1
- package/dist/providers/codex.d.ts.map +0 -1
- package/dist/providers/codex.js.map +0 -1
- package/dist/providers/copilot.d.ts.map +0 -1
- package/dist/providers/copilot.js.map +0 -1
- package/dist/providers/email/headers.d.ts.map +0 -1
- package/dist/providers/email/headers.js.map +0 -1
- package/dist/providers/email/index.d.ts.map +0 -1
- package/dist/providers/email/index.js.map +0 -1
- package/dist/providers/email/provider.d.ts.map +0 -1
- package/dist/providers/email/provider.js.map +0 -1
- package/dist/providers/email/sendgrid.d.ts.map +0 -1
- package/dist/providers/email/sendgrid.js.map +0 -1
- package/dist/providers/email/smtp.d.ts.map +0 -1
- package/dist/providers/email/smtp.js.map +0 -1
- package/dist/providers/index.d.ts.map +0 -1
- package/dist/providers/index.js.map +0 -1
- package/dist/providers/none.d.ts.map +0 -1
- package/dist/providers/none.js.map +0 -1
- package/dist/providers/opencode.d.ts.map +0 -1
- package/dist/providers/opencode.js.map +0 -1
- package/dist/providers/provider.d.ts.map +0 -1
- package/dist/providers/provider.js.map +0 -1
- package/dist/services/agent-provisioner.d.ts.map +0 -1
- package/dist/services/agent-provisioner.js.map +0 -1
- package/dist/services/agy-project.d.ts.map +0 -1
- package/dist/services/agy-project.js.map +0 -1
- package/dist/services/auth-socket.d.ts.map +0 -1
- package/dist/services/auth-socket.js.map +0 -1
- package/dist/services/auth-web.d.ts.map +0 -1
- package/dist/services/auth-web.js.map +0 -1
- package/dist/services/budget-awareness.d.ts.map +0 -1
- package/dist/services/budget-awareness.js.map +0 -1
- package/dist/services/cloud/activity.d.ts.map +0 -1
- package/dist/services/cloud/activity.js.map +0 -1
- package/dist/services/cloud/aws.d.ts.map +0 -1
- package/dist/services/cloud/aws.js.map +0 -1
- package/dist/services/cloud/cost.d.ts.map +0 -1
- package/dist/services/cloud/cost.js.map +0 -1
- package/dist/services/cloud/idle-manager.d.ts.map +0 -1
- package/dist/services/cloud/idle-manager.js.map +0 -1
- package/dist/services/cloud/lifecycle.d.ts.map +0 -1
- package/dist/services/cloud/lifecycle.js.map +0 -1
- package/dist/services/cloud/task-wrapper.d.ts.map +0 -1
- package/dist/services/cloud/task-wrapper.js.map +0 -1
- package/dist/services/cloud/types.d.ts.map +0 -1
- package/dist/services/cloud/types.js.map +0 -1
- package/dist/services/cloud-sync.d.ts.map +0 -1
- package/dist/services/cloud-sync.js.map +0 -1
- package/dist/services/context-admission.d.ts.map +0 -1
- package/dist/services/context-admission.js.map +0 -1
- package/dist/services/credential-cleanup.d.ts.map +0 -1
- package/dist/services/credential-cleanup.js.map +0 -1
- package/dist/services/credential-store.d.ts.map +0 -1
- package/dist/services/credential-store.js.map +0 -1
- package/dist/services/event-bus.d.ts.map +0 -1
- package/dist/services/event-bus.js.map +0 -1
- package/dist/services/exit-drain.d.ts.map +0 -1
- package/dist/services/exit-drain.js.map +0 -1
- package/dist/services/file-chunker.d.ts.map +0 -1
- package/dist/services/file-chunker.js.map +0 -1
- package/dist/services/file-transfer-relay.d.ts.map +0 -1
- package/dist/services/file-transfer-relay.js.map +0 -1
- package/dist/services/file-transfer.d.ts.map +0 -1
- package/dist/services/file-transfer.js.map +0 -1
- package/dist/services/git-config.d.ts.map +0 -1
- package/dist/services/git-config.js.map +0 -1
- package/dist/services/github-app.d.ts.map +0 -1
- package/dist/services/github-app.js.map +0 -1
- package/dist/services/http-transport.d.ts.map +0 -1
- package/dist/services/http-transport.js.map +0 -1
- package/dist/services/hub-client.d.ts.map +0 -1
- package/dist/services/hub-client.js.map +0 -1
- package/dist/services/icons.d.ts.map +0 -1
- package/dist/services/icons.js.map +0 -1
- package/dist/services/jwt.d.ts.map +0 -1
- package/dist/services/jwt.js.map +0 -1
- package/dist/services/knowledge/audn.d.ts.map +0 -1
- package/dist/services/knowledge/audn.js.map +0 -1
- package/dist/services/knowledge/file-hash.d.ts.map +0 -1
- package/dist/services/knowledge/file-hash.js.map +0 -1
- package/dist/services/knowledge/http-provider.d.ts.map +0 -1
- package/dist/services/knowledge/http-provider.js.map +0 -1
- package/dist/services/knowledge/kb-config.d.ts.map +0 -1
- package/dist/services/knowledge/kb-config.js.map +0 -1
- package/dist/services/knowledge/kb-providers.d.ts.map +0 -1
- package/dist/services/knowledge/kb-providers.js.map +0 -1
- package/dist/services/knowledge/kb-scope-input.d.ts +0 -5
- package/dist/services/knowledge/kb-scope-input.d.ts.map +0 -1
- package/dist/services/knowledge/kb-scope-input.js +0 -11
- package/dist/services/knowledge/kb-scope-input.js.map +0 -1
- package/dist/services/knowledge/kb-service.d.ts.map +0 -1
- package/dist/services/knowledge/kb-service.js.map +0 -1
- package/dist/services/knowledge/path-validation.d.ts.map +0 -1
- package/dist/services/knowledge/path-validation.js.map +0 -1
- package/dist/services/knowledge/pre-init.d.ts.map +0 -1
- package/dist/services/knowledge/pre-init.js.map +0 -1
- package/dist/services/knowledge/project-slug.d.ts.map +0 -1
- package/dist/services/knowledge/project-slug.js.map +0 -1
- package/dist/services/knowledge/require-sqlite-project.d.ts.map +0 -1
- package/dist/services/knowledge/require-sqlite-project.js.map +0 -1
- package/dist/services/knowledge/sqlite-provider.d.ts.map +0 -1
- package/dist/services/knowledge/sqlite-provider.js.map +0 -1
- package/dist/services/knowledge/types.d.ts.map +0 -1
- package/dist/services/knowledge/types.js.map +0 -1
- package/dist/services/known-hosts.d.ts.map +0 -1
- package/dist/services/known-hosts.js.map +0 -1
- package/dist/services/known-sessions.d.ts.map +0 -1
- package/dist/services/known-sessions.js.map +0 -1
- package/dist/services/member-home.d.ts.map +0 -1
- package/dist/services/member-home.js.map +0 -1
- package/dist/services/member-remote-url.d.ts.map +0 -1
- package/dist/services/member-remote-url.js.map +0 -1
- package/dist/services/model-pricing.d.ts.map +0 -1
- package/dist/services/model-pricing.js.map +0 -1
- package/dist/services/onboarding.d.ts.map +0 -1
- package/dist/services/onboarding.js.map +0 -1
- package/dist/services/orphan-recovery.d.ts.map +0 -1
- package/dist/services/orphan-recovery.js.map +0 -1
- package/dist/services/pending-responses.d.ts.map +0 -1
- package/dist/services/pending-responses.js.map +0 -1
- package/dist/services/preflight-check.d.ts.map +0 -1
- package/dist/services/preflight-check.js.map +0 -1
- package/dist/services/registry.d.ts.map +0 -1
- package/dist/services/registry.js.map +0 -1
- package/dist/services/relay-context.d.ts.map +0 -1
- package/dist/services/relay-context.js.map +0 -1
- package/dist/services/relay-executor.d.ts.map +0 -1
- package/dist/services/relay-executor.js.map +0 -1
- package/dist/services/relay-request.d.ts.map +0 -1
- package/dist/services/relay-request.js.map +0 -1
- package/dist/services/relay-strategy.d.ts.map +0 -1
- package/dist/services/relay-strategy.js.map +0 -1
- package/dist/services/secret-token.d.ts.map +0 -1
- package/dist/services/secret-token.js.map +0 -1
- package/dist/services/service-manager/index.d.ts.map +0 -1
- package/dist/services/service-manager/index.js.map +0 -1
- package/dist/services/service-manager/linux.d.ts.map +0 -1
- package/dist/services/service-manager/linux.js.map +0 -1
- package/dist/services/service-manager/macos.d.ts.map +0 -1
- package/dist/services/service-manager/macos.js.map +0 -1
- package/dist/services/service-manager/types.d.ts.map +0 -1
- package/dist/services/service-manager/types.js.map +0 -1
- package/dist/services/service-manager/windows.d.ts.map +0 -1
- package/dist/services/service-manager/windows.js.map +0 -1
- package/dist/services/session-registry.d.ts.map +0 -1
- package/dist/services/session-registry.js.map +0 -1
- package/dist/services/sftp.d.ts.map +0 -1
- package/dist/services/sftp.js.map +0 -1
- package/dist/services/shell-probe.d.ts.map +0 -1
- package/dist/services/shell-probe.js.map +0 -1
- package/dist/services/singleton.d.ts.map +0 -1
- package/dist/services/singleton.js.map +0 -1
- package/dist/services/sprint-coordination.d.ts.map +0 -1
- package/dist/services/sprint-coordination.js.map +0 -1
- package/dist/services/ssh.d.ts.map +0 -1
- package/dist/services/ssh.js.map +0 -1
- package/dist/services/stall/find-log-file.d.ts.map +0 -1
- package/dist/services/stall/find-log-file.js.map +0 -1
- package/dist/services/stall/index.d.ts.map +0 -1
- package/dist/services/stall/index.js.map +0 -1
- package/dist/services/stall/log-path-resolver.d.ts.map +0 -1
- package/dist/services/stall/log-path-resolver.js.map +0 -1
- package/dist/services/stall/read-log-tail.d.ts.map +0 -1
- package/dist/services/stall/read-log-tail.js.map +0 -1
- package/dist/services/stall/stall-detector.d.ts.map +0 -1
- package/dist/services/stall/stall-detector.js.map +0 -1
- package/dist/services/stall/stall-poller.d.ts.map +0 -1
- package/dist/services/stall/stall-poller.js.map +0 -1
- package/dist/services/stall/time-utils.d.ts.map +0 -1
- package/dist/services/stall/time-utils.js.map +0 -1
- package/dist/services/statusline.d.ts.map +0 -1
- package/dist/services/statusline.js.map +0 -1
- package/dist/services/strategy.d.ts.map +0 -1
- package/dist/services/strategy.js.map +0 -1
- package/dist/services/substitution-engine.d.ts.map +0 -1
- package/dist/services/substitution-engine.js.map +0 -1
- package/dist/services/task-cleanup.d.ts.map +0 -1
- package/dist/services/task-cleanup.js.map +0 -1
- package/dist/services/token-issuer.d.ts.map +0 -1
- package/dist/services/token-issuer.js.map +0 -1
- package/dist/services/tool-registry.d.ts.map +0 -1
- package/dist/services/tool-registry.js.map +0 -1
- package/dist/services/update-check.d.ts.map +0 -1
- package/dist/services/update-check.js.map +0 -1
- package/dist/services/user-config.d.ts.map +0 -1
- package/dist/services/user-config.js.map +0 -1
- package/dist/services/vcs/azure-devops.d.ts.map +0 -1
- package/dist/services/vcs/azure-devops.js.map +0 -1
- package/dist/services/vcs/bitbucket.d.ts.map +0 -1
- package/dist/services/vcs/bitbucket.js.map +0 -1
- package/dist/services/vcs/constants.d.ts.map +0 -1
- package/dist/services/vcs/constants.js.map +0 -1
- package/dist/services/vcs/github.d.ts.map +0 -1
- package/dist/services/vcs/github.js.map +0 -1
- package/dist/services/vcs/types.d.ts.map +0 -1
- package/dist/services/vcs/types.js.map +0 -1
- package/dist/services/version-check.d.ts.map +0 -1
- package/dist/services/version-check.js.map +0 -1
- package/dist/services/watch/fleet-log.d.ts.map +0 -1
- package/dist/services/watch/fleet-log.js.map +0 -1
- package/dist/services/watch/project-resolver.d.ts.map +0 -1
- package/dist/services/watch/project-resolver.js.map +0 -1
- package/dist/services/watch/transcript-formatter.d.ts.map +0 -1
- package/dist/services/watch/transcript-formatter.js.map +0 -1
- package/dist/smoke-test.d.ts.map +0 -1
- package/dist/smoke-test.js.map +0 -1
- package/dist/tools/check-status.d.ts.map +0 -1
- package/dist/tools/check-status.js.map +0 -1
- package/dist/tools/child-id-allocator.d.ts.map +0 -1
- package/dist/tools/child-id-allocator.js.map +0 -1
- package/dist/tools/cloud-control.d.ts.map +0 -1
- package/dist/tools/cloud-control.js.map +0 -1
- package/dist/tools/code-intelligence-codebase-memory.d.ts.map +0 -1
- package/dist/tools/code-intelligence-codebase-memory.js.map +0 -1
- package/dist/tools/code-intelligence-freshness.d.ts.map +0 -1
- package/dist/tools/code-intelligence-freshness.js.map +0 -1
- package/dist/tools/code-intelligence-gitnexus.d.ts.map +0 -1
- package/dist/tools/code-intelligence-gitnexus.js.map +0 -1
- package/dist/tools/code-intelligence-kb-enrich.d.ts.map +0 -1
- package/dist/tools/code-intelligence-kb-enrich.js.map +0 -1
- package/dist/tools/code-intelligence-reindex.d.ts.map +0 -1
- package/dist/tools/code-intelligence-reindex.js.map +0 -1
- package/dist/tools/code-intelligence-telemetry.d.ts.map +0 -1
- package/dist/tools/code-intelligence-telemetry.js.map +0 -1
- package/dist/tools/code-intelligence-tests.d.ts.map +0 -1
- package/dist/tools/code-intelligence-tests.js.map +0 -1
- package/dist/tools/code-intelligence.d.ts.map +0 -1
- package/dist/tools/code-intelligence.js.map +0 -1
- package/dist/tools/compose-permissions.d.ts.map +0 -1
- package/dist/tools/compose-permissions.js.map +0 -1
- package/dist/tools/credential-store-delete.d.ts.map +0 -1
- package/dist/tools/credential-store-delete.js.map +0 -1
- package/dist/tools/credential-store-list.d.ts.map +0 -1
- package/dist/tools/credential-store-list.js.map +0 -1
- package/dist/tools/credential-store-set.d.ts.map +0 -1
- package/dist/tools/credential-store-set.js.map +0 -1
- package/dist/tools/credential-store-update.d.ts.map +0 -1
- package/dist/tools/credential-store-update.js.map +0 -1
- package/dist/tools/dolt-push-mutex.d.ts.map +0 -1
- package/dist/tools/dolt-push-mutex.js.map +0 -1
- package/dist/tools/execute-command.d.ts.map +0 -1
- package/dist/tools/execute-command.js.map +0 -1
- package/dist/tools/execute-prompt.d.ts.map +0 -1
- package/dist/tools/execute-prompt.js.map +0 -1
- package/dist/tools/get-member-model-pricing.d.ts.map +0 -1
- package/dist/tools/get-member-model-pricing.js.map +0 -1
- package/dist/tools/kb-capture.d.ts.map +0 -1
- package/dist/tools/kb-capture.js.map +0 -1
- package/dist/tools/kb-context.d.ts.map +0 -1
- package/dist/tools/kb-context.js.map +0 -1
- package/dist/tools/kb-export.d.ts.map +0 -1
- package/dist/tools/kb-export.js.map +0 -1
- package/dist/tools/kb-feedback.d.ts.map +0 -1
- package/dist/tools/kb-feedback.js.map +0 -1
- package/dist/tools/kb-freshness-sweep.d.ts.map +0 -1
- package/dist/tools/kb-freshness-sweep.js.map +0 -1
- package/dist/tools/kb-harvest.d.ts.map +0 -1
- package/dist/tools/kb-harvest.js.map +0 -1
- package/dist/tools/kb-import.d.ts.map +0 -1
- package/dist/tools/kb-import.js.map +0 -1
- package/dist/tools/kb-invalidate.d.ts.map +0 -1
- package/dist/tools/kb-invalidate.js.map +0 -1
- package/dist/tools/kb-list.d.ts.map +0 -1
- package/dist/tools/kb-list.js.map +0 -1
- package/dist/tools/kb-promote.d.ts.map +0 -1
- package/dist/tools/kb-promote.js.map +0 -1
- package/dist/tools/kb-query.d.ts.map +0 -1
- package/dist/tools/kb-query.js.map +0 -1
- package/dist/tools/kb-reconcile-prefilter.d.ts.map +0 -1
- package/dist/tools/kb-reconcile-prefilter.js.map +0 -1
- package/dist/tools/kb-resolve-contradiction.d.ts.map +0 -1
- package/dist/tools/kb-resolve-contradiction.js.map +0 -1
- package/dist/tools/kb-session-prime.d.ts.map +0 -1
- package/dist/tools/kb-session-prime.js.map +0 -1
- package/dist/tools/kb-setup.d.ts.map +0 -1
- package/dist/tools/kb-setup.js.map +0 -1
- package/dist/tools/kb-stats.d.ts.map +0 -1
- package/dist/tools/kb-stats.js.map +0 -1
- package/dist/tools/list-members.d.ts.map +0 -1
- package/dist/tools/list-members.js.map +0 -1
- package/dist/tools/member-detail.d.ts.map +0 -1
- package/dist/tools/member-detail.js.map +0 -1
- package/dist/tools/member-reservation.d.ts.map +0 -1
- package/dist/tools/member-reservation.js.map +0 -1
- package/dist/tools/monitor-task.d.ts.map +0 -1
- package/dist/tools/monitor-task.js.map +0 -1
- package/dist/tools/provision-auth.d.ts.map +0 -1
- package/dist/tools/provision-auth.js.map +0 -1
- package/dist/tools/provision-vcs-auth.d.ts.map +0 -1
- package/dist/tools/provision-vcs-auth.js.map +0 -1
- package/dist/tools/receive-files.d.ts.map +0 -1
- package/dist/tools/receive-files.js.map +0 -1
- package/dist/tools/register-member.d.ts.map +0 -1
- package/dist/tools/register-member.js.map +0 -1
- package/dist/tools/remove-member.d.ts.map +0 -1
- package/dist/tools/remove-member.js.map +0 -1
- package/dist/tools/report-status.d.ts.map +0 -1
- package/dist/tools/report-status.js.map +0 -1
- package/dist/tools/respond-to-message.d.ts.map +0 -1
- package/dist/tools/respond-to-message.js.map +0 -1
- package/dist/tools/revoke-vcs-auth.d.ts.map +0 -1
- package/dist/tools/revoke-vcs-auth.js.map +0 -1
- package/dist/tools/send-email.d.ts.map +0 -1
- package/dist/tools/send-email.js.map +0 -1
- package/dist/tools/send-files.d.ts.map +0 -1
- package/dist/tools/send-files.js.map +0 -1
- package/dist/tools/send-message.d.ts.map +0 -1
- package/dist/tools/send-message.js.map +0 -1
- package/dist/tools/setup-git-app.d.ts.map +0 -1
- package/dist/tools/setup-git-app.js.map +0 -1
- package/dist/tools/setup-ssh-key.d.ts.map +0 -1
- package/dist/tools/setup-ssh-key.js.map +0 -1
- package/dist/tools/shutdown-server.d.ts.map +0 -1
- package/dist/tools/shutdown-server.js.map +0 -1
- package/dist/tools/stop-prompt.d.ts.map +0 -1
- package/dist/tools/stop-prompt.js.map +0 -1
- package/dist/tools/update-agent-cli.d.ts.map +0 -1
- package/dist/tools/update-agent-cli.js.map +0 -1
- package/dist/tools/update-member.d.ts.map +0 -1
- package/dist/tools/update-member.js.map +0 -1
- package/dist/tools/vcs-credential-exec.d.ts.map +0 -1
- package/dist/tools/vcs-credential-exec.js.map +0 -1
- package/dist/tools/version.d.ts.map +0 -1
- package/dist/tools/version.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/utils/agent-helpers.d.ts.map +0 -1
- package/dist/utils/agent-helpers.js.map +0 -1
- package/dist/utils/ansi.d.ts.map +0 -1
- package/dist/utils/ansi.js.map +0 -1
- package/dist/utils/auth-env.d.ts.map +0 -1
- package/dist/utils/auth-env.js.map +0 -1
- package/dist/utils/collect-secret.d.ts.map +0 -1
- package/dist/utils/collect-secret.js.map +0 -1
- package/dist/utils/credential-validation.d.ts.map +0 -1
- package/dist/utils/credential-validation.js.map +0 -1
- package/dist/utils/crypto.d.ts.map +0 -1
- package/dist/utils/crypto.js.map +0 -1
- package/dist/utils/deep-merge.d.ts.map +0 -1
- package/dist/utils/deep-merge.js.map +0 -1
- package/dist/utils/file-permissions.d.ts.map +0 -1
- package/dist/utils/file-permissions.js.map +0 -1
- package/dist/utils/gpu-parser.d.ts.map +0 -1
- package/dist/utils/gpu-parser.js.map +0 -1
- package/dist/utils/log-helpers.d.ts.map +0 -1
- package/dist/utils/log-helpers.js.map +0 -1
- package/dist/utils/oob-timeout.d.ts.map +0 -1
- package/dist/utils/oob-timeout.js.map +0 -1
- package/dist/utils/opencode-model-validation.d.ts.map +0 -1
- package/dist/utils/opencode-model-validation.js.map +0 -1
- package/dist/utils/pid-helpers.d.ts.map +0 -1
- package/dist/utils/pid-helpers.js.map +0 -1
- package/dist/utils/platform.d.ts.map +0 -1
- package/dist/utils/platform.js.map +0 -1
- package/dist/utils/process-utils.d.ts.map +0 -1
- package/dist/utils/process-utils.js.map +0 -1
- package/dist/utils/prompt-errors.d.ts.map +0 -1
- package/dist/utils/prompt-errors.js.map +0 -1
- package/dist/utils/resolve-member.d.ts.map +0 -1
- package/dist/utils/resolve-member.js.map +0 -1
- package/dist/utils/secure-input.d.ts.map +0 -1
- package/dist/utils/secure-input.js.map +0 -1
- package/dist/utils/session-caller.d.ts.map +0 -1
- package/dist/utils/session-caller.js.map +0 -1
- package/dist/utils/shell-escape.d.ts.map +0 -1
- package/dist/utils/shell-escape.js.map +0 -1
- package/dist/utils/skill-matrix.d.ts.map +0 -1
- package/dist/utils/skill-matrix.js.map +0 -1
- package/dist/utils/ssh-error-messages.d.ts.map +0 -1
- package/dist/utils/ssh-error-messages.js.map +0 -1
- package/dist/utils/vcs-provider-detect.d.ts.map +0 -1
- package/dist/utils/vcs-provider-detect.js.map +0 -1
- package/dist/utils/work-folder-validation.d.ts.map +0 -1
- package/dist/utils/work-folder-validation.js.map +0 -1
- package/dist/utils/workspace-trust.d.ts.map +0 -1
- package/dist/utils/workspace-trust.js.map +0 -1
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js.map +0 -1
|
@@ -0,0 +1,987 @@
|
|
|
1
|
+
// =============================================================================
|
|
2
|
+
// fleet-sprint supervisor -- PID-liveness watchdog + four-status classifier
|
|
3
|
+
// (apra-fleet-eft.4.3, Plan Part 2.1, process model B)
|
|
4
|
+
// =============================================================================
|
|
5
|
+
//
|
|
6
|
+
// Same-instance exit events (the spawner's own child 'exit' listener) are
|
|
7
|
+
// authoritative WHILE this supervisor process stays up, but a supervisor
|
|
8
|
+
// RESTART is the severance point: after a restart the detached children live
|
|
9
|
+
// on with no in-memory 'exit' listener wired to them. So we cannot rely on exit
|
|
10
|
+
// events alone to know a sprint's true state -- we need an out-of-band probe.
|
|
11
|
+
//
|
|
12
|
+
// This module is that probe. On a short, CONFIGURABLE interval it PID-probes
|
|
13
|
+
// every ledger-listed sprint and combines three independent signals -- PID
|
|
14
|
+
// liveness, child HTTP reachability (the child's own `/state` endpoint on
|
|
15
|
+
// its --viewer-port), and (apra-fleet-p2to.3.1) that SAME `/state` payload's
|
|
16
|
+
// generic `pause.status` field (see packages/apra-fleet-workflow/src/viewer/
|
|
17
|
+
// index.mjs's `state.pause`, apra-fleet-p2to.2.1) -- into EXACTLY SIX statuses:
|
|
18
|
+
//
|
|
19
|
+
// running-healthy PID alive (and plausibly OUR child), HTTP answering,
|
|
20
|
+
// and not paused.
|
|
21
|
+
// paused PID alive, HTTP answering, and the child's own
|
|
22
|
+
// `state.pause.status` reads 'paused' (apra-fleet-
|
|
23
|
+
// p2to.1's engine pause primitive has actually
|
|
24
|
+
// engaged, not merely been requested). Treated as a
|
|
25
|
+
// LIVE, healthy state -- never stalled/dead, and
|
|
26
|
+
// (like running-healthy/running-unresponsive) its
|
|
27
|
+
// reservation is never auto-released.
|
|
28
|
+
// running-unresponsive PID alive but HTTP silent. This is an OPERATOR-ATTENTION
|
|
29
|
+
// signal, NOT a death sentence: a wedged/slow child is
|
|
30
|
+
// never auto-declared crashed and is never killed here.
|
|
31
|
+
// crashed PID gone, and NO terminal state persisted in old_runs/
|
|
32
|
+
// (or the legacy old_sprints/, apra-fleet-eft.37.1),
|
|
33
|
+
// and NOT within the launch-failed window.
|
|
34
|
+
// finished PID gone, and a terminal state IS persisted in old_runs/
|
|
35
|
+
// (or the legacy old_sprints/, apra-fleet-eft.37.1)
|
|
36
|
+
// launch-failed PID gone within the configurable launch window (default 60s),
|
|
37
|
+
// NO terminal state, a symptom of immediate child exit
|
|
38
|
+
//
|
|
39
|
+
// CRITICAL invariants (acceptance criteria):
|
|
40
|
+
// * The classifier returns EXACTLY ONE of the six statuses per sprint.
|
|
41
|
+
// * A hung child (PID alive, HTTP not answering) is running-unresponsive --
|
|
42
|
+
// never crashed, never killed.
|
|
43
|
+
// * A live-pid child the engine has actually paused is `paused`, never
|
|
44
|
+
// stalled/dead/crashed -- and, like every other live status, its
|
|
45
|
+
// reservation is never force-released (apra-fleet-p2to.3.1).
|
|
46
|
+
// * PID-gone WITH an old_runs/ (or legacy old_sprints/) terminal state =>
|
|
47
|
+
// finished; WITHOUT one => crashed.
|
|
48
|
+
// * PID reuse is guarded: the liveness probe validates the PID is plausibly
|
|
49
|
+
// OUR child (its command line still carries the sprint's unique
|
|
50
|
+
// `--viewer-port <port>` marker), not just any process that reused the PID
|
|
51
|
+
// number after our child exited.
|
|
52
|
+
// * This module NEVER auto-kills or auto-restarts anything. It only observes
|
|
53
|
+
// and classifies; remediation is an operator decision.
|
|
54
|
+
// =============================================================================
|
|
55
|
+
|
|
56
|
+
import http from 'node:http';
|
|
57
|
+
import fs from 'node:fs';
|
|
58
|
+
import { execFile } from 'node:child_process';
|
|
59
|
+
import { promisify } from 'node:util';
|
|
60
|
+
import { isPidAlive } from './reconcile.mjs';
|
|
61
|
+
import { getTerminalRunStatePath, getRunningRunStatePath } from '@apralabs/apra-fleet-workflow/viewer/run-state-paths';
|
|
62
|
+
import { writeJsonFileAtomic } from '@apralabs/apra-fleet-workflow/viewer/debounced-writer';
|
|
63
|
+
import { withTimestamps } from './log-timestamp.mjs';
|
|
64
|
+
import { HISTORY_EVENTS } from './history.mjs';
|
|
65
|
+
|
|
66
|
+
/** The six statuses the classifier may return. */
|
|
67
|
+
export const WATCHDOG_STATUS = Object.freeze({
|
|
68
|
+
RUNNING_HEALTHY: 'running-healthy',
|
|
69
|
+
// (apra-fleet-p2to.3.1) A live, HTTP-reachable child the engine has
|
|
70
|
+
// actually paused (state.pause.status === 'paused', not merely
|
|
71
|
+
// 'pausing') -- see this module's file-level doc comment.
|
|
72
|
+
PAUSED: 'paused',
|
|
73
|
+
RUNNING_UNRESPONSIVE: 'running-unresponsive',
|
|
74
|
+
CRASHED: 'crashed',
|
|
75
|
+
FINISHED: 'finished',
|
|
76
|
+
LAUNCH_FAILED: 'launch-failed',
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
/** Default watchdog probe interval (ms). Overridable via createWatchdog opts. */
|
|
80
|
+
export const WATCHDOG_DEFAULT_INTERVAL_MS = 5000;
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Default timeout (ms) for a single child HTTP reachability probe.
|
|
84
|
+
*
|
|
85
|
+
* apra-fleet CI incident (2026-08-04, run 30880131689): the windows-latest CI
|
|
86
|
+
* job deterministically misclassified a genuinely-healthy child as
|
|
87
|
+
* running-unresponsive. Root cause: the PID-reuse guard's Windows
|
|
88
|
+
* command-line read (readCmdlineViaWmic, falling back to readCmdlineViaCim)
|
|
89
|
+
* used a SYNCHRONOUS `spawnSync()` external process call, executed for every
|
|
90
|
+
* live-PID sprint BEFORE that sprint's HTTP probe was even sent
|
|
91
|
+
* (classifySprint() called isChildAlive() synchronously, then `await`ed
|
|
92
|
+
* probeHttp()). Because Array.prototype.map() invokes each classifySprint()
|
|
93
|
+
* call synchronously up to its first await, ALL of a given classifyAll()
|
|
94
|
+
* pass's synchronous readCmdline calls ran back-to-back and BLOCKED the
|
|
95
|
+
* whole Node.js event loop -- including already-dispatched HTTP
|
|
96
|
+
* request/response processing for an earlier sprint in the same pass -- for
|
|
97
|
+
* their combined duration. Measured on a dev box: `wmic` alone costs ~400ms
|
|
98
|
+
* per call; the `Get-CimInstance` PowerShell fallback costs over 1.2s per
|
|
99
|
+
* call. This starvation is not CI-only: the same production supervisor log
|
|
100
|
+
* this fix's own investigation reviewed showed a live "[watchdog] tick
|
|
101
|
+
* skipped: previous classifyAll() still in flight" line minutes after the
|
|
102
|
+
* CI-only timeout band-aid (a prior revision of this constant) had already
|
|
103
|
+
* landed, confirming the blocking read starves the event loop in real runs
|
|
104
|
+
* too, not just on a loaded CI runner.
|
|
105
|
+
*
|
|
106
|
+
* Fix (this revision): `readCmdlineViaWmic`/`readCmdlineViaCim`/
|
|
107
|
+
* `readCmdlineViaPs` now use `child_process.execFile` via
|
|
108
|
+
* `util.promisify` (same pattern as `scripts/lib/exec-bd.mjs`'s
|
|
109
|
+
* `execBdAsync`) instead of `spawnSync`, and `readProcessCmdline` /
|
|
110
|
+
* `makeChildPidProbe()`'s returned probe / `classifySprint()`'s call site
|
|
111
|
+
* are all async now (see their doc comments) -- so a cmdline read never
|
|
112
|
+
* blocks the event loop, and one sprint's PID-reuse-guard read can no longer
|
|
113
|
+
* starve another sprint's already-in-flight HTTP probe within the same
|
|
114
|
+
* `classifyAll()` pass. With the root cause (event-loop starvation) fixed
|
|
115
|
+
* rather than timed around, the timeout goes back to a single flat value on
|
|
116
|
+
* every platform -- the previous CI-only 6000ms carve-out is removed
|
|
117
|
+
* entirely rather than kept as a "just in case" margin, so CI stays exposed
|
|
118
|
+
* to (i.e. is not blind to) any future regression in this same class of bug.
|
|
119
|
+
*/
|
|
120
|
+
export const WATCHDOG_DEFAULT_HTTP_TIMEOUT_MS = 1500;
|
|
121
|
+
|
|
122
|
+
/** Promise-based `child_process.execFile`, used by the Windows/POSIX cmdline
|
|
123
|
+
* readers below instead of `spawnSync` so a PID-reuse-guard read never blocks
|
|
124
|
+
* the Node.js event loop (see WATCHDOG_DEFAULT_HTTP_TIMEOUT_MS's doc comment
|
|
125
|
+
* for the incident this fixes). Mirrors `scripts/lib/exec-bd.mjs`'s
|
|
126
|
+
* `promisify(execFile)` precedent. */
|
|
127
|
+
const execFileAsync = promisify(execFile);
|
|
128
|
+
|
|
129
|
+
/** Default launch-failed window (ms, i.e. 60 seconds). A child exiting within
|
|
130
|
+
* this window from its reservedAt timestamp is classified launch-failed. */
|
|
131
|
+
export const WATCHDOG_DEFAULT_LAUNCH_FAILED_WINDOW_MS = 60000;
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* `ps`-based command-line reader for POSIX platforms with no `/proc` (macOS,
|
|
135
|
+
* and a fallback for any other POSIX system where the `/proc` read fails).
|
|
136
|
+
* Uses `execFile` (via `execFileAsync`), not `spawnSync` -- POSIX `ps` is
|
|
137
|
+
* fast and rarely the bottleneck, but this is converted for consistency with
|
|
138
|
+
* the Windows readers below (see WATCHDOG_DEFAULT_HTTP_TIMEOUT_MS's doc
|
|
139
|
+
* comment for why blocking reads matter here at all).
|
|
140
|
+
* @param {number} pid
|
|
141
|
+
* @returns {Promise<string|null>}
|
|
142
|
+
*/
|
|
143
|
+
async function readCmdlineViaPs(pid) {
|
|
144
|
+
try {
|
|
145
|
+
const { stdout } = await execFileAsync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf-8', windowsHide: true });
|
|
146
|
+
const out = (stdout || '').trim();
|
|
147
|
+
return out.length > 0 ? out : null;
|
|
148
|
+
} catch {
|
|
149
|
+
return null;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* Windows command-line reader via WMIC (`wmic process where ProcessId=<pid>
|
|
155
|
+
* get CommandLine`). WMIC's output is a header line ("CommandLine") followed
|
|
156
|
+
* by the value line(s); we drop the header and join the rest.
|
|
157
|
+
* @param {number} pid
|
|
158
|
+
* @returns {Promise<string|null>}
|
|
159
|
+
*/
|
|
160
|
+
async function readCmdlineViaWmic(pid) {
|
|
161
|
+
try {
|
|
162
|
+
const { stdout } = await execFileAsync(
|
|
163
|
+
'wmic',
|
|
164
|
+
['process', 'where', `ProcessId=${pid}`, 'get', 'CommandLine'],
|
|
165
|
+
{ encoding: 'utf-8', windowsHide: true },
|
|
166
|
+
);
|
|
167
|
+
const lines = (stdout || '').split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
|
|
168
|
+
// First non-empty line is the "CommandLine" header; the rest is the value.
|
|
169
|
+
if (lines.length < 2) return null;
|
|
170
|
+
const value = lines.slice(1).join(' ').trim();
|
|
171
|
+
return value.length > 0 ? value : null;
|
|
172
|
+
} catch {
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Windows command-line reader via PowerShell's `Get-CimInstance`, used as a
|
|
179
|
+
* fallback where WMIC is unavailable (WMIC is deprecated/absent on some
|
|
180
|
+
* modern Windows builds; CIM is the supported replacement).
|
|
181
|
+
* @param {number} pid
|
|
182
|
+
* @returns {Promise<string|null>}
|
|
183
|
+
*/
|
|
184
|
+
async function readCmdlineViaCim(pid) {
|
|
185
|
+
try {
|
|
186
|
+
const { stdout } = await execFileAsync(
|
|
187
|
+
'powershell',
|
|
188
|
+
['-NoProfile', '-NonInteractive', '-Command', `(Get-CimInstance Win32_Process -Filter "ProcessId=${pid}").CommandLine`],
|
|
189
|
+
{ encoding: 'utf-8', windowsHide: true },
|
|
190
|
+
);
|
|
191
|
+
const out = (stdout || '').trim();
|
|
192
|
+
return out.length > 0 ? out : null;
|
|
193
|
+
} catch {
|
|
194
|
+
return null;
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Best-effort, per-platform read of a process's command line, for the
|
|
200
|
+
* PID-reuse guard:
|
|
201
|
+
* - Linux: `/proc/<pid>/cmdline` (NUL-separated argv, joined with spaces).
|
|
202
|
+
* - Windows: WMIC (`wmic process ... get CommandLine`), falling back to
|
|
203
|
+
* PowerShell's `Get-CimInstance` when WMIC is unavailable.
|
|
204
|
+
* - Everything else (macOS and other POSIX platforms without `/proc`): `ps`.
|
|
205
|
+
* Returns `null` when the command line cannot be read on the current
|
|
206
|
+
* platform (missing tool, permission denied, or the pid is gone) -- callers
|
|
207
|
+
* treat `null` as "cannot verify", never as a false negative.
|
|
208
|
+
*
|
|
209
|
+
* Async (previously synchronous via `spawnSync`): the Windows readers in
|
|
210
|
+
* particular cost 400ms-1.2s+ per call, and running them synchronously
|
|
211
|
+
* blocked the whole Node.js event loop -- including an already-dispatched
|
|
212
|
+
* HTTP probe for another sprint in the same `classifyAll()` pass. See
|
|
213
|
+
* WATCHDOG_DEFAULT_HTTP_TIMEOUT_MS's doc comment for the full incident.
|
|
214
|
+
* @param {number} pid
|
|
215
|
+
* @returns {Promise<string|null>}
|
|
216
|
+
*/
|
|
217
|
+
export async function readProcessCmdline(pid) {
|
|
218
|
+
if (!Number.isInteger(pid) || pid <= 0) return null;
|
|
219
|
+
if (process.platform === 'linux') {
|
|
220
|
+
try {
|
|
221
|
+
const raw = fs.readFileSync(`/proc/${pid}/cmdline`);
|
|
222
|
+
// argv entries are NUL-separated (and NUL-terminated); normalize to spaces.
|
|
223
|
+
const cmd = raw.toString('utf-8').replace(/\0/g, ' ').trim();
|
|
224
|
+
if (cmd.length > 0) return cmd;
|
|
225
|
+
} catch {
|
|
226
|
+
// fall through to the `ps` fallback below (e.g. /proc unreadable)
|
|
227
|
+
}
|
|
228
|
+
return readCmdlineViaPs(pid);
|
|
229
|
+
}
|
|
230
|
+
if (process.platform === 'win32') {
|
|
231
|
+
return (await readCmdlineViaWmic(pid)) ?? (await readCmdlineViaCim(pid));
|
|
232
|
+
}
|
|
233
|
+
return readCmdlineViaPs(pid);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Build the PID-liveness probe WITH the PID-reuse guard. A sprint's PID counts
|
|
238
|
+
* as alive only if:
|
|
239
|
+
* 1. the process exists (signal-0 probe, EPERM => exists), AND
|
|
240
|
+
* 2. if a `marker` is supplied AND the process command line is readable, that
|
|
241
|
+
* command line still contains the marker (the sprint's unique
|
|
242
|
+
* `--viewer-port <port>` string). If the command line cannot be read, we
|
|
243
|
+
* fall back to existence-only -- a documented best-effort, never a false
|
|
244
|
+
* "crashed".
|
|
245
|
+
*
|
|
246
|
+
* The marker being the unique viewer-port makes this a genuine reuse guard: an
|
|
247
|
+
* unrelated process that merely inherited our exited child's PID number will
|
|
248
|
+
* not be running with that exact `--viewer-port`, so it is correctly treated as
|
|
249
|
+
* NOT our child (=> the sprint is PID-gone -> crashed/finished, never reported
|
|
250
|
+
* as a healthy/unresponsive live sprint that happens to share a PID).
|
|
251
|
+
*
|
|
252
|
+
* Async (the returned probe and `readCmdline` may both return a Promise):
|
|
253
|
+
* `readProcessCmdline`'s Windows implementation now shells out via
|
|
254
|
+
* `child_process.execFile` rather than `spawnSync` (see
|
|
255
|
+
* WATCHDOG_DEFAULT_HTTP_TIMEOUT_MS's doc comment), so this never blocks the
|
|
256
|
+
* event loop. `await`ing a plain (non-Promise) return value from an injected
|
|
257
|
+
* synchronous `readCmdline`/`isAlive` test double is a no-op, so existing
|
|
258
|
+
* synchronous test doubles keep working unchanged.
|
|
259
|
+
* @param {{ readCmdline?: (pid: number) => string|null|Promise<string|null>, isAlive?: (pid: number) => boolean|Promise<boolean> }} [deps]
|
|
260
|
+
* @returns {(pid: number, marker?: string|number|null) => Promise<boolean>}
|
|
261
|
+
*/
|
|
262
|
+
export function makeChildPidProbe(deps = {}) {
|
|
263
|
+
const readCmdline = deps.readCmdline ?? readProcessCmdline;
|
|
264
|
+
const isAlive = deps.isAlive ?? isPidAlive;
|
|
265
|
+
return async (pid, marker) => {
|
|
266
|
+
if (!(await isAlive(pid))) return false;
|
|
267
|
+
if (marker === undefined || marker === null || marker === '') return true;
|
|
268
|
+
const cmd = await readCmdline(pid);
|
|
269
|
+
if (cmd == null) return true; // cannot verify -> best-effort existence
|
|
270
|
+
return cmd.includes(String(marker));
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Default child HTTP reachability probe: a short-timeout GET to the child's own
|
|
276
|
+
* viewer `/state` endpoint. "Reachable" means the child answered with any HTTP
|
|
277
|
+
* status at all (it is serving) -- a connection refused / reset / timeout means
|
|
278
|
+
* unreachable. Never throws: resolves to a boolean.
|
|
279
|
+
* @param {number} port
|
|
280
|
+
* @param {{ host?: string, path?: string, timeoutMs?: number }} [opts]
|
|
281
|
+
* @returns {Promise<boolean>}
|
|
282
|
+
*/
|
|
283
|
+
export function probeChildHttp(port, opts = {}) {
|
|
284
|
+
const host = opts.host ?? '127.0.0.1';
|
|
285
|
+
const path = opts.path ?? '/state';
|
|
286
|
+
const timeoutMs = Number.isInteger(opts.timeoutMs) ? opts.timeoutMs : WATCHDOG_DEFAULT_HTTP_TIMEOUT_MS;
|
|
287
|
+
if (!Number.isInteger(port) || port <= 0) return Promise.resolve(false);
|
|
288
|
+
return new Promise((resolve) => {
|
|
289
|
+
let settled = false;
|
|
290
|
+
const done = (val) => { if (!settled) { settled = true; resolve(val); } };
|
|
291
|
+
const req = http.request({ host, port, path, method: 'GET', timeout: timeoutMs }, (res) => {
|
|
292
|
+
// Any response means the child's HTTP server is answering. Drain and
|
|
293
|
+
// discard the body so the socket can be freed.
|
|
294
|
+
res.resume();
|
|
295
|
+
done(true);
|
|
296
|
+
});
|
|
297
|
+
req.on('timeout', () => { req.destroy(); done(false); });
|
|
298
|
+
req.on('error', () => done(false));
|
|
299
|
+
req.end();
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* (apra-fleet-p2to.3.1) Default pause-status probe: GETs the SAME child
|
|
305
|
+
* `/state` endpoint probeChildHttp() above already knows is reachable, and
|
|
306
|
+
* reads its generic `pause.status` field (packages/apra-fleet-workflow/src/
|
|
307
|
+
* viewer/index.mjs's `state.pause`, set by the p2to.2.1 viewer from the
|
|
308
|
+
* p2to.1 engine's own 'pause:requested'/'paused'/'resumed' events). Returns
|
|
309
|
+
* the raw string ('none' | 'pausing' | 'paused'), or `null` when the request
|
|
310
|
+
* fails, times out, or the body is not parseable JSON -- "cannot verify" is
|
|
311
|
+
* never conflated with "not paused": classifySprint() below only acts on an
|
|
312
|
+
* explicit `'paused'` value, so a probe failure here just leaves the child
|
|
313
|
+
* classified by PID+HTTP alone (running-healthy), exactly as it was before
|
|
314
|
+
* this probe existed. Never throws.
|
|
315
|
+
* @param {number} port
|
|
316
|
+
* @param {{ host?: string, path?: string, timeoutMs?: number }} [opts]
|
|
317
|
+
* @returns {Promise<string|null>}
|
|
318
|
+
*/
|
|
319
|
+
export function probeChildPauseStatus(port, opts = {}) {
|
|
320
|
+
const host = opts.host ?? '127.0.0.1';
|
|
321
|
+
const path = opts.path ?? '/state';
|
|
322
|
+
const timeoutMs = Number.isInteger(opts.timeoutMs) ? opts.timeoutMs : WATCHDOG_DEFAULT_HTTP_TIMEOUT_MS;
|
|
323
|
+
if (!Number.isInteger(port) || port <= 0) return Promise.resolve(null);
|
|
324
|
+
return new Promise((resolve) => {
|
|
325
|
+
let settled = false;
|
|
326
|
+
const done = (val) => { if (!settled) { settled = true; resolve(val); } };
|
|
327
|
+
const req = http.request({ host, port, path, method: 'GET', timeout: timeoutMs }, (res) => {
|
|
328
|
+
const chunks = [];
|
|
329
|
+
res.on('data', (c) => chunks.push(c));
|
|
330
|
+
res.on('end', () => {
|
|
331
|
+
try {
|
|
332
|
+
const body = JSON.parse(Buffer.concat(chunks).toString('utf-8'));
|
|
333
|
+
const status = body && typeof body === 'object' ? body?.pause?.status : null;
|
|
334
|
+
done(typeof status === 'string' ? status : null);
|
|
335
|
+
} catch {
|
|
336
|
+
done(null);
|
|
337
|
+
}
|
|
338
|
+
});
|
|
339
|
+
res.on('error', () => done(null));
|
|
340
|
+
});
|
|
341
|
+
req.on('timeout', () => { req.destroy(); done(null); });
|
|
342
|
+
req.on('error', () => done(null));
|
|
343
|
+
req.end();
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* apra-fleet-k7b.3: formats a human-readable exit-detail string for a
|
|
349
|
+
* PID-gone sprint from whatever the ledger's recordExit() (spawner's own
|
|
350
|
+
* SAME-INSTANCE 'exit' listener, see spawner.mjs/bin/serve.mjs's wiring)
|
|
351
|
+
* annotated onto its reservation -- e.g. "exited 1 at
|
|
352
|
+
* 2026-07-30T21:25:50.000Z" or "killed by signal SIGKILL at ...". Falls back
|
|
353
|
+
* to the previous bare "pid gone" when nothing was ever recorded (a restart
|
|
354
|
+
* severed the in-memory 'exit' listener before this instance ever observed
|
|
355
|
+
* the child exit -- see this module's own file-level doc comment).
|
|
356
|
+
* @param {{ exitCode?: number|null, signal?: string|null, exitedAt?: string|null }} [info]
|
|
357
|
+
* @returns {string}
|
|
358
|
+
*/
|
|
359
|
+
export function formatExitDetail(info = {}) {
|
|
360
|
+
const { exitCode, signal, exitedAt } = info;
|
|
361
|
+
if (exitCode == null && !signal) return 'pid gone';
|
|
362
|
+
const at = exitedAt ? ` at ${exitedAt}` : '';
|
|
363
|
+
if (exitCode != null) {
|
|
364
|
+
return signal ? `exited ${exitCode} (signal ${signal})${at}` : `exited ${exitCode}${at}`;
|
|
365
|
+
}
|
|
366
|
+
return `killed by signal ${signal}${at}`;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* apra-fleet-k7b.2: reads and parses a persisted terminal run-state file
|
|
371
|
+
* (old_runs/, or the legacy old_sprints/ directory) at `path`. Returns the
|
|
372
|
+
* parsed object, or `null` when the file does not exist or is not valid
|
|
373
|
+
* JSON -- a reader-side concern only, never thrown: a missing/corrupt
|
|
374
|
+
* terminal-state file must never take the watchdog's classification down
|
|
375
|
+
* with it.
|
|
376
|
+
* @param {string} path
|
|
377
|
+
* @returns {object|null}
|
|
378
|
+
*/
|
|
379
|
+
function readTerminalRunState(path) {
|
|
380
|
+
try {
|
|
381
|
+
return JSON.parse(fs.readFileSync(path, 'utf-8'));
|
|
382
|
+
} catch {
|
|
383
|
+
return null;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* apra-fleet-k7b.2: resolves whether a PID-gone sprint actually FINISHED, by
|
|
389
|
+
* run-id first and -- ONLY as a fallback for a reservation claimed before
|
|
390
|
+
* apra-fleet-k7b.1's run-id plumbing shipped -- by the sprint's launch
|
|
391
|
+
* branch. Returns the PARSED terminal-state object (not a boolean) so the
|
|
392
|
+
* watchdog log line / sprint-history.json event / History view can copy the
|
|
393
|
+
* engine's own `terminalReason` / `extensions.terminal.verdict` verbatim,
|
|
394
|
+
* instead of a generic message. Returns `null` when no terminal state is
|
|
395
|
+
* found under either key.
|
|
396
|
+
*
|
|
397
|
+
* The legacy branch-key fallback exists because a reservation claimed
|
|
398
|
+
* BEFORE k7b.1 has a sprintId (branch-derived, pre-run-id) that may not
|
|
399
|
+
* match the run-id the engine itself used to write its terminal state; a
|
|
400
|
+
* reservation's OWN recorded `branch` (ledger.mjs's Reservation.branch) is
|
|
401
|
+
* the only other identity available to look that pre-fix terminal state up
|
|
402
|
+
* by. A reservation claimed AFTER k7b.1 always resolves by run-id alone
|
|
403
|
+
* (branch is never consulted once the run-id lookup succeeds).
|
|
404
|
+
* @param {string} sprintId
|
|
405
|
+
* @param {string|null|undefined} branch
|
|
406
|
+
* @param {NodeJS.ProcessEnv} env
|
|
407
|
+
* @returns {object|null}
|
|
408
|
+
*/
|
|
409
|
+
export function defaultHasTerminalState(sprintId, branch, env = process.env) {
|
|
410
|
+
try {
|
|
411
|
+
const byRunId = getTerminalRunStatePath(sprintId, env);
|
|
412
|
+
if (fs.existsSync(byRunId)) {
|
|
413
|
+
return readTerminalRunState(byRunId);
|
|
414
|
+
}
|
|
415
|
+
} catch {
|
|
416
|
+
// fall through to the branch fallback below
|
|
417
|
+
}
|
|
418
|
+
if (typeof branch === 'string' && branch.length > 0 && branch !== sprintId) {
|
|
419
|
+
try {
|
|
420
|
+
const byBranch = getTerminalRunStatePath(branch, env);
|
|
421
|
+
if (fs.existsSync(byBranch)) {
|
|
422
|
+
return readTerminalRunState(byBranch);
|
|
423
|
+
}
|
|
424
|
+
} catch {
|
|
425
|
+
// no legacy terminal state under the branch key either
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
return null;
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* apra-fleet-k7b.2: formats a human-readable FINISHED detail string from a
|
|
433
|
+
* persisted terminal run-state, copying the engine's own `terminalReason`
|
|
434
|
+
* and `extensions.terminal.verdict` VERBATIM (never paraphrased/relabeled)
|
|
435
|
+
* -- e.g. "terminalReason=SPRINT_STALLED verdict=needs-changes". Falls back
|
|
436
|
+
* to a bare "finished" when the state carries neither field (or is not an
|
|
437
|
+
* object -- e.g. a truthy-but-non-object test double), so this never throws
|
|
438
|
+
* on an unexpected shape.
|
|
439
|
+
* @param {unknown} state
|
|
440
|
+
* @returns {string}
|
|
441
|
+
*/
|
|
442
|
+
export function formatFinishedDetail(state) {
|
|
443
|
+
if (!state || typeof state !== 'object') return 'finished';
|
|
444
|
+
const terminalReason = state.terminalReason ?? null;
|
|
445
|
+
const verdict = state?.extensions?.terminal?.verdict ?? null;
|
|
446
|
+
const parts = [];
|
|
447
|
+
if (terminalReason) parts.push(`terminalReason=${terminalReason}`);
|
|
448
|
+
if (verdict) parts.push(`verdict=${verdict}`);
|
|
449
|
+
return parts.length > 0 ? parts.join(' ') : 'finished';
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* apra-fleet-k7b.2: default FINISHED recorder, invoked the FIRST time a
|
|
454
|
+
* sprint is observed transitioning into FINISHED (PID gone, terminal state
|
|
455
|
+
* found). Mirrors apra-fleet-eft.20.3's defaultRecordTerminalError below,
|
|
456
|
+
* but for the "actually finished" case instead of "crashed": logs an
|
|
457
|
+
* explicit, greppable watchdog line copying the engine's own
|
|
458
|
+
* terminalReason/verdict verbatim (replacing the generic CRASHED-sounding
|
|
459
|
+
* language a mis-resolved terminal-state lookup used to fall through to),
|
|
460
|
+
* and -- when a `history` collaborator is injected -- appends a durable
|
|
461
|
+
* FINISHED event to sprint-history.json carrying the same fields.
|
|
462
|
+
* @param {{ sprintId: string, state: object|null, env: NodeJS.ProcessEnv, logger: { log?: Function, error?: Function }, history?: { record: (entry: object) => Promise<object> } }} info
|
|
463
|
+
*/
|
|
464
|
+
export function defaultRecordFinished({ sprintId, state, logger, history }) {
|
|
465
|
+
const log = (logger && (logger.log ?? logger.error)) ?? (() => {});
|
|
466
|
+
const detail = formatFinishedDetail(state);
|
|
467
|
+
log(`[watchdog] FINISHED: Sprint '${sprintId}' finished (${detail}).`);
|
|
468
|
+
if (!history || typeof history.record !== 'function') return;
|
|
469
|
+
const terminalReason = (state && typeof state === 'object' && state.terminalReason) || null;
|
|
470
|
+
const verdict = (state && typeof state === 'object' && state?.extensions?.terminal?.verdict) || null;
|
|
471
|
+
try {
|
|
472
|
+
const result = history.record({ sprintId, event: HISTORY_EVENTS.FINISHED, terminalReason, verdict });
|
|
473
|
+
// history.record() is async; a rejection must never take the
|
|
474
|
+
// classifier down with it (same discipline as recordTerminalError
|
|
475
|
+
// below), so it is observed but not awaited by the caller.
|
|
476
|
+
if (result && typeof result.catch === 'function') {
|
|
477
|
+
result.catch((err) => {
|
|
478
|
+
const logErr = (logger && (logger.error ?? logger.log)) ?? (() => {});
|
|
479
|
+
logErr(`[watchdog] history.record(FINISHED) failed for '${sprintId}':`, err);
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
} catch (err) {
|
|
483
|
+
const logErr = (logger && (logger.error ?? logger.log)) ?? (() => {});
|
|
484
|
+
logErr(`[watchdog] history.record(FINISHED) failed for '${sprintId}':`, err);
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* apra-fleet-eft.20.3: default terminal-error recorder, invoked the FIRST
|
|
490
|
+
* time a sprint is observed transitioning into CRASHED. The apra-fleet-eft.20
|
|
491
|
+
* smoke-test symptom this fixes: a doer sub-session died mid-Develop and the
|
|
492
|
+
* run just went silent -- no error, no exception, no exit line anywhere, and
|
|
493
|
+
* nothing about the failure was ever persisted. This makes that death
|
|
494
|
+
* observable in two places an operator (or another automated system) would
|
|
495
|
+
* actually look:
|
|
496
|
+
* (a) an explicit, greppable line via the watchdog's own logger -- the SAME
|
|
497
|
+
* fleet server log every other watchdog/supervisor line already goes
|
|
498
|
+
* to, so no new log surface to monitor;
|
|
499
|
+
* (b) the sprint's own state file, read+merged+written back atomically (the
|
|
500
|
+
* apra-fleet-eft.20.1 single-pass-JSON.stringify-plus-atomic-rename
|
|
501
|
+
* primitive) with a `status: 'failed'` and a `lastError` describing
|
|
502
|
+
* what the watchdog observed. Written back to running/ IN PLACE
|
|
503
|
+
* (never moved to old_runs/) so classifySprint()'s FINISHED/CRASHED
|
|
504
|
+
* distinction -- which keys off old_runs/ (or legacy old_sprints/)
|
|
505
|
+
* membership -- is not disturbed by this write: a sprint the watchdog
|
|
506
|
+
* declared crashed stays classified crashed on every later tick, it
|
|
507
|
+
* never silently becomes "finished" just because this recorder
|
|
508
|
+
* touched its file.
|
|
509
|
+
* apra-fleet-k7b.3: `detail` (formatExitDetail() above) reports the child's
|
|
510
|
+
* OWN recorded exit code/signal/time when this instance's spawner witnessed
|
|
511
|
+
* it (e.g. "exited 1 at 2026-07-30T21:25:50.000Z"), replacing the previously
|
|
512
|
+
* unconditional bare "pid gone" -- unchanged fallback when nothing was
|
|
513
|
+
* recorded (a restart severed the in-memory exit listener, see this
|
|
514
|
+
* module's file-level doc comment).
|
|
515
|
+
* @param {{ sprintId: string, childPid: number|null, env: NodeJS.ProcessEnv, logger: { log?: Function, error?: Function }, detail?: string }} info
|
|
516
|
+
*/
|
|
517
|
+
export function defaultRecordTerminalError({ sprintId, childPid, env, logger, detail }) {
|
|
518
|
+
const log = (logger && (logger.error ?? logger.log)) ?? (() => {});
|
|
519
|
+
const exitDetail = detail ?? 'pid gone';
|
|
520
|
+
const message = `Sprint '${sprintId}' (pid ${childPid ?? 'unknown'}) is no longer alive (${exitDetail}) and never recorded a terminal state -- classified CRASHED by the PID-liveness watchdog.`;
|
|
521
|
+
log(`[watchdog] TERMINAL ERROR: ${message}`);
|
|
522
|
+
try {
|
|
523
|
+
const statePath = getRunningRunStatePath(sprintId, env);
|
|
524
|
+
let existing = {};
|
|
525
|
+
try {
|
|
526
|
+
existing = JSON.parse(fs.readFileSync(statePath, 'utf-8'));
|
|
527
|
+
} catch {
|
|
528
|
+
// No readable prior state (the child died before ever writing one,
|
|
529
|
+
// or its last write was left malformed) -- still record the crash
|
|
530
|
+
// with whatever we know; never let a missing/bad prior file block
|
|
531
|
+
// reporting the failure itself.
|
|
532
|
+
}
|
|
533
|
+
writeJsonFileAtomic(statePath, {
|
|
534
|
+
...existing,
|
|
535
|
+
status: 'failed',
|
|
536
|
+
terminalReason: existing.terminalReason || `watchdog: crashed (${exitDetail}, no terminal state ever persisted)`,
|
|
537
|
+
lastError: {
|
|
538
|
+
message,
|
|
539
|
+
sprintId,
|
|
540
|
+
childPid: childPid ?? null,
|
|
541
|
+
detectedAt: new Date().toISOString(),
|
|
542
|
+
},
|
|
543
|
+
});
|
|
544
|
+
} catch (err) {
|
|
545
|
+
log(`[watchdog] failed to persist terminal error state for '${sprintId}': ${(err && err.message) || err}`);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/**
|
|
550
|
+
* Create the PID-liveness watchdog seam. Every collaborator is injectable so a
|
|
551
|
+
* test can drive deterministic PID/HTTP/terminal-state signals without real
|
|
552
|
+
* processes, sockets, or files.
|
|
553
|
+
*
|
|
554
|
+
* @param {{
|
|
555
|
+
* ledger: { list: () => Array<{ sprintId: string, childPid: number|null }> },
|
|
556
|
+
* resolvePort?: (sprintId: string) => number|undefined,
|
|
557
|
+
* isChildAlive?: (pid: number, marker?: string|number|null) => boolean|Promise<boolean>,
|
|
558
|
+
* probeHttp?: (port: number) => Promise<boolean>|boolean,
|
|
559
|
+
* probePauseState?: (port: number) => Promise<string|null>|string|null,
|
|
560
|
+
* hasTerminalState?: (sprintId: string, branch?: string|null) => object|boolean|null,
|
|
561
|
+
* recordTerminalError?: (info: { sprintId: string, childPid: number|null, env: NodeJS.ProcessEnv, logger: object }) => void,
|
|
562
|
+
* recordFinished?: (info: { sprintId: string, state: object|null, env: NodeJS.ProcessEnv, logger: object, history: object|null }) => void,
|
|
563
|
+
* history?: { record: (entry: object) => Promise<object> },
|
|
564
|
+
* intervalMs?: number,
|
|
565
|
+
* env?: NodeJS.ProcessEnv,
|
|
566
|
+
* setIntervalFn?: typeof setInterval,
|
|
567
|
+
* clearIntervalFn?: typeof clearInterval,
|
|
568
|
+
* logger?: { log?: Function, error?: Function },
|
|
569
|
+
* }} deps
|
|
570
|
+
* @returns {{
|
|
571
|
+
* name: string,
|
|
572
|
+
* start(): Promise<void>,
|
|
573
|
+
* stop(): Promise<void>,
|
|
574
|
+
* classifySprint(entry: object): Promise<object>,
|
|
575
|
+
* classifyAll(): Promise<Array<object>>,
|
|
576
|
+
* getSnapshot(): Array<object>,
|
|
577
|
+
* intervalMs: number,
|
|
578
|
+
* }}
|
|
579
|
+
*/
|
|
580
|
+
export function createWatchdog(deps = {}) {
|
|
581
|
+
const ledger = deps.ledger;
|
|
582
|
+
if (!ledger || typeof ledger.list !== 'function') {
|
|
583
|
+
throw new TypeError('createWatchdog requires a ledger with a list() method');
|
|
584
|
+
}
|
|
585
|
+
const env = deps.env ?? process.env;
|
|
586
|
+
const resolvePort = deps.resolvePort ?? (() => undefined);
|
|
587
|
+
const isChildAlive = deps.isChildAlive ?? makeChildPidProbe();
|
|
588
|
+
const probeHttp = deps.probeHttp ?? probeChildHttp;
|
|
589
|
+
// (apra-fleet-p2to.3.1) Only ever consulted when probeHttp above already
|
|
590
|
+
// confirmed the child is reachable -- see classifySprint()'s pidAlive
|
|
591
|
+
// branch below. Injectable so a test can drive a deterministic 'paused'/
|
|
592
|
+
// 'pausing'/'none'/null value without a real child HTTP server.
|
|
593
|
+
const probePauseState = deps.probePauseState ?? probeChildPauseStatus;
|
|
594
|
+
// apra-fleet-k7b.2: resolves by run-id first, falling back to the
|
|
595
|
+
// reservation's own recorded `branch` (ledger.mjs's Reservation.branch,
|
|
596
|
+
// legacy pre-run-id lookup key) ONLY when the run-id lookup misses; see
|
|
597
|
+
// defaultHasTerminalState()'s doc comment. Returns the PARSED terminal
|
|
598
|
+
// state (or `null`), not a boolean, so classifySprint() below can copy
|
|
599
|
+
// terminalReason/verdict verbatim -- injected test doubles that return a
|
|
600
|
+
// plain boolean keep working since only truthiness is checked before
|
|
601
|
+
// this value is passed on for (best-effort) field reads.
|
|
602
|
+
const hasTerminalState = deps.hasTerminalState
|
|
603
|
+
?? ((sprintId, branch) => defaultHasTerminalState(sprintId, branch, env));
|
|
604
|
+
// apra-fleet-eft.20.3: the CRASHED-transition recorder (log line + a
|
|
605
|
+
// persisted failed/lastError in the sprint's running/ state file, see
|
|
606
|
+
// defaultRecordTerminalError above). Injectable so a test can assert on a
|
|
607
|
+
// spy instead of the real fs/logger.
|
|
608
|
+
const recordTerminalError = deps.recordTerminalError ?? defaultRecordTerminalError;
|
|
609
|
+
// apra-fleet-k7b.2: the FINISHED-transition recorder (log line + optional
|
|
610
|
+
// sprint-history.json FINISHED event, see defaultRecordFinished above).
|
|
611
|
+
// Same injectable-for-tests discipline as recordTerminalError.
|
|
612
|
+
const recordFinished = deps.recordFinished ?? defaultRecordFinished;
|
|
613
|
+
// apra-fleet-k7b.2: optional durable history log (bin/serve.mjs wires the
|
|
614
|
+
// real createHistory() instance in); when absent, defaultRecordFinished
|
|
615
|
+
// still logs the watchdog line, it just skips the sprint-history.json
|
|
616
|
+
// event -- classification itself must never depend on history being
|
|
617
|
+
// wired.
|
|
618
|
+
const history = deps.history ?? null;
|
|
619
|
+
const intervalMs = Number.isInteger(deps.intervalMs) && deps.intervalMs > 0
|
|
620
|
+
? deps.intervalMs
|
|
621
|
+
: WATCHDOG_DEFAULT_INTERVAL_MS;
|
|
622
|
+
const launchFailedWindowMs = Number.isInteger(deps.launchFailedWindowMs) && deps.launchFailedWindowMs > 0
|
|
623
|
+
? deps.launchFailedWindowMs
|
|
624
|
+
: WATCHDOG_DEFAULT_LAUNCH_FAILED_WINDOW_MS;
|
|
625
|
+
const setIntervalFn = deps.setIntervalFn ?? setInterval;
|
|
626
|
+
const clearIntervalFn = deps.clearIntervalFn ?? clearInterval;
|
|
627
|
+
// apra-fleet-k7b.2: ISO-timestamp-prefix every log line from this module.
|
|
628
|
+
const logger = withTimestamps(deps.logger ?? console);
|
|
629
|
+
const logError = (...a) => (logger.error ?? logger.log)?.(...a);
|
|
630
|
+
const log = (...a) => (logger.log ?? logger.error)?.(...a);
|
|
631
|
+
|
|
632
|
+
/** Latest classification snapshot, refreshed each interval tick. */
|
|
633
|
+
let snapshot = [];
|
|
634
|
+
/** @type {ReturnType<typeof setInterval>|null} */
|
|
635
|
+
let timer = null;
|
|
636
|
+
// apra-fleet-eft.4.9: reentrancy guard for the setInterval-triggered
|
|
637
|
+
// classifyAll() below. classifyAll() does blocking per-sprint spawnSync
|
|
638
|
+
// cmdline reads plus ~1.5s HTTP liveness probes per sprint, so with
|
|
639
|
+
// several tracked sprints a single tick can exceed intervalMs and the
|
|
640
|
+
// NEXT tick can fire while the previous classifyAll() promise is still
|
|
641
|
+
// unresolved. Two overlapping classifyAll() runs both mutate the shared
|
|
642
|
+
// recordedCrashes/recordedFinishes/recordedLaunchFailures sets and can
|
|
643
|
+
// both race to write the same sprint's persisted state file
|
|
644
|
+
// (recordTerminalError/recordFinished), so a tick that fires while the
|
|
645
|
+
// prior one is still in flight is skipped here rather than allowed to
|
|
646
|
+
// start a second, overlapping classifyAll() run. This flag guards ONLY
|
|
647
|
+
// the interval-triggered path below -- a direct classifyAll() call (e.g.
|
|
648
|
+
// start()'s initial prime, or a test/consumer calling it explicitly)
|
|
649
|
+
// is never blocked by it.
|
|
650
|
+
let intervalClassifyInFlight = false;
|
|
651
|
+
// Sprint ids the CRASHED recorder has already fired for, so a wedged
|
|
652
|
+
// ledger entry that stays CRASHED across many interval ticks gets
|
|
653
|
+
// exactly ONE terminal-error log line + state write, not one per tick.
|
|
654
|
+
// Scoped to this watchdog instance/process lifetime -- a supervisor
|
|
655
|
+
// restart is itself a fresh watchdog instance, and by then the sprint's
|
|
656
|
+
// running/ state file already carries the persisted failed/lastError
|
|
657
|
+
// from before the restart, so there is nothing to re-report.
|
|
658
|
+
const recordedCrashes = new Set();
|
|
659
|
+
// apra-fleet-k7b.2: same one-shot-per-sprint-per-instance discipline as
|
|
660
|
+
// recordedCrashes above, but for the FINISHED transition -- without this
|
|
661
|
+
// guard classifySprint() running on every interval tick would append a
|
|
662
|
+
// fresh FINISHED event to sprint-history.json every tick for as long as
|
|
663
|
+
// the ledger keeps listing that (still-reserved-until-released) sprint.
|
|
664
|
+
const recordedFinishes = new Set();
|
|
665
|
+
// apra-fleet-gey.1: sprint ids for which we've recorded a launch-failed
|
|
666
|
+
// event, using the same one-shot discipline as recordedCrashes/recordedFinishes
|
|
667
|
+
// to avoid duplicate history entries on repeat ticks.
|
|
668
|
+
const recordedLaunchFailures = new Set();
|
|
669
|
+
|
|
670
|
+
/**
|
|
671
|
+
* apra-fleet-gey.1: detects if a sprint exited within the launch-failed window
|
|
672
|
+
* (i.e., very quickly after being reserved, with no terminal state recorded).
|
|
673
|
+
* Returns true only if we have reliable evidence that the child exited
|
|
674
|
+
* within the window AND that the child actually ran for a non-zero time.
|
|
675
|
+
* @param {object} entry - the ledger entry
|
|
676
|
+
* @returns {boolean}
|
|
677
|
+
*/
|
|
678
|
+
function isLaunchFailed(entry) {
|
|
679
|
+
// Must have exit info recorded (in-memory listener was not severed by restart)
|
|
680
|
+
if (!entry.exitedAt) return false;
|
|
681
|
+
// Must have reservation claim time
|
|
682
|
+
if (!entry.reservedAt) return false;
|
|
683
|
+
// Calculate how long the sprint ran
|
|
684
|
+
const exitedMs = new Date(entry.exitedAt).getTime();
|
|
685
|
+
const reservedMs = new Date(entry.reservedAt).getTime();
|
|
686
|
+
if (!Number.isFinite(exitedMs) || !Number.isFinite(reservedMs)) return false;
|
|
687
|
+
const runTimeMs = Math.max(0, exitedMs - reservedMs);
|
|
688
|
+
// True if the child exited within the launch window AND ran for at least
|
|
689
|
+
// 1ms (to distinguish real fast exits from test artifacts where
|
|
690
|
+
// reservedAt == exitedAt)
|
|
691
|
+
return runTimeMs > 0 && runTimeMs < launchFailedWindowMs;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
/**
|
|
695
|
+
* apra-fleet-0j1 / apra-fleet-cvb.1: release a still-held reservation the
|
|
696
|
+
* moment its sprint is classified CRASHED or FINISHED, mirroring what
|
|
697
|
+
* reconcile()'s restart-time sweep and forceRelease()'s operator-initiated
|
|
698
|
+
* teardown already do (src/supervisor/reconcile.mjs) -- same
|
|
699
|
+
* `ledger.release(sprintId)` call, just invoked continuously from this
|
|
700
|
+
* loop instead of once at startup / on-demand. This does NOT touch
|
|
701
|
+
* classification itself: it only runs AFTER classifySprint() has already
|
|
702
|
+
* decided CRASHED/FINISHED below.
|
|
703
|
+
*
|
|
704
|
+
* Idempotent by construction: `ledger.release()` deletes the whole entry
|
|
705
|
+
* and returns `false` (a no-op) when nothing is held, so calling this on
|
|
706
|
+
* every interval tick for a sprint that reconcile(), an operator's
|
|
707
|
+
* force-release, or this SAME function on a prior tick already released
|
|
708
|
+
* is always safe -- the audit log line / history event below fire only
|
|
709
|
+
* on the tick that ACTUALLY performs the release.
|
|
710
|
+
*
|
|
711
|
+
* A `ledger` collaborator without a `release()` method (e.g. a minimal
|
|
712
|
+
* test double exposing only `list()`, as most of this module's existing
|
|
713
|
+
* tests inject) is treated as "release not supported here" and silently
|
|
714
|
+
* skipped -- never an error, since createWatchdog()'s own required
|
|
715
|
+
* interface has always been `list()` alone; requiring more of every
|
|
716
|
+
* existing injected fake would be an (out-of-scope) breaking change to
|
|
717
|
+
* this module's test seam.
|
|
718
|
+
* @param {string} sprintId
|
|
719
|
+
* @param {'crashed'|'finished'} status
|
|
720
|
+
* @param {string} detail
|
|
721
|
+
*/
|
|
722
|
+
async function releaseTerminalReservation(sprintId, status, detail) {
|
|
723
|
+
if (typeof ledger.release !== 'function') return;
|
|
724
|
+
const entry = typeof ledger.get === 'function' ? ledger.get(sprintId) : undefined;
|
|
725
|
+
const members = entry?.members ?? [];
|
|
726
|
+
const issueRoots = entry?.issueRoots ?? [];
|
|
727
|
+
let released = false;
|
|
728
|
+
try {
|
|
729
|
+
released = await ledger.release(sprintId);
|
|
730
|
+
} catch (err) {
|
|
731
|
+
logError(`[watchdog] ledger.release failed for '${sprintId}':`, err);
|
|
732
|
+
return;
|
|
733
|
+
}
|
|
734
|
+
if (!released) return; // already released (reconcile, force-release, or a prior tick) -- nothing to report
|
|
735
|
+
// Same "[<module>] <what happened>" shape reconcile()'s own restart
|
|
736
|
+
// audit line uses ("[reconcile] restart: released N dead, retained M
|
|
737
|
+
// live"), so an operator watching the log sees WHY a reservation
|
|
738
|
+
// cleared regardless of which of the three release paths did it.
|
|
739
|
+
log(`[watchdog] auto-released reservation for '${sprintId}' (status=${status}, ${detail})`);
|
|
740
|
+
if (!history || typeof history.record !== 'function') return;
|
|
741
|
+
try {
|
|
742
|
+
const result = history.record({
|
|
743
|
+
sprintId,
|
|
744
|
+
event: HISTORY_EVENTS.AUTO_RELEASED,
|
|
745
|
+
reason: `watchdog: classified ${status} (${detail})`,
|
|
746
|
+
members,
|
|
747
|
+
issueRoots,
|
|
748
|
+
});
|
|
749
|
+
// history.record() is async; a rejection must never take the
|
|
750
|
+
// classifier down with it, same discipline as recordFinished's
|
|
751
|
+
// history call above.
|
|
752
|
+
if (result && typeof result.catch === 'function') {
|
|
753
|
+
result.catch((err) => logError(`[watchdog] history.record(AUTO_RELEASED) failed for '${sprintId}':`, err));
|
|
754
|
+
}
|
|
755
|
+
} catch (err) {
|
|
756
|
+
logError(`[watchdog] history.record(AUTO_RELEASED) failed for '${sprintId}':`, err);
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/**
|
|
761
|
+
* Classify a SINGLE ledger entry into exactly one of the six statuses.
|
|
762
|
+
* @param {{ sprintId: string, childPid: number|null }} entry
|
|
763
|
+
* @returns {Promise<{ sprintId: string, status: string, pidAlive: boolean, httpOk: boolean, childPid: number|null, port: number|undefined }>}
|
|
764
|
+
*/
|
|
765
|
+
async function classifySprint(entry) {
|
|
766
|
+
const sprintId = entry.sprintId;
|
|
767
|
+
const childPid = entry.childPid ?? null;
|
|
768
|
+
const port = resolvePort(sprintId);
|
|
769
|
+
// The reuse-guard marker is the sprint's unique --viewer-port string, so
|
|
770
|
+
// a PID-number collision with an unrelated process is not mistaken for
|
|
771
|
+
// our child. When the port is unknown (e.g. a child re-adopted across a
|
|
772
|
+
// restart before its port is rediscovered), we fall back to
|
|
773
|
+
// existence-only liveness rather than fabricating a status.
|
|
774
|
+
const marker = Number.isInteger(port) ? `--viewer-port ${port}` : null;
|
|
775
|
+
|
|
776
|
+
const pidAlive = childPid != null && (await isChildAlive(childPid, marker));
|
|
777
|
+
|
|
778
|
+
if (pidAlive) {
|
|
779
|
+
// PID alive: the HTTP signal splits healthy vs unresponsive. A hung
|
|
780
|
+
// child (HTTP silent) is unresponsive -- NEVER auto-declared crashed.
|
|
781
|
+
let httpOk = false;
|
|
782
|
+
if (Number.isInteger(port)) {
|
|
783
|
+
try {
|
|
784
|
+
httpOk = await probeHttp(port);
|
|
785
|
+
} catch {
|
|
786
|
+
httpOk = false;
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
// (apra-fleet-p2to.3.1) Only consult the pause-state probe once
|
|
790
|
+
// HTTP is already known reachable -- an unresponsive child cannot
|
|
791
|
+
// be asked anything, so it stays running-unresponsive exactly as
|
|
792
|
+
// before this probe existed. A live, HTTP-reachable child the
|
|
793
|
+
// engine has actually engaged a pause on (state.pause.status ===
|
|
794
|
+
// 'paused', NOT the deferred 'pausing') is classified PAUSED
|
|
795
|
+
// instead of running-healthy -- still a live, non-terminal
|
|
796
|
+
// status, so it falls through the same "pidAlive" branch as
|
|
797
|
+
// running-healthy/running-unresponsive and is NEVER routed
|
|
798
|
+
// through releaseTerminalReservation() below (that only runs in
|
|
799
|
+
// the PID-gone branch).
|
|
800
|
+
let pauseStatus = null;
|
|
801
|
+
if (httpOk && Number.isInteger(port)) {
|
|
802
|
+
try {
|
|
803
|
+
pauseStatus = await probePauseState(port);
|
|
804
|
+
} catch {
|
|
805
|
+
pauseStatus = null;
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
const status = pauseStatus === 'paused'
|
|
809
|
+
? WATCHDOG_STATUS.PAUSED
|
|
810
|
+
: (httpOk ? WATCHDOG_STATUS.RUNNING_HEALTHY : WATCHDOG_STATUS.RUNNING_UNRESPONSIVE);
|
|
811
|
+
return {
|
|
812
|
+
sprintId,
|
|
813
|
+
status,
|
|
814
|
+
pidAlive: true,
|
|
815
|
+
httpOk,
|
|
816
|
+
childPid,
|
|
817
|
+
port,
|
|
818
|
+
...(pauseStatus ? { pauseStatus } : {}),
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// PID gone: a persisted terminal state in old_runs/ (or legacy
|
|
823
|
+
// old_sprints/) means it FINISHED; its absence means either CRASHED
|
|
824
|
+
// or LAUNCH_FAILED (both died without recording a terminal state).
|
|
825
|
+
//
|
|
826
|
+
// apra-fleet-k7b.3: `detail` reports whatever this SAME instance's
|
|
827
|
+
// spawner actually witnessed (ledger.recordExit(), see spawner.mjs/
|
|
828
|
+
// bin/serve.mjs's wiring) -- e.g. "exited 1 at ..." -- falling back to
|
|
829
|
+
// the previous bare "pid gone" when nothing was recorded (a restart
|
|
830
|
+
// severed the in-memory exit listener before this instance ever saw
|
|
831
|
+
// the child exit; see this module's file-level doc comment).
|
|
832
|
+
const detail = formatExitDetail(entry);
|
|
833
|
+
// apra-fleet-k7b.2: resolves by run-id first, falling back to this
|
|
834
|
+
// reservation's own recorded `branch` for a pre-k7b.1 reservation
|
|
835
|
+
// (see defaultHasTerminalState()'s doc comment). Returns the parsed
|
|
836
|
+
// terminal state object (or `null`/falsy for an injected boolean
|
|
837
|
+
// test double), never just a boolean.
|
|
838
|
+
const terminalState = hasTerminalState(sprintId, entry.branch ?? null);
|
|
839
|
+
const finished = Boolean(terminalState);
|
|
840
|
+
|
|
841
|
+
// apra-fleet-gey.1: determine if this is a launch-failed sprint
|
|
842
|
+
// (exited within the launch window with no terminal state).
|
|
843
|
+
const launchFailed = !finished && isLaunchFailed(entry);
|
|
844
|
+
const classifiedStatus = finished ? WATCHDOG_STATUS.FINISHED : (launchFailed ? WATCHDOG_STATUS.LAUNCH_FAILED : WATCHDOG_STATUS.CRASHED);
|
|
845
|
+
|
|
846
|
+
if (!finished) {
|
|
847
|
+
if (launchFailed) {
|
|
848
|
+
// apra-fleet-gey.1: record the launch-failed event once per sprint
|
|
849
|
+
if (!recordedLaunchFailures.has(sprintId)) {
|
|
850
|
+
recordedLaunchFailures.add(sprintId);
|
|
851
|
+
if (history && typeof history.record === 'function') {
|
|
852
|
+
try {
|
|
853
|
+
const result = history.record({
|
|
854
|
+
sprintId,
|
|
855
|
+
event: HISTORY_EVENTS.LAUNCH_FAILED,
|
|
856
|
+
reason: `watchdog: child exited within launch window (${detail})`,
|
|
857
|
+
});
|
|
858
|
+
// history.record() is async; rejection must never take
|
|
859
|
+
// classification down with it, same discipline as other
|
|
860
|
+
// history recorders.
|
|
861
|
+
if (result && typeof result.catch === 'function') {
|
|
862
|
+
result.catch((err) => logError(`[watchdog] history.record(LAUNCH_FAILED) failed for '${sprintId}':`, err));
|
|
863
|
+
}
|
|
864
|
+
} catch (err) {
|
|
865
|
+
logError(`[watchdog] history.record(LAUNCH_FAILED) threw for '${sprintId}':`, err);
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
log(`[watchdog] LAUNCH_FAILED: Sprint '${sprintId}' exited within launch window (${detail}).`);
|
|
869
|
+
}
|
|
870
|
+
} else {
|
|
871
|
+
// apra-fleet-eft.20.3: this is the silent-death case the
|
|
872
|
+
// apra-fleet-eft.20 smoke test exposed -- a doer/orchestrator
|
|
873
|
+
// child died mid-Develop with zero diagnostic signal anywhere.
|
|
874
|
+
// Make it observable the FIRST time this sprint is classified
|
|
875
|
+
// CRASHED: an explicit log line, plus a persisted failed/lastError
|
|
876
|
+
// written into its own running/ state file.
|
|
877
|
+
if (!recordedCrashes.has(sprintId)) {
|
|
878
|
+
recordedCrashes.add(sprintId);
|
|
879
|
+
try {
|
|
880
|
+
recordTerminalError({ sprintId, childPid, env, logger, detail });
|
|
881
|
+
} catch (err) {
|
|
882
|
+
// The recorder itself must never take the classifier down
|
|
883
|
+
// with it -- classification (the watchdog's core contract)
|
|
884
|
+
// must keep proceeding even if diagnostics reporting fails.
|
|
885
|
+
logError(`[watchdog] recordTerminalError threw for '${sprintId}':`, err);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
} else if (!recordedFinishes.has(sprintId)) {
|
|
890
|
+
// apra-fleet-k7b.2: the FIRST time this sprint is classified
|
|
891
|
+
// FINISHED, log + (best-effort) record the engine's own
|
|
892
|
+
// terminalReason/verdict verbatim -- gated exactly like
|
|
893
|
+
// recordedCrashes above so a still-reserved (until released)
|
|
894
|
+
// finished sprint does not append a fresh history event on every
|
|
895
|
+
// subsequent interval tick.
|
|
896
|
+
recordedFinishes.add(sprintId);
|
|
897
|
+
try {
|
|
898
|
+
recordFinished({ sprintId, state: terminalState, env, logger, history });
|
|
899
|
+
} catch (err) {
|
|
900
|
+
logError(`[watchdog] recordFinished threw for '${sprintId}':`, err);
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
// apra-fleet-0j1 / apra-fleet-cvb.1: act on the classification just
|
|
904
|
+
// made (CRASHED or FINISHED or LAUNCH_FAILED, any terminal state
|
|
905
|
+
// classification means a still-reserved sprint's reservation is stale)
|
|
906
|
+
// -- run on EVERY tick, not gated by recordedCrashes/recordedFinishes
|
|
907
|
+
// above, since ledger.release() is itself idempotent (a no-op once
|
|
908
|
+
// released) and this is what makes a sprint's reservation clear within
|
|
909
|
+
// one watchdog poll interval of it going CRASHED/FINISHED/LAUNCH_FAILED,
|
|
910
|
+
// with no operator action or restart needed.
|
|
911
|
+
try {
|
|
912
|
+
await releaseTerminalReservation(sprintId, classifiedStatus, detail);
|
|
913
|
+
} catch (err) {
|
|
914
|
+
logError(`[watchdog] releaseTerminalReservation threw for '${sprintId}':`, err);
|
|
915
|
+
}
|
|
916
|
+
return {
|
|
917
|
+
sprintId,
|
|
918
|
+
status: classifiedStatus,
|
|
919
|
+
pidAlive: false,
|
|
920
|
+
httpOk: false,
|
|
921
|
+
childPid,
|
|
922
|
+
port,
|
|
923
|
+
detail,
|
|
924
|
+
// apra-fleet-k7b.2: only populated when finished === true;
|
|
925
|
+
// undefined (not null) for CRASHED/LAUNCH_FAILED so existing assertions
|
|
926
|
+
// on the classifier's return shape for the crashed/running paths are
|
|
927
|
+
// unaffected.
|
|
928
|
+
...(finished ? { terminalState } : {}),
|
|
929
|
+
};
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
/**
|
|
933
|
+
* Classify every ledger-listed sprint. The ledger snapshot is taken once so
|
|
934
|
+
* concurrent ledger mutation cannot disturb this pass.
|
|
935
|
+
* @returns {Promise<Array<object>>}
|
|
936
|
+
*/
|
|
937
|
+
async function classifyAll() {
|
|
938
|
+
const entries = ledger.list();
|
|
939
|
+
const results = await Promise.all(entries.map((e) => classifySprint(e)));
|
|
940
|
+
snapshot = results;
|
|
941
|
+
return results;
|
|
942
|
+
}
|
|
943
|
+
|
|
944
|
+
return {
|
|
945
|
+
name: 'watchdog',
|
|
946
|
+
intervalMs,
|
|
947
|
+
|
|
948
|
+
async start() {
|
|
949
|
+
if (timer) return;
|
|
950
|
+
// Prime an initial snapshot immediately so a status query right after
|
|
951
|
+
// start() does not race the first interval tick.
|
|
952
|
+
try { await classifyAll(); } catch (err) { logError('[watchdog] initial classify failed:', err); }
|
|
953
|
+
timer = setIntervalFn(() => {
|
|
954
|
+
// apra-fleet-eft.4.9: skip this tick if the previous
|
|
955
|
+
// interval-triggered classifyAll() has not resolved yet, so
|
|
956
|
+
// overlapping runs never race on recordedCrashes/
|
|
957
|
+
// recordedFinishes/recordedLaunchFailures or interleave
|
|
958
|
+
// their state-file writes. The `finally` always clears the
|
|
959
|
+
// flag -- even a classifyAll() rejection (already caught
|
|
960
|
+
// below) or a thrown error unlocks the NEXT tick.
|
|
961
|
+
if (intervalClassifyInFlight) {
|
|
962
|
+
log('[watchdog] tick skipped: previous classifyAll() still in flight');
|
|
963
|
+
return;
|
|
964
|
+
}
|
|
965
|
+
intervalClassifyInFlight = true;
|
|
966
|
+
classifyAll()
|
|
967
|
+
.catch((err) => logError('[watchdog] interval classify failed:', err))
|
|
968
|
+
.finally(() => { intervalClassifyInFlight = false; });
|
|
969
|
+
}, intervalMs);
|
|
970
|
+
// Never let the watchdog's own interval keep the process alive on its
|
|
971
|
+
// account -- the supervisor lifecycle owns process liveness.
|
|
972
|
+
if (timer && typeof timer.unref === 'function') timer.unref();
|
|
973
|
+
},
|
|
974
|
+
|
|
975
|
+
async stop() {
|
|
976
|
+
if (timer) {
|
|
977
|
+
clearIntervalFn(timer);
|
|
978
|
+
timer = null;
|
|
979
|
+
}
|
|
980
|
+
},
|
|
981
|
+
|
|
982
|
+
classifySprint,
|
|
983
|
+
classifyAll,
|
|
984
|
+
/** The most recent classification snapshot (last interval tick). */
|
|
985
|
+
getSnapshot() { return snapshot.map((s) => ({ ...s })); },
|
|
986
|
+
};
|
|
987
|
+
}
|