@monoes/monomindcli 2.16.12 → 2.16.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/core/coder.md +1 -1
- package/.claude/agents/engineering/engineering-devops-automator.md +1 -1
- package/.claude/agents/github/repo-architect.md +1 -1
- package/.claude/agents/testing/production-validator.md +1 -1
- package/.claude/agents/testing/testing-tool-evaluator.md +1 -1
- package/.claude/commands/github/pr-manager.md +1 -1
- package/.claude/helpers/pick-rank.cjs +34 -4
- package/.claude/skills/monolean-audit/SKILL.md +1 -1
- package/README.md +1 -1
- package/dist/src/commands/cleanup-data.d.ts +30 -0
- package/dist/src/commands/cleanup-data.d.ts.map +1 -0
- package/dist/src/commands/cleanup-data.js +138 -0
- package/dist/src/commands/cleanup-data.js.map +1 -0
- package/dist/src/commands/cleanup-scratch.d.ts +43 -0
- package/dist/src/commands/cleanup-scratch.d.ts.map +1 -0
- package/dist/src/commands/cleanup-scratch.js +125 -0
- package/dist/src/commands/cleanup-scratch.js.map +1 -0
- package/dist/src/commands/cleanup.d.ts +2 -39
- package/dist/src/commands/cleanup.d.ts.map +1 -1
- package/dist/src/commands/cleanup.js +5 -247
- package/dist/src/commands/cleanup.js.map +1 -1
- package/dist/src/commands/doctor-gitignore-checks.d.ts +10 -0
- package/dist/src/commands/doctor-gitignore-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-gitignore-checks.js +141 -0
- package/dist/src/commands/doctor-gitignore-checks.js.map +1 -0
- package/dist/src/commands/doctor-helpers-checks.d.ts +10 -0
- package/dist/src/commands/doctor-helpers-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-helpers-checks.js +292 -0
- package/dist/src/commands/doctor-helpers-checks.js.map +1 -0
- package/dist/src/commands/doctor-memory-checks.d.ts +47 -0
- package/dist/src/commands/doctor-memory-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-memory-checks.js +298 -0
- package/dist/src/commands/doctor-memory-checks.js.map +1 -0
- package/dist/src/commands/doctor-monograph-checks.d.ts +9 -0
- package/dist/src/commands/doctor-monograph-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-monograph-checks.js +331 -0
- package/dist/src/commands/doctor-monograph-checks.js.map +1 -0
- package/dist/src/commands/doctor-project-checks.d.ts +7 -66
- package/dist/src/commands/doctor-project-checks.d.ts.map +1 -1
- package/dist/src/commands/doctor-project-checks.js +10 -1331
- package/dist/src/commands/doctor-project-checks.js.map +1 -1
- package/dist/src/commands/doctor-routing-checks.d.ts +10 -0
- package/dist/src/commands/doctor-routing-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-routing-checks.js +194 -0
- package/dist/src/commands/doctor-routing-checks.js.map +1 -0
- package/dist/src/commands/doctor-worker-checks.d.ts +17 -0
- package/dist/src/commands/doctor-worker-checks.d.ts.map +1 -0
- package/dist/src/commands/doctor-worker-checks.js +122 -0
- package/dist/src/commands/doctor-worker-checks.js.map +1 -0
- package/dist/src/commands/hooks-routing-commands.js +1 -1
- package/dist/src/commands/hooks-routing-commands.js.map +1 -1
- package/dist/src/commands/init.d.ts.map +1 -1
- package/dist/src/commands/init.js +6 -15
- package/dist/src/commands/init.js.map +1 -1
- package/dist/src/commands/org-control.d.ts +117 -0
- package/dist/src/commands/org-control.d.ts.map +1 -0
- package/dist/src/commands/org-control.js +274 -0
- package/dist/src/commands/org-control.js.map +1 -0
- package/dist/src/commands/org-lifecycle.d.ts +7 -0
- package/dist/src/commands/org-lifecycle.d.ts.map +1 -0
- package/dist/src/commands/org-lifecycle.js +298 -0
- package/dist/src/commands/org-lifecycle.js.map +1 -0
- package/dist/src/commands/org-manage.d.ts +7 -0
- package/dist/src/commands/org-manage.d.ts.map +1 -0
- package/dist/src/commands/org-manage.js +357 -0
- package/dist/src/commands/org-manage.js.map +1 -0
- package/dist/src/commands/org-memory-command.d.ts +3 -0
- package/dist/src/commands/org-memory-command.d.ts.map +1 -0
- package/dist/src/commands/org-memory-command.js +178 -0
- package/dist/src/commands/org-memory-command.js.map +1 -0
- package/dist/src/commands/org-observe-approvals.d.ts +8 -0
- package/dist/src/commands/org-observe-approvals.d.ts.map +1 -0
- package/dist/src/commands/org-observe-approvals.js +211 -0
- package/dist/src/commands/org-observe-approvals.js.map +1 -0
- package/dist/src/commands/org-observe-checkpoint.d.ts +26 -0
- package/dist/src/commands/org-observe-checkpoint.d.ts.map +1 -0
- package/dist/src/commands/org-observe-checkpoint.js +180 -0
- package/dist/src/commands/org-observe-checkpoint.js.map +1 -0
- package/dist/src/commands/org-observe-config.d.ts +8 -0
- package/dist/src/commands/org-observe-config.d.ts.map +1 -0
- package/dist/src/commands/org-observe-config.js +160 -0
- package/dist/src/commands/org-observe-config.js.map +1 -0
- package/dist/src/commands/org-observe-gates.d.ts +4 -0
- package/dist/src/commands/org-observe-gates.d.ts.map +1 -0
- package/dist/src/commands/org-observe-gates.js +182 -0
- package/dist/src/commands/org-observe-gates.js.map +1 -0
- package/dist/src/commands/org-observe-inbox.d.ts +18 -0
- package/dist/src/commands/org-observe-inbox.d.ts.map +1 -0
- package/dist/src/commands/org-observe-inbox.js +154 -0
- package/dist/src/commands/org-observe-inbox.js.map +1 -0
- package/dist/src/commands/org-observe-logs.d.ts +31 -0
- package/dist/src/commands/org-observe-logs.d.ts.map +1 -0
- package/dist/src/commands/org-observe-logs.js +272 -0
- package/dist/src/commands/org-observe-logs.js.map +1 -0
- package/dist/src/commands/org-observe-questions.d.ts +8 -0
- package/dist/src/commands/org-observe-questions.d.ts.map +1 -0
- package/dist/src/commands/org-observe-questions.js +201 -0
- package/dist/src/commands/org-observe-questions.js.map +1 -0
- package/dist/src/commands/org-observe-report.d.ts +8 -0
- package/dist/src/commands/org-observe-report.d.ts.map +1 -0
- package/dist/src/commands/org-observe-report.js +366 -0
- package/dist/src/commands/org-observe-report.js.map +1 -0
- package/dist/src/commands/org-observe-shared.d.ts +20 -0
- package/dist/src/commands/org-observe-shared.d.ts.map +1 -0
- package/dist/src/commands/org-observe-shared.js +55 -0
- package/dist/src/commands/org-observe-shared.js.map +1 -0
- package/dist/src/commands/org-observe.d.ts +9 -101
- package/dist/src/commands/org-observe.d.ts.map +1 -1
- package/dist/src/commands/org-observe.js +12 -1675
- package/dist/src/commands/org-observe.js.map +1 -1
- package/dist/src/commands/org-poll.d.ts +57 -0
- package/dist/src/commands/org-poll.d.ts.map +1 -0
- package/dist/src/commands/org-poll.js +194 -0
- package/dist/src/commands/org-poll.js.map +1 -0
- package/dist/src/commands/org-run.d.ts +3 -0
- package/dist/src/commands/org-run.d.ts.map +1 -0
- package/dist/src/commands/org-run.js +449 -0
- package/dist/src/commands/org-run.js.map +1 -0
- package/dist/src/commands/org-serve.d.ts +19 -0
- package/dist/src/commands/org-serve.d.ts.map +1 -0
- package/dist/src/commands/org-serve.js +395 -0
- package/dist/src/commands/org-serve.js.map +1 -0
- package/dist/src/commands/org-subcommands-config.d.ts +8 -0
- package/dist/src/commands/org-subcommands-config.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-config.js +93 -0
- package/dist/src/commands/org-subcommands-config.js.map +1 -0
- package/dist/src/commands/org-subcommands-hil.d.ts +11 -0
- package/dist/src/commands/org-subcommands-hil.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-hil.js +216 -0
- package/dist/src/commands/org-subcommands-hil.js.map +1 -0
- package/dist/src/commands/org-subcommands-observe.d.ts +12 -0
- package/dist/src/commands/org-subcommands-observe.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-observe.js +249 -0
- package/dist/src/commands/org-subcommands-observe.js.map +1 -0
- package/dist/src/commands/org-subcommands-runtime.d.ts +12 -0
- package/dist/src/commands/org-subcommands-runtime.d.ts.map +1 -0
- package/dist/src/commands/org-subcommands-runtime.js +163 -0
- package/dist/src/commands/org-subcommands-runtime.js.map +1 -0
- package/dist/src/commands/org.d.ts +3 -151
- package/dist/src/commands/org.d.ts.map +1 -1
- package/dist/src/commands/org.js +43 -2735
- package/dist/src/commands/org.js.map +1 -1
- package/dist/src/index.js +2 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/init/never-commit.d.ts +3 -3
- package/dist/src/init/types.d.ts +1 -1
- package/dist/src/init/types.js +1 -1
- package/dist/src/knowledge/document-store.js +1 -1
- package/dist/src/knowledge/document-store.js.map +1 -1
- package/dist/src/mcp-tools/hooks-edit-command.d.ts +10 -0
- package/dist/src/mcp-tools/hooks-edit-command.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-edit-command.js +247 -0
- package/dist/src/mcp-tools/hooks-edit-command.js.map +1 -0
- package/dist/src/mcp-tools/hooks-learning.d.ts +9 -0
- package/dist/src/mcp-tools/hooks-learning.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-learning.js +408 -0
- package/dist/src/mcp-tools/hooks-learning.js.map +1 -0
- package/dist/src/mcp-tools/hooks-metrics-list.d.ts +8 -0
- package/dist/src/mcp-tools/hooks-metrics-list.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-metrics-list.js +205 -0
- package/dist/src/mcp-tools/hooks-metrics-list.js.map +1 -0
- package/dist/src/mcp-tools/hooks-route.d.ts +12 -0
- package/dist/src/mcp-tools/hooks-route.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-route.js +251 -0
- package/dist/src/mcp-tools/hooks-route.js.map +1 -0
- package/dist/src/mcp-tools/hooks-routing.d.ts +6 -27
- package/dist/src/mcp-tools/hooks-routing.d.ts.map +1 -1
- package/dist/src/mcp-tools/hooks-routing.js +6 -1727
- package/dist/src/mcp-tools/hooks-routing.js.map +1 -1
- package/dist/src/mcp-tools/hooks-session.d.ts +9 -0
- package/dist/src/mcp-tools/hooks-session.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-session.js +243 -0
- package/dist/src/mcp-tools/hooks-session.js.map +1 -0
- package/dist/src/mcp-tools/hooks-task.d.ts +17 -0
- package/dist/src/mcp-tools/hooks-task.d.ts.map +1 -0
- package/dist/src/mcp-tools/hooks-task.js +427 -0
- package/dist/src/mcp-tools/hooks-task.js.map +1 -0
- package/dist/src/mcp-tools/hooks-tools.d.ts +2 -1
- package/dist/src/mcp-tools/hooks-tools.d.ts.map +1 -1
- package/dist/src/mcp-tools/hooks-tools.js +2 -1
- package/dist/src/mcp-tools/hooks-tools.js.map +1 -1
- package/dist/src/mcp-tools/memory-tools.js +1 -1
- package/dist/src/mcp-tools/memory-tools.js.map +1 -1
- package/dist/src/memory/bm25-index.d.ts +2 -2
- package/dist/src/memory/bm25-index.js +2 -2
- package/dist/src/memory/embedding-operations.js +2 -2
- package/dist/src/memory/embedding-operations.js.map +1 -1
- package/dist/src/memory/entity-name-key.d.ts +1 -1
- package/dist/src/memory/entity-name-key.js +1 -1
- package/dist/src/memory/hnsw-operations.d.ts +1 -1
- package/dist/src/memory/hnsw-operations.js +1 -1
- package/dist/src/memory/memory-bridge-backend.d.ts +43 -0
- package/dist/src/memory/memory-bridge-backend.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-backend.js +363 -0
- package/dist/src/memory/memory-bridge-backend.js.map +1 -0
- package/dist/src/memory/memory-bridge-core.d.ts +42 -0
- package/dist/src/memory/memory-bridge-core.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-core.js +126 -0
- package/dist/src/memory/memory-bridge-core.js.map +1 -0
- package/dist/src/memory/memory-bridge-entries.d.ts +131 -0
- package/dist/src/memory/memory-bridge-entries.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-entries.js +227 -0
- package/dist/src/memory/memory-bridge-entries.js.map +1 -0
- package/dist/src/memory/memory-bridge-feedback.d.ts +60 -0
- package/dist/src/memory/memory-bridge-feedback.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-feedback.js +118 -0
- package/dist/src/memory/memory-bridge-feedback.js.map +1 -0
- package/dist/src/memory/memory-bridge-paths.d.ts +123 -0
- package/dist/src/memory/memory-bridge-paths.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-paths.js +335 -0
- package/dist/src/memory/memory-bridge-paths.js.map +1 -0
- package/dist/src/memory/memory-bridge-search.d.ts +48 -0
- package/dist/src/memory/memory-bridge-search.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-search.js +383 -0
- package/dist/src/memory/memory-bridge-search.js.map +1 -0
- package/dist/src/memory/memory-bridge-sessions.d.ts +38 -0
- package/dist/src/memory/memory-bridge-sessions.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-sessions.js +109 -0
- package/dist/src/memory/memory-bridge-sessions.js.map +1 -0
- package/dist/src/memory/memory-bridge-store.d.ts +58 -0
- package/dist/src/memory/memory-bridge-store.d.ts.map +1 -0
- package/dist/src/memory/memory-bridge-store.js +201 -0
- package/dist/src/memory/memory-bridge-store.js.map +1 -0
- package/dist/src/memory/memory-bridge.d.ts +10 -438
- package/dist/src/memory/memory-bridge.d.ts.map +1 -1
- package/dist/src/memory/memory-bridge.js +14 -1787
- package/dist/src/memory/memory-bridge.js.map +1 -1
- package/dist/src/memory/memory-kg-claims.d.ts +52 -0
- package/dist/src/memory/memory-kg-claims.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-claims.js +99 -0
- package/dist/src/memory/memory-kg-claims.js.map +1 -0
- package/dist/src/memory/memory-kg-index.d.ts +86 -0
- package/dist/src/memory/memory-kg-index.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-index.js +256 -0
- package/dist/src/memory/memory-kg-index.js.map +1 -0
- package/dist/src/memory/memory-kg-ingest.d.ts +44 -0
- package/dist/src/memory/memory-kg-ingest.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-ingest.js +361 -0
- package/dist/src/memory/memory-kg-ingest.js.map +1 -0
- package/dist/src/memory/memory-kg-model.d.ts +168 -0
- package/dist/src/memory/memory-kg-model.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-model.js +158 -0
- package/dist/src/memory/memory-kg-model.js.map +1 -0
- package/dist/src/memory/memory-kg-names.d.ts +64 -0
- package/dist/src/memory/memory-kg-names.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-names.js +187 -0
- package/dist/src/memory/memory-kg-names.js.map +1 -0
- package/dist/src/memory/memory-kg-promote.d.ts +79 -0
- package/dist/src/memory/memory-kg-promote.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-promote.js +248 -0
- package/dist/src/memory/memory-kg-promote.js.map +1 -0
- package/dist/src/memory/memory-kg-rebuild.d.ts +77 -0
- package/dist/src/memory/memory-kg-rebuild.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-rebuild.js +299 -0
- package/dist/src/memory/memory-kg-rebuild.js.map +1 -0
- package/dist/src/memory/memory-kg-rollback.d.ts +47 -0
- package/dist/src/memory/memory-kg-rollback.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-rollback.js +193 -0
- package/dist/src/memory/memory-kg-rollback.js.map +1 -0
- package/dist/src/memory/memory-kg-rules.d.ts +64 -0
- package/dist/src/memory/memory-kg-rules.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-rules.js +251 -0
- package/dist/src/memory/memory-kg-rules.js.map +1 -0
- package/dist/src/memory/memory-kg-scan.d.ts +118 -0
- package/dist/src/memory/memory-kg-scan.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-scan.js +209 -0
- package/dist/src/memory/memory-kg-scan.js.map +1 -0
- package/dist/src/memory/memory-kg-search.d.ts +80 -0
- package/dist/src/memory/memory-kg-search.d.ts.map +1 -0
- package/dist/src/memory/memory-kg-search.js +239 -0
- package/dist/src/memory/memory-kg-search.js.map +1 -0
- package/dist/src/memory/memory-kg.d.ts +19 -547
- package/dist/src/memory/memory-kg.d.ts.map +1 -1
- package/dist/src/memory/memory-kg.js +10 -2389
- package/dist/src/memory/memory-kg.js.map +1 -1
- package/dist/src/orgrt/daemon-types.d.ts +196 -0
- package/dist/src/orgrt/daemon-types.d.ts.map +1 -0
- package/dist/src/orgrt/daemon-types.js +38 -0
- package/dist/src/orgrt/daemon-types.js.map +1 -0
- package/dist/src/orgrt/daemon.d.ts +70 -300
- package/dist/src/orgrt/daemon.d.ts.map +1 -1
- package/dist/src/orgrt/daemon.js +41 -2536
- package/dist/src/orgrt/daemon.js.map +1 -1
- package/dist/src/orgrt/idle-watchdog.d.ts +39 -0
- package/dist/src/orgrt/idle-watchdog.d.ts.map +1 -0
- package/dist/src/orgrt/idle-watchdog.js +289 -0
- package/dist/src/orgrt/idle-watchdog.js.map +1 -0
- package/dist/src/orgrt/org-reload.d.ts +7 -0
- package/dist/src/orgrt/org-reload.d.ts.map +1 -0
- package/dist/src/orgrt/org-reload.js +111 -0
- package/dist/src/orgrt/org-reload.js.map +1 -0
- package/dist/src/orgrt/org-start-steps.d.ts +21 -0
- package/dist/src/orgrt/org-start-steps.d.ts.map +1 -0
- package/dist/src/orgrt/org-start-steps.js +305 -0
- package/dist/src/orgrt/org-start-steps.js.map +1 -0
- package/dist/src/orgrt/org-start.d.ts +7 -0
- package/dist/src/orgrt/org-start.d.ts.map +1 -0
- package/dist/src/orgrt/org-start.js +435 -0
- package/dist/src/orgrt/org-start.js.map +1 -0
- package/dist/src/orgrt/org-state-file.d.ts +8 -0
- package/dist/src/orgrt/org-state-file.d.ts.map +1 -0
- package/dist/src/orgrt/org-state-file.js +116 -0
- package/dist/src/orgrt/org-state-file.js.map +1 -0
- package/dist/src/orgrt/org-stop.d.ts +6 -0
- package/dist/src/orgrt/org-stop.d.ts.map +1 -0
- package/dist/src/orgrt/org-stop.js +332 -0
- package/dist/src/orgrt/org-stop.js.map +1 -0
- package/dist/src/orgrt/org-tools.d.ts +11 -0
- package/dist/src/orgrt/org-tools.d.ts.map +1 -0
- package/dist/src/orgrt/org-tools.js +305 -0
- package/dist/src/orgrt/org-tools.js.map +1 -0
- package/dist/src/orgrt/role-incarnation.d.ts +18 -0
- package/dist/src/orgrt/role-incarnation.d.ts.map +1 -0
- package/dist/src/orgrt/role-incarnation.js +374 -0
- package/dist/src/orgrt/role-incarnation.js.map +1 -0
- package/dist/src/orgrt/role-respawn.d.ts +7 -0
- package/dist/src/orgrt/role-respawn.d.ts.map +1 -0
- package/dist/src/orgrt/role-respawn.js +318 -0
- package/dist/src/orgrt/role-respawn.js.map +1 -0
- package/dist/src/orgrt/role-session-opts.d.ts +389 -0
- package/dist/src/orgrt/role-session-opts.d.ts.map +1 -0
- package/dist/src/orgrt/role-session-opts.js +289 -0
- package/dist/src/orgrt/role-session-opts.js.map +1 -0
- package/dist/src/orgrt/runner-resolve.d.ts +43 -0
- package/dist/src/orgrt/runner-resolve.d.ts.map +1 -0
- package/dist/src/orgrt/runner-resolve.js +77 -0
- package/dist/src/orgrt/runner-resolve.js.map +1 -0
- package/dist/src/orgrt/session-gate.d.ts +34 -0
- package/dist/src/orgrt/session-gate.d.ts.map +1 -0
- package/dist/src/orgrt/session-gate.js +74 -0
- package/dist/src/orgrt/session-gate.js.map +1 -0
- package/dist/src/orgrt/session-prompt.d.ts +31 -0
- package/dist/src/orgrt/session-prompt.d.ts.map +1 -0
- package/dist/src/orgrt/session-prompt.js +115 -0
- package/dist/src/orgrt/session-prompt.js.map +1 -0
- package/dist/src/orgrt/session-run.d.ts +18 -0
- package/dist/src/orgrt/session-run.d.ts.map +1 -0
- package/dist/src/orgrt/session-run.js +429 -0
- package/dist/src/orgrt/session-run.js.map +1 -0
- package/dist/src/orgrt/session-stream.d.ts +21 -0
- package/dist/src/orgrt/session-stream.d.ts.map +1 -0
- package/dist/src/orgrt/session-stream.js +173 -0
- package/dist/src/orgrt/session-stream.js.map +1 -0
- package/dist/src/orgrt/session-types.d.ts +219 -0
- package/dist/src/orgrt/session-types.d.ts.map +1 -0
- package/dist/src/orgrt/session-types.js +2 -0
- package/dist/src/orgrt/session-types.js.map +1 -0
- package/dist/src/orgrt/session-usage.d.ts +29 -0
- package/dist/src/orgrt/session-usage.d.ts.map +1 -0
- package/dist/src/orgrt/session-usage.js +64 -0
- package/dist/src/orgrt/session-usage.js.map +1 -0
- package/dist/src/orgrt/session.d.ts +5 -281
- package/dist/src/orgrt/session.d.ts.map +1 -1
- package/dist/src/orgrt/session.js +8 -1117
- package/dist/src/orgrt/session.js.map +1 -1
- package/dist/src/ui/org-hil.mjs +1 -1
- package/dist/src/ui/routes-org-agents.mjs +318 -0
- package/dist/src/ui/routes-org-approvals.mjs +133 -0
- package/dist/src/ui/routes-org-config.mjs +459 -0
- package/dist/src/ui/routes-org-control.mjs +300 -0
- package/dist/src/ui/routes-org-files.mjs +172 -0
- package/dist/src/ui/routes-org-helpers.mjs +84 -0
- package/dist/src/ui/routes-org-knowledge.mjs +262 -0
- package/dist/src/ui/routes-org-lifecycle.mjs +224 -0
- package/dist/src/ui/routes-org-live.mjs +278 -0
- package/dist/src/ui/routes-org-mastermind.mjs +346 -0
- package/dist/src/ui/routes-org-planning.mjs +398 -0
- package/dist/src/ui/routes-org-runs.mjs +430 -0
- package/dist/src/ui/routes-org-status.mjs +342 -0
- package/dist/src/ui/routes-org-workflows.mjs +212 -0
- package/dist/src/ui/routes-org.mjs +26 -3847
- package/dist/src/ui/server.mjs +1 -1
- package/dist/src/utils/native-error.d.ts +1 -1
- package/dist/src/utils/native-error.js +1 -1
- package/dist/src/utils/paths.d.ts +2 -1
- package/dist/src/utils/paths.d.ts.map +1 -1
- package/dist/src/utils/paths.js +2 -1
- package/dist/src/utils/paths.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/org-skills/adversarial-reviewer/SKILL.md +1 -1
- package/org-skills/board-deck-builder/SKILL.md +1 -1
- package/org-skills/code-reviewer/SKILL.md +1 -1
- package/org-skills/debugging-wizard/SKILL.md +1 -1
- package/org-skills/scientific-thinking-literature-review/SKILL.md +1 -1
- package/org-skills/scientific-thinking-scholar-evaluation/SKILL.md +1 -1
- package/package.json +5 -5
package/dist/src/orgrt/daemon.js
CHANGED
|
@@ -1,64 +1,25 @@
|
|
|
1
1
|
// packages/@monomind/cli/src/orgrt/daemon.ts
|
|
2
2
|
// monolean: single-process inter-org — upgrade path = daemon-to-daemon HTTP when multi-host is real
|
|
3
|
-
import {
|
|
4
|
-
import { randomUUID } from 'node:crypto';
|
|
5
|
-
import { existsSync, mkdirSync, readFileSync, unlinkSync } from 'node:fs';
|
|
3
|
+
import { existsSync } from 'node:fs';
|
|
6
4
|
import { isAbsolute, join } from 'node:path';
|
|
7
|
-
import { resolveOrgDefBlueprints } from '../catalog/blueprints.js';
|
|
8
|
-
import { writeJsonFileAtomic } from '../utils/json-file.js';
|
|
9
|
-
import { configureResourceLimits, getResourceLimits, reapOrphanedSdkProcesses, } from '../utils/resource-governor.js';
|
|
10
|
-
import { AntigravityAgentRunner } from './antigravity-runner.js';
|
|
11
5
|
// ── Extracted module imports ────────────────────────────────────────────
|
|
12
6
|
import * as approvalOps from './approvals.js';
|
|
13
|
-
import { wakeDueBlockRechecks } from './block-recheck.js';
|
|
14
|
-
import { BrokerLease, normalizeCredential } from './broker.js';
|
|
15
|
-
import { onBudgetBusEvent, reopenBudgetClosedRoles, rolesOnDefTokenCaps, } from './budget-closure.js';
|
|
16
|
-
import { OrgBus } from './bus.js';
|
|
17
|
-
import { captureCheckpoint, generateChecksum, isCheckpointExpired, migrateCheckpoint, restoredRoleStatus, restoreMailboxQueue, validateCheckpoint, } from './checkpoint.js';
|
|
18
7
|
import * as checkpointOps from './checkpoint-ops.js';
|
|
19
|
-
import { CodexAgentRunner } from './codex-runner.js';
|
|
20
|
-
import { checkCompletion } from './completion-gate.js';
|
|
21
|
-
import { CopilotAgentRunner } from './copilot-runner.js';
|
|
22
8
|
import * as crossOrg from './cross-org.js';
|
|
23
|
-
import { CrushAgentRunner } from './crush-runner.js';
|
|
24
9
|
import * as decisionOps from './decisions.js';
|
|
25
|
-
import { openTaskCount } from './decisions.js';
|
|
26
|
-
import { agentRoles, hasActiveEndpointWait, isEndpointRole, retryQueuedEndpoints, startEndpointRetryLoop, stopEndpointRetries, } from './endpoint-roles.js';
|
|
27
|
-
import { createFenceForRole, loadGlobalFenceConfig, mergeFenceConfigs, } from './fence.js';
|
|
28
|
-
import { fileToolRoots } from './file-roots.js';
|
|
29
|
-
import { attachForwarder } from './forwarder.js';
|
|
30
|
-
import { GrokAgentRunner } from './grok-runner.js';
|
|
31
|
-
import { HermesAgentRunner } from './hermes-runner.js';
|
|
32
|
-
import { advanceHold, clearIdleRecord, hookedOnWork, noProgressRoles, projectIdleStop, writeIdleRecord, } from './idle-deadline.js';
|
|
33
|
-
import { drainInbox, newMessageId, queueMessage } from './inbox.js';
|
|
34
|
-
import { KimiCodeAgentRunner } from './kimicode-runner.js';
|
|
35
|
-
import { loadoutCatalog, resolveLoadout, sessionLoadoutFor, taskTag } from './loadouts.js';
|
|
36
|
-
import { isRecoverableCloseReason, Mailbox } from './mailbox.js';
|
|
37
|
-
import { OpencodeAgentRunner } from './opencode-runner.js';
|
|
38
10
|
import * as orgMemory from './org-memory.js';
|
|
39
|
-
import
|
|
40
|
-
import
|
|
41
|
-
import
|
|
42
|
-
import
|
|
43
|
-
import { resolveRoleProvider } from './provider.js';
|
|
11
|
+
import * as orgReload from './org-reload.js';
|
|
12
|
+
import * as orgStart from './org-start.js';
|
|
13
|
+
import * as orgStateFile from './org-state-file.js';
|
|
14
|
+
import * as orgStop from './org-stop.js';
|
|
44
15
|
import * as questionOps from './questions.js';
|
|
45
|
-
import
|
|
46
|
-
import
|
|
47
|
-
import {
|
|
48
|
-
import { buildRespawnReceipt, computeReplacementBudget, mergeEffectiveRoleConfig, redactRoleConfig, validateRespawnInput, } from './role-slot.js';
|
|
49
|
-
import { currentRoleTrace, endTurn, withTrace } from './role-trace.js';
|
|
16
|
+
import * as roleIncarnation from './role-incarnation.js';
|
|
17
|
+
import * as roleRespawn from './role-respawn.js';
|
|
18
|
+
import { currentRoleTrace } from './role-trace.js';
|
|
50
19
|
import { buildRuntimeOptions } from './runtime-options.js';
|
|
51
|
-
import { sandboxStubs } from './sandbox-stubs.js';
|
|
52
20
|
import * as scheduler from './scheduler-integration.js';
|
|
53
|
-
import {
|
|
54
|
-
import {
|
|
55
|
-
import { effectiveToolProviders } from './skill-library.js';
|
|
56
|
-
import { TaskProcesses } from './task-cancel.js';
|
|
57
|
-
import { TaskDag } from './task-dag.js';
|
|
58
|
-
import { resolveAutoAssignee } from './task-match.js';
|
|
59
|
-
import { roleProviderPrefixes, ToolProviderHub } from './tool-providers.js';
|
|
60
|
-
import { ORG_DIR, OrgDefSchema, } from './types.js';
|
|
61
|
-
import { VercelAgentRunner } from './vercel-runner.js';
|
|
21
|
+
import { ToolProviderHub } from './tool-providers.js';
|
|
22
|
+
import { ORG_DIR, } from './types.js';
|
|
62
23
|
/** OpenTelemetry tracing helper - creates spans for major operations */
|
|
63
24
|
class _OtelTracer {
|
|
64
25
|
enabled = false;
|
|
@@ -87,178 +48,11 @@ class _OtelTracer {
|
|
|
87
48
|
// Could emit to bus for collection
|
|
88
49
|
}
|
|
89
50
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
/** Auto-resolve runtime from provider kind. Returns undefined for Claude default. */
|
|
95
|
-
function autoRuntimeFromProvider(kind) {
|
|
96
|
-
if (kind === 'vercel-api-key')
|
|
97
|
-
return 'vercel';
|
|
98
|
-
if (kind === 'codex')
|
|
99
|
-
return 'codex';
|
|
100
|
-
if (kind === 'antigravity')
|
|
101
|
-
return 'antigravity';
|
|
102
|
-
return undefined;
|
|
103
|
-
}
|
|
104
|
-
export function resolveRunner(orgRuntime, providerKind, provider) {
|
|
105
|
-
const selected = orgRuntime ??
|
|
106
|
-
autoRuntimeFromProvider(providerKind) ??
|
|
107
|
-
process.env.MONOMIND_RUNTIME;
|
|
108
|
-
if (selected === 'opencode')
|
|
109
|
-
return new OpencodeAgentRunner();
|
|
110
|
-
if (selected === 'kimicode')
|
|
111
|
-
return new KimiCodeAgentRunner();
|
|
112
|
-
if (selected === 'vercel')
|
|
113
|
-
return new VercelAgentRunner();
|
|
114
|
-
if (selected === 'codex')
|
|
115
|
-
return new CodexAgentRunner();
|
|
116
|
-
if (selected === 'antigravity')
|
|
117
|
-
return new AntigravityAgentRunner();
|
|
118
|
-
if (selected === 'grok')
|
|
119
|
-
return new GrokAgentRunner();
|
|
120
|
-
if (selected === 'qwen')
|
|
121
|
-
return new QwenAgentRunner();
|
|
122
|
-
if (selected === 'crush') {
|
|
123
|
-
// Issue #177: usage-proxy accounting is opt-in via provider.usageProxy +
|
|
124
|
-
// provider.baseUrl (the upstream the crush CLI's own provider config
|
|
125
|
-
// points at). Absent either, CrushAgentRunner falls back to its
|
|
126
|
-
// documented 0-token behavior — this never blocks a turn either way.
|
|
127
|
-
if (provider?.usageProxy && provider.baseUrl) {
|
|
128
|
-
return new CrushAgentRunner({
|
|
129
|
-
usageProxy: { upstreamBaseUrl: provider.baseUrl, baseUrlEnvVar: provider.usageProxyEnvVar },
|
|
130
|
-
});
|
|
131
|
-
}
|
|
132
|
-
return new CrushAgentRunner();
|
|
133
|
-
}
|
|
134
|
-
if (selected === 'copilot')
|
|
135
|
-
return new CopilotAgentRunner();
|
|
136
|
-
if (selected === 'pi')
|
|
137
|
-
return new PiAgentRunner();
|
|
138
|
-
if (selected === 'pi-rpc')
|
|
139
|
-
return new PiRpcAgentRunner();
|
|
140
|
-
if (selected === 'qwen-rpc')
|
|
141
|
-
return new QwenRpcAgentRunner();
|
|
142
|
-
if (selected === 'hermes')
|
|
143
|
-
return new HermesAgentRunner();
|
|
144
|
-
return undefined;
|
|
145
|
-
}
|
|
146
|
-
/** Per-session variant: a role's own `runtime` field wins over the org-level
|
|
147
|
-
* one (and the env var) — including `role.runtime === 'claude'`, which forces
|
|
148
|
-
* the default Claude path even when the org/env select another runtime.
|
|
149
|
-
* Roles without a `runtime` inherit the org-level resolution unchanged.
|
|
150
|
-
* If no explicit runtime is set, auto-resolve from the provider kind. */
|
|
151
|
-
export function resolveRoleRunner(roleRuntime, orgRuntime, roleProviderKind, orgProviderKind, roleProvider) {
|
|
152
|
-
const explicit = roleRuntime ?? orgRuntime;
|
|
153
|
-
if (explicit)
|
|
154
|
-
return resolveRunner(explicit, undefined, roleProvider);
|
|
155
|
-
return resolveRunner(undefined, roleProviderKind ?? orgProviderKind, roleProvider);
|
|
156
|
-
}
|
|
51
|
+
export { activeRoleCount, roleTokenBudget, ScrollbackBuffer, } from './daemon-types.js';
|
|
52
|
+
export { resolvedIdleNudgeCount } from './idle-watchdog.js';
|
|
53
|
+
export { resolveOrgComplete } from './role-session-opts.js';
|
|
54
|
+
export { resolveRoleRunner, resolveRunner, } from './runner-resolve.js';
|
|
157
55
|
export { resolveAutoAssignee } from './task-match.js';
|
|
158
|
-
/** Per-role token budget: a role's own `budget_tokens` wins; otherwise the
|
|
159
|
-
* even split of run_config.budget_tokens across all roles. */
|
|
160
|
-
export function roleTokenBudget(role, def) {
|
|
161
|
-
return (role.budget_tokens ??
|
|
162
|
-
Math.floor((def.run_config.budget_tokens ?? 1_000_000) / Math.max(1, agentRoles(def.roles).length)));
|
|
163
|
-
}
|
|
164
|
-
/** Idle watchdog's per-tick recovery check: given the previous nudge timestamp,
|
|
165
|
-
* the cumulative nudge count, and the timestamp of the most recent real tool
|
|
166
|
-
* call, returns the nudge count that should carry forward now that fresh
|
|
167
|
-
* activity means the org is no longer idle.
|
|
168
|
-
*
|
|
169
|
-
* `nudgedAt !== 0` means we're recovering from an outstanding nudge. Resetting
|
|
170
|
-
* the counter here means it only ever tracks UNRESOLVED idle spells in a row,
|
|
171
|
-
* not a lifetime total — a long-running org that goes idle and recovers any
|
|
172
|
-
* number of times (e.g. periodic checkpoints on a slow background task) is
|
|
173
|
-
* never punished for having had several separate, healthy idle spells over
|
|
174
|
-
* its lifetime. Before this existed, `nudges` only ever incremented, so a run
|
|
175
|
-
* that answered every single nudge with real work still hit the "org idle
|
|
176
|
-
* again after 3 nudges" cap and got force-stopped on its 4th idle spell —
|
|
177
|
-
* observed live killing an in-progress 24h soak test under 90 minutes in.
|
|
178
|
-
*
|
|
179
|
-
* But "recovering" must mean genuine forward progress, not just any bus
|
|
180
|
-
* event — a bare, content-free reply to the nudge (a boss that answers with
|
|
181
|
-
* "✓ Complete" and calls no tools at all) still updates lastActivity, so the
|
|
182
|
-
* org isn't flagged as silent, but it accomplishes nothing: nobody outside
|
|
183
|
-
* the boss's own turn ever sees it, since role coordination only happens via
|
|
184
|
-
* tool calls (org_send, org_task, ...). Requiring `lastToolActivity >=
|
|
185
|
-
* nudgedAt` — a real tool call happened AFTER this nudge was sent — closes
|
|
186
|
-
* that gap: observed live, a boss stuck responding to four consecutive
|
|
187
|
-
* 10-minute nudges with one-line acknowledgments and zero tool calls looped
|
|
188
|
-
* indefinitely making no progress, because every trivial reply reset the cap
|
|
189
|
-
* that was supposed to catch exactly this. */
|
|
190
|
-
export function resolvedIdleNudgeCount(nudgedAt, nudges, lastToolActivity) {
|
|
191
|
-
return nudgedAt !== 0 && lastToolActivity >= nudgedAt ? 0 : nudges;
|
|
192
|
-
}
|
|
193
|
-
/** Bounded ring buffer for agent terminal scrollback. */
|
|
194
|
-
export class ScrollbackBuffer {
|
|
195
|
-
maxLines;
|
|
196
|
-
lines = [];
|
|
197
|
-
constructor(maxLines = 500) {
|
|
198
|
-
this.maxLines = maxLines;
|
|
199
|
-
}
|
|
200
|
-
push(line) {
|
|
201
|
-
this.lines.push(line);
|
|
202
|
-
if (this.lines.length > this.maxLines)
|
|
203
|
-
this.lines.splice(0, this.lines.length - this.maxLines);
|
|
204
|
-
}
|
|
205
|
-
snapshot() {
|
|
206
|
-
return [...this.lines];
|
|
207
|
-
}
|
|
208
|
-
clear() {
|
|
209
|
-
this.lines.length = 0;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
/** Bug 4: number of roles for this org that are actually spawned and running
|
|
213
|
-
* right now — the live count run_config.max_concurrent_agents caps. A role
|
|
214
|
-
* that crashed or ended no longer counts, so a slot frees up automatically
|
|
215
|
-
* the moment that happens; nothing needs to explicitly decrement a counter. */
|
|
216
|
-
export function activeRoleCount(org) {
|
|
217
|
-
let n = 0;
|
|
218
|
-
for (const rt of org.agents.values())
|
|
219
|
-
if (rt.status === 'running')
|
|
220
|
-
n++;
|
|
221
|
-
return n;
|
|
222
|
-
}
|
|
223
|
-
/** #302: the `org_complete` consent gate's emit-and-return logic, extracted
|
|
224
|
-
* (matching `resolvedIdleNudgeCount`'s and `runOutcomeResult`'s precedent)
|
|
225
|
-
* so it is unit-testable without a live daemon, a running org, or the SDK's
|
|
226
|
-
* own tool-calling loop — none of which a test can drive directly, since
|
|
227
|
-
* `queryFn` replaces the whole SDK `query()`, tool orchestration included.
|
|
228
|
-
* `onComplete` gathers the facts (needs `this`/`running`) and calls this;
|
|
229
|
-
* this does the one decision (`checkCompletion`) and its two possible
|
|
230
|
-
* side effects. Returns the refusal string, or `null` to allow — exactly
|
|
231
|
-
* what the org_complete tool handler (session.ts) relays as the result. */
|
|
232
|
-
export function resolveOrgComplete(bus, role, outcome, summary, blocker, blockerDetail, runFacts) {
|
|
233
|
-
const refusal = checkCompletion({ outcome, blocker, blockerDetail, ...runFacts });
|
|
234
|
-
if (refusal) {
|
|
235
|
-
// Visible in `org logs` — a silent refusal reads to an operator as a
|
|
236
|
-
// hung boss, not a boss that was told no.
|
|
237
|
-
bus.emit({
|
|
238
|
-
type: 'audit',
|
|
239
|
-
from: role,
|
|
240
|
-
reason: 'org-complete-refused',
|
|
241
|
-
msg: refusal,
|
|
242
|
-
data: { outcome, blocker, blockerDetail },
|
|
243
|
-
});
|
|
244
|
-
return refusal;
|
|
245
|
-
}
|
|
246
|
-
// #302 AC6: the blocker must show up in RENDERED output, not just `data`
|
|
247
|
-
// — `org logs`'s formatter prints an event's `msg` verbatim and never
|
|
248
|
-
// looks at `data`, so a blocker recorded only there would satisfy a unit
|
|
249
|
-
// assertion and never reach a human reading the actual log.
|
|
250
|
-
const blockerSuffix = blocker
|
|
251
|
-
? ` (blocker: ${blocker}${blockerDetail ? ` — ${blockerDetail}` : ''})`
|
|
252
|
-
: '';
|
|
253
|
-
bus.emit({
|
|
254
|
-
type: 'status',
|
|
255
|
-
from: role,
|
|
256
|
-
reason: 'org-complete',
|
|
257
|
-
msg: `run outcome: ${outcome}${blockerSuffix}`,
|
|
258
|
-
data: { outcome, summary, blocker, blockerDetail },
|
|
259
|
-
});
|
|
260
|
-
return null;
|
|
261
|
-
}
|
|
262
56
|
export class OrgDaemon {
|
|
263
57
|
root;
|
|
264
58
|
opts;
|
|
@@ -290,6 +84,7 @@ export class OrgDaemon {
|
|
|
290
84
|
/** @internal */ restarting = new Set();
|
|
291
85
|
// #3: recognizes provider context-window-overflow errors so the boss can be told
|
|
292
86
|
// to chunk the work instead of re-dispatching the same oversized task verbatim.
|
|
87
|
+
/** @internal */
|
|
293
88
|
static CONTEXT_LIMIT_RE = /context[- ]?(window|length|size|limit)|maximum context|exceeds?.{0,12}(context|token)|too many tokens|prompt is too long/i;
|
|
294
89
|
/** @internal */ recallUsage = new Map();
|
|
295
90
|
/** @internal */ orgLearnedRuns = new Set();
|
|
@@ -328,104 +123,7 @@ export class OrgDaemon {
|
|
|
328
123
|
* Removed roles are NOT killed — they finish their current work and won't be re-spawned.
|
|
329
124
|
* Returns a summary of what changed. */
|
|
330
125
|
reloadOrgDef(name) {
|
|
331
|
-
|
|
332
|
-
if (!running)
|
|
333
|
-
throw new Error(`org ${name} is not running`);
|
|
334
|
-
const defPath = join(this.root, ORG_DIR, `${name}.json`);
|
|
335
|
-
const parsedDef = OrgDefSchema.parse(JSON.parse(readFileSync(defPath, 'utf8')));
|
|
336
|
-
const bp = resolveOrgDefBlueprints(parsedDef, this.root);
|
|
337
|
-
if (bp.errors.length)
|
|
338
|
-
throw new Error(`org ${name}: ${bp.errors.join('; ')}`);
|
|
339
|
-
const newDef = expandOrgPolicyPathVars(bp.def, promptVarsFor(this.root));
|
|
340
|
-
const changed = [];
|
|
341
|
-
const newRoles = [];
|
|
342
|
-
const removedRoles = [];
|
|
343
|
-
const onDefTokenCaps = rolesOnDefTokenCaps(running);
|
|
344
|
-
if (newDef.goal !== running.def.goal) {
|
|
345
|
-
running.def.goal = newDef.goal;
|
|
346
|
-
changed.push('goal');
|
|
347
|
-
}
|
|
348
|
-
const oldRc = running.def.run_config;
|
|
349
|
-
const newRc = newDef.run_config;
|
|
350
|
-
for (const key of new Set([...Object.keys(oldRc), ...Object.keys(newRc)])) {
|
|
351
|
-
if (JSON.stringify(oldRc[key]) !== JSON.stringify(newRc[key])) {
|
|
352
|
-
oldRc[key] = newRc[key];
|
|
353
|
-
changed.push(`run_config.${key}`);
|
|
354
|
-
}
|
|
355
|
-
}
|
|
356
|
-
// M1 (C-37): apply changes to EXISTING roles' tool_providers, endpoint,
|
|
357
|
-
// kind and policy. Fields are replaced on the live role object (sessions
|
|
358
|
-
// read tool_providers at their next start, checkApproval reads policy
|
|
359
|
-
// live) and a running role's PolicyEngine gets the new policy now.
|
|
360
|
-
// #343: budget_usd / budget_tokens too — the live PolicyEngine gets the
|
|
361
|
-
// new caps with its spend kept, and a role closed for budget reopens
|
|
362
|
-
// below once it is no longer over them.
|
|
363
|
-
const RELOADABLE_ROLE_FIELDS = [
|
|
364
|
-
'tool_providers',
|
|
365
|
-
'endpoint',
|
|
366
|
-
'kind',
|
|
367
|
-
'policy',
|
|
368
|
-
'budget_usd',
|
|
369
|
-
'budget_tokens',
|
|
370
|
-
];
|
|
371
|
-
for (const next of newDef.roles) {
|
|
372
|
-
const live = running.def.roles.find((r) => r.id === next.id);
|
|
373
|
-
if (!live)
|
|
374
|
-
continue;
|
|
375
|
-
const liveRec = live;
|
|
376
|
-
const nextRec = next;
|
|
377
|
-
for (const field of RELOADABLE_ROLE_FIELDS) {
|
|
378
|
-
if (JSON.stringify(liveRec[field]) === JSON.stringify(nextRec[field]))
|
|
379
|
-
continue;
|
|
380
|
-
const targets = new Set([liveRec]);
|
|
381
|
-
const slotRole = running.roleSlots.get(next.id)?.effectiveRole;
|
|
382
|
-
if (slotRole)
|
|
383
|
-
targets.add(slotRole);
|
|
384
|
-
const pending = running.pendingRoles?.get(next.id);
|
|
385
|
-
if (pending)
|
|
386
|
-
targets.add(pending);
|
|
387
|
-
for (const t of targets) {
|
|
388
|
-
if (nextRec[field] === undefined)
|
|
389
|
-
delete t[field];
|
|
390
|
-
else
|
|
391
|
-
t[field] = nextRec[field];
|
|
392
|
-
}
|
|
393
|
-
if (field === 'policy')
|
|
394
|
-
running.agents.get(next.id)?.policy.updatePolicy(next.policy ?? {});
|
|
395
|
-
if (field === 'budget_usd' || field === 'budget_tokens')
|
|
396
|
-
running.agents.get(next.id)?.policy.setBudgetCaps({
|
|
397
|
-
maxTokens: live.policy?.maxTokens ?? computeReplacementBudget(running.def, next.id),
|
|
398
|
-
maxUsd: live.policy?.maxUsd ?? live.budget_usd,
|
|
399
|
-
});
|
|
400
|
-
changed.push(`role:${next.id}:${field}`);
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
const reopened = reopenBudgetClosedRoles(this, name, running, onDefTokenCaps);
|
|
404
|
-
const existingRoleIds = new Set(running.def.roles.map((r) => r.id));
|
|
405
|
-
const newRoleIds = new Set(newDef.roles.map((r) => r.id));
|
|
406
|
-
for (const role of newDef.roles) {
|
|
407
|
-
if (!existingRoleIds.has(role.id)) {
|
|
408
|
-
running.def.roles.push(role);
|
|
409
|
-
// M2: an endpoint role never gets a session — nothing to lazy-spawn.
|
|
410
|
-
if (!isEndpointRole(role)) {
|
|
411
|
-
if (!running.pendingRoles)
|
|
412
|
-
running.pendingRoles = new Map();
|
|
413
|
-
running.pendingRoles.set(role.id, role);
|
|
414
|
-
}
|
|
415
|
-
newRoles.push(role.id);
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
for (const id of existingRoleIds) {
|
|
419
|
-
if (!newRoleIds.has(id))
|
|
420
|
-
removedRoles.push(id);
|
|
421
|
-
}
|
|
422
|
-
running.bus.emit({
|
|
423
|
-
type: 'audit',
|
|
424
|
-
reason: 'hot-reload',
|
|
425
|
-
msg: `org def reloaded: ${changed.length} fields changed, ${newRoles.length} new roles, ${removedRoles.length} removed roles${reopened.length ? `, reopened ${reopened.join(', ')}` : ''}`,
|
|
426
|
-
data: { changed, newRoles, removedRoles },
|
|
427
|
-
});
|
|
428
|
-
return { changed, newRoles, removedRoles };
|
|
126
|
+
return orgReload.reloadOrgDef(this, name);
|
|
429
127
|
}
|
|
430
128
|
/** Names of the orgs this daemon currently has running. Snapshot — safe to
|
|
431
129
|
* iterate while stopOrg() mutates the underlying map. */
|
|
@@ -470,6 +168,7 @@ export class OrgDaemon {
|
|
|
470
168
|
* A relative path is resolved against the project root rather than the
|
|
471
169
|
* daemon's cwd, which is not the same directory when `org serve` is started
|
|
472
170
|
* from a subdirectory. */
|
|
171
|
+
/** @internal */
|
|
473
172
|
workspaceSetting(def) {
|
|
474
173
|
const ws = def.run_config.workspace ?? 'repo';
|
|
475
174
|
if (ws === 'repo' || ws === 'isolated' || ws === 'worktree' || ws === 'worktree-per-role')
|
|
@@ -477,921 +176,7 @@ export class OrgDaemon {
|
|
|
477
176
|
return isAbsolute(ws) ? ws : join(this.root, ws);
|
|
478
177
|
}
|
|
479
178
|
async startOrg(name, taskOverride, options) {
|
|
480
|
-
|
|
481
|
-
// cap holds; any other (explicit) start resets it so a manual re-run gets a
|
|
482
|
-
// fresh budget.
|
|
483
|
-
if (!this.restarting.has(name))
|
|
484
|
-
this.bossRestartCounts.delete(name);
|
|
485
|
-
// Join any in-flight stop for this org before checking `this.orgs` — otherwise a
|
|
486
|
-
// start racing a stop's drain window (up to stopWaitMs) can share the stopping
|
|
487
|
-
// run's worktree path while it's still being force-removed.
|
|
488
|
-
const inflightStop = this.stopping.get(name);
|
|
489
|
-
if (inflightStop)
|
|
490
|
-
await inflightStop;
|
|
491
|
-
// Bug 2 (TOCTOU race): this existence check is synchronous, but the real
|
|
492
|
-
// registration into `this.orgs` doesn't happen until deep inside
|
|
493
|
-
// startOrgInner, after several genuine `await` points (the provider
|
|
494
|
-
// validation dynamic import, `git worktree add` for workspace:
|
|
495
|
-
// 'worktree'). Two concurrent startOrg(name) calls — e.g. the
|
|
496
|
-
// scheduler's tick, the runfile poll loop, and autoWake firing close
|
|
497
|
-
// together — could each pass this check before either registered,
|
|
498
|
-
// spawning two duplicate runs with separate budget/policy counters that
|
|
499
|
-
// both write the same shared per-org files. Reserve the name in
|
|
500
|
-
// `startingOrgs` synchronously, in the same tick as the check, so a
|
|
501
|
-
// second concurrent call sees the reservation and is rejected instead of
|
|
502
|
-
// racing ahead to spawn a duplicate.
|
|
503
|
-
if (this.orgs.has(name))
|
|
504
|
-
throw new Error(`org ${name} already running`);
|
|
505
|
-
if (this.startingOrgs.has(name))
|
|
506
|
-
throw new Error(`org ${name} already starting`);
|
|
507
|
-
this.startingOrgs.add(name);
|
|
508
|
-
// #345: the run's --auto-approve list — kept by a boss auto-restart like
|
|
509
|
-
// the crash counter, replaced by any other start. Set only after the
|
|
510
|
-
// guards above, so a rejected start can't rewrite a live run's list.
|
|
511
|
-
if (!this.restarting.has(name)) {
|
|
512
|
-
if (options?.autoApprove?.length)
|
|
513
|
-
this.runAutoApprove.set(name, options.autoApprove);
|
|
514
|
-
else
|
|
515
|
-
this.runAutoApprove.delete(name);
|
|
516
|
-
}
|
|
517
|
-
try {
|
|
518
|
-
return await this.startOrgInner(name, taskOverride, options);
|
|
519
|
-
}
|
|
520
|
-
catch (err) {
|
|
521
|
-
// startOrgInner registers the org in `this.orgs` (and spawns the boss,
|
|
522
|
-
// installs the exit listener, starts the broker lease) well before it
|
|
523
|
-
// returns; persistState (ENOSPC/EACCES) and BrokerLease.start() can
|
|
524
|
-
// still throw after that. Left alone, that was a live, unreachable org:
|
|
525
|
-
// sessions running, `this.orgs` still holding it, every later startOrg
|
|
526
|
-
// rejected with "already running", and nothing ever calling stopOrg.
|
|
527
|
-
// Only this call can have registered the name (the reservation above
|
|
528
|
-
// holds until `finally`), so anything in the map is ours to tear down.
|
|
529
|
-
if (this.orgs.has(name)) {
|
|
530
|
-
// #302: tag the real cause so a run's history/report can never read
|
|
531
|
-
// this as a boss-attributed outcome — nothing here asked the boss.
|
|
532
|
-
await this.stopOrg(name, { closedBy: 'failed-start' }).catch((stopErr) => console.error(`org ${name}: teardown after failed start failed:`, stopErr instanceof Error ? stopErr.message : stopErr));
|
|
533
|
-
}
|
|
534
|
-
throw err;
|
|
535
|
-
}
|
|
536
|
-
finally {
|
|
537
|
-
this.startingOrgs.delete(name);
|
|
538
|
-
}
|
|
539
|
-
}
|
|
540
|
-
/** The actual startOrg implementation. Split out of startOrg() so the
|
|
541
|
-
* reservation guard above runs synchronously, before any `await` in here —
|
|
542
|
-
* see the bug 2 comment in startOrg(). */
|
|
543
|
-
async startOrgInner(name, taskOverride, options) {
|
|
544
|
-
// #301: the PRIMARY fix — the stop-side backstops below (finishStop,
|
|
545
|
-
// process 'exit') can only run for a run that ends through code we
|
|
546
|
-
// control; a SIGKILL leaves .git/worktrees/<name> metadata behind with
|
|
547
|
-
// nothing left to clean it up, and so did every run that leaked before
|
|
548
|
-
// this fix existed. Pruning here turns "did the last run clean up,
|
|
549
|
-
// however it died?" into "a run always begins clean" — the only thing
|
|
550
|
-
// that recovers both the SIGKILL case and worktrees already orphaned
|
|
551
|
-
// before this daemon process started. Unconditional and best-effort for
|
|
552
|
-
// the same reasons as the stop-side prune (see finishStop): it only
|
|
553
|
-
// drops metadata whose worktree directory is already gone, so it cannot
|
|
554
|
-
// touch a live worktree, including the owner's.
|
|
555
|
-
try {
|
|
556
|
-
execFileSync('git', ['worktree', 'prune'], {
|
|
557
|
-
cwd: this.root,
|
|
558
|
-
stdio: 'ignore',
|
|
559
|
-
timeout: 30_000,
|
|
560
|
-
});
|
|
561
|
-
}
|
|
562
|
-
catch {
|
|
563
|
-
/* best-effort: not a git repo, git missing, or a wedged hook */
|
|
564
|
-
}
|
|
565
|
-
const defPath = join(this.root, ORG_DIR, `${name}.json`);
|
|
566
|
-
const parsedDef = OrgDefSchema.parse(JSON.parse(readFileSync(defPath, 'utf8')));
|
|
567
|
-
const bp = resolveOrgDefBlueprints(parsedDef, this.root);
|
|
568
|
-
// {{home}} / {{org_root}} in policy paths, before any root or sandbox sees them.
|
|
569
|
-
const def = expandOrgPolicyPathVars(bp.def, promptVarsFor(this.root));
|
|
570
|
-
const autoApproveError = approvalOps.unknownAutoApproveError(options?.autoApprove ?? [], def.roles);
|
|
571
|
-
if (autoApproveError)
|
|
572
|
-
throw new Error(autoApproveError);
|
|
573
|
-
let run;
|
|
574
|
-
let checkpoint;
|
|
575
|
-
if (options?.resume) {
|
|
576
|
-
const rtPath = join(this.root, ORG_DIR, name, 'runtime.json');
|
|
577
|
-
if (!existsSync(rtPath))
|
|
578
|
-
throw new Error(`cannot resume org "${name}": runtime.json not found`);
|
|
579
|
-
const rt = JSON.parse(readFileSync(rtPath, 'utf8'));
|
|
580
|
-
if (!rt?.run || !rt?.checkpoint)
|
|
581
|
-
throw new Error(`cannot resume org "${name}": no valid checkpoint found`);
|
|
582
|
-
if (isCheckpointExpired(rt.checkpoint))
|
|
583
|
-
throw new Error(`cannot resume org "${name}": checkpoint expired`);
|
|
584
|
-
// Migrate an older-schema checkpoint (verifying ITS OWN stored checksum
|
|
585
|
-
// first) before validating it against CHECKPOINT_VERSION — see
|
|
586
|
-
// migrateCheckpoint's doc comment in checkpoint.ts.
|
|
587
|
-
const migrated = migrateCheckpoint(rt.checkpoint);
|
|
588
|
-
if (!migrated || !validateCheckpoint(migrated))
|
|
589
|
-
throw new Error(`cannot resume org "${name}": checkpoint validation failed`);
|
|
590
|
-
rt.checkpoint = migrated;
|
|
591
|
-
run = rt.run;
|
|
592
|
-
checkpoint = rt.checkpoint;
|
|
593
|
-
if (rt.abandonedRoles) {
|
|
594
|
-
this.abandoned.set(name, new Set(rt.abandonedRoles));
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
else {
|
|
598
|
-
this.abandoned.delete(name); // a previous run's missing roles say nothing about this one
|
|
599
|
-
this.memoryErrors.delete(name); // nor does its memory-store failure (#293)
|
|
600
|
-
approvalOps.clearApprovalsForFreshStart(this, name); // a previous run's approvals are moot for this one
|
|
601
|
-
questionOps.clearQuestionsForFreshStart(this, name); // nor do its unanswered questions (#248)
|
|
602
|
-
// random suffix: second-precision stamps collide across processes (two CLI
|
|
603
|
-
// invocations in the same second would share a run dir and its bus.jsonl)
|
|
604
|
-
run = `run-${new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14)}-${Math.random().toString(36).slice(2, 6)}`;
|
|
605
|
-
}
|
|
606
|
-
const dir = join(this.root, ORG_DIR, name, run);
|
|
607
|
-
mkdirSync(dir, { recursive: true });
|
|
608
|
-
// Role sessions run at the project root by default. They used to run in an
|
|
609
|
-
// empty scratch dir under .monomind/orgs/<name>/workspace, which the policy
|
|
610
|
-
// engine's workdir check ("path escapes org workdir") then confined every
|
|
611
|
-
// path to — so a development org could not Read or Edit a single file of
|
|
612
|
-
// the project it was created to work on. Roles fell back to Bash, which is
|
|
613
|
-
// not path-scoped, meaning the sandbox blocked the safe tools and let the
|
|
614
|
-
// unrestricted one through. Opt back in with run_config.workspace:
|
|
615
|
-
// 'isolated', or pin an absolute path.
|
|
616
|
-
const ws = this.workspaceSetting(def);
|
|
617
|
-
let cwd;
|
|
618
|
-
let worktreePath;
|
|
619
|
-
if (ws === 'worktree') {
|
|
620
|
-
worktreePath = join(this.root, ORG_DIR, name, 'worktree');
|
|
621
|
-
const { execFileSync } = await import('node:child_process');
|
|
622
|
-
try {
|
|
623
|
-
// Remove stale worktree from a previous run
|
|
624
|
-
if (existsSync(worktreePath)) {
|
|
625
|
-
// R4: bound the call — a wedged git hook (git-lfs, gc lock, gpg sign
|
|
626
|
-
// prompt) would otherwise hang the whole daemon indefinitely.
|
|
627
|
-
// SEC-5: execFileSync + argv — no shell interpolation of worktreePath.
|
|
628
|
-
execFileSync('git', ['worktree', 'remove', '--force', worktreePath], {
|
|
629
|
-
cwd: this.root,
|
|
630
|
-
stdio: 'ignore',
|
|
631
|
-
timeout: 30_000,
|
|
632
|
-
});
|
|
633
|
-
}
|
|
634
|
-
}
|
|
635
|
-
catch {
|
|
636
|
-
/* best-effort cleanup */
|
|
637
|
-
}
|
|
638
|
-
execFileSync('git', ['worktree', 'add', worktreePath, 'HEAD', '--detach'], {
|
|
639
|
-
cwd: this.root,
|
|
640
|
-
stdio: 'ignore',
|
|
641
|
-
timeout: 30_000,
|
|
642
|
-
});
|
|
643
|
-
cwd = worktreePath;
|
|
644
|
-
}
|
|
645
|
-
else {
|
|
646
|
-
cwd =
|
|
647
|
-
ws === 'repo'
|
|
648
|
-
? this.root
|
|
649
|
-
: ws === 'isolated'
|
|
650
|
-
? join(this.root, ORG_DIR, name, 'workspace')
|
|
651
|
-
: ws;
|
|
652
|
-
}
|
|
653
|
-
mkdirSync(cwd, { recursive: true });
|
|
654
|
-
// An org must be able to seat its whole roster. maxSdkProcesses is sized for
|
|
655
|
-
// the machine (cpus - 2), so any org with more roles than that had its tail
|
|
656
|
-
// roles deferred forever — a 7-role org on an 8-core box permanently lost
|
|
657
|
-
// its 7th, and the work that role owned simply never happened. Raise the
|
|
658
|
-
// ceiling to the role count. An explicit MONOMIND_MAX_SDK_PROCS still wins:
|
|
659
|
-
// if the operator named a number, that number is the answer.
|
|
660
|
-
//
|
|
661
|
-
// Runner process model (relevant for sizing): ClaudeAgentRunner and
|
|
662
|
-
// VercelAgentRunner are in-process (no subprocess per role). KimiCodeAgentRunner,
|
|
663
|
-
// OpencodeAgentRunner, and CodexAgentRunner each spawn one subprocess per role.
|
|
664
|
-
// The current sizing (def.roles.length) is therefore safe — it over-provisions
|
|
665
|
-
// for in-process runners but never under-provisions for subprocess runners.
|
|
666
|
-
const sessionRoleCount = agentRoles(def.roles).length;
|
|
667
|
-
if (!process.env.MONOMIND_MAX_SDK_PROCS &&
|
|
668
|
-
getResourceLimits().maxSdkProcesses < sessionRoleCount) {
|
|
669
|
-
configureResourceLimits({ maxSdkProcesses: sessionRoleCount });
|
|
670
|
-
}
|
|
671
|
-
// ADR-O001 D8: a cost tier that can't resolve a model for a role's
|
|
672
|
-
// provider must stop the run here. The alternative — resolving it at
|
|
673
|
-
// session start — would either crash one role ten minutes in or, worse,
|
|
674
|
-
// quietly leave that role on a different model than the tier claimed.
|
|
675
|
-
const { validateCostTiers } = await import('./cost-tier.js');
|
|
676
|
-
const tierErrors = validateCostTiers(def);
|
|
677
|
-
if (tierErrors.length) {
|
|
678
|
-
throw new Error(`org ${name}: ${tierErrors.join('; ')}`);
|
|
679
|
-
}
|
|
680
|
-
// ADR-O001 D7: an oversized or unresolvable loadout catalog stops the run
|
|
681
|
-
// here, for the same reason — not ten minutes in, at some role's spawn.
|
|
682
|
-
const { validateLoadouts } = await import('./loadouts.js');
|
|
683
|
-
const loadoutErrors = validateLoadouts(def, this.root).errors;
|
|
684
|
-
if (loadoutErrors.length) {
|
|
685
|
-
throw new Error(`org ${name}: ${loadoutErrors.join('; ')}`);
|
|
686
|
-
}
|
|
687
|
-
const { validateRoleSkills } = await import('./skill-library.js');
|
|
688
|
-
const skillErrors = [
|
|
689
|
-
...bp.errors,
|
|
690
|
-
...def.roles.flatMap((r) => validateRoleSkills(r, this.root)),
|
|
691
|
-
];
|
|
692
|
-
if (skillErrors.length) {
|
|
693
|
-
throw new Error(`org ${name}: ${skillErrors.join('; ')}`);
|
|
694
|
-
}
|
|
695
|
-
// Validate per-role providers before spawning anything (fail-fast: a
|
|
696
|
-
// missing env var discovered 10 minutes into a run wastes the entire run).
|
|
697
|
-
const { resolveProviderEnv: validateProvider, resolveRoleProvider } = await import('./provider.js');
|
|
698
|
-
for (const role of def.roles) {
|
|
699
|
-
try {
|
|
700
|
-
if (role.provider) {
|
|
701
|
-
validateProvider(role.provider);
|
|
702
|
-
}
|
|
703
|
-
else if (role.adapter_config?.provider) {
|
|
704
|
-
// Named provider (`monomind providers configure`): resolve now so a
|
|
705
|
-
// missing/misconfigured entry fails the run at start, not mid-flight.
|
|
706
|
-
resolveRoleProvider(role, this.root);
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
catch (err) {
|
|
710
|
-
throw new Error(`org ${name}: role "${role.id}" provider validation failed — ${err instanceof Error ? err.message : err}`);
|
|
711
|
-
}
|
|
712
|
-
// provider.kind 'gemini'/'openai' only sets env vars (GEMINI_API_KEY /
|
|
713
|
-
// OPENAI_API_KEY — see provider.ts) for a CLI that never reads them:
|
|
714
|
-
// autoRuntimeFromProvider has no case for either kind, so
|
|
715
|
-
// resolveRoleRunner falls through to `undefined` and session.ts spawns
|
|
716
|
-
// the default ClaudeAgentRunner. The role silently runs on Claude while
|
|
717
|
-
// its config claims gemini/openai — surface that loudly at start time
|
|
718
|
-
// instead of leaving it to be discovered mid-run.
|
|
719
|
-
const kind = role.provider?.kind;
|
|
720
|
-
if ((kind === 'gemini' || kind === 'openai') &&
|
|
721
|
-
!resolveRoleRunner(role.runtime, def.runtime, kind, undefined, role.provider)) {
|
|
722
|
-
console.error(`org ${name}: role "${role.id}" sets provider.kind="${kind}" but no runtime honors it — ` +
|
|
723
|
-
`this role will actually run on the Claude Agent SDK, not ${kind}. ` +
|
|
724
|
-
`Set role.runtime (or the org's runtime) explicitly, or use provider.kind="vercel-api-key" ` +
|
|
725
|
-
`with vendor="${kind === 'gemini' ? 'google' : 'openai'}" to route through a real ${kind} model.`);
|
|
726
|
-
}
|
|
727
|
-
}
|
|
728
|
-
const bus = new OrgBus(name, run, dir);
|
|
729
|
-
// Lightweight in-memory tail for busEvents() (test-loop, /api/history).
|
|
730
|
-
// Full events (including Write content snapshots) live on disk in bus.jsonl;
|
|
731
|
-
// the in-memory copy strips bulky data.content to keep RAM flat.
|
|
732
|
-
const MAX_COLLECTED = 1000;
|
|
733
|
-
const collected = [];
|
|
734
|
-
let lastActivity = Date.now();
|
|
735
|
-
// Separate from lastActivity: only real tool calls (org_send, org_task,
|
|
736
|
-
// Bash, ...) count here, not status pings or chat-only turns. The idle
|
|
737
|
-
// watchdog's nudge-recovery check (resolvedIdleNudgeCount) uses this to
|
|
738
|
-
// tell genuine forward progress apart from a boss that "answers" a nudge
|
|
739
|
-
// with a bare acknowledgment ("✓ Complete") and does nothing — a
|
|
740
|
-
// content-free reply still updates lastActivity (so the org isn't
|
|
741
|
-
// flagged as silent), but must not reset the cumulative nudge cap, or a
|
|
742
|
-
// boss that's genuinely out of ideas can loop forever making zero
|
|
743
|
-
// progress without ever tripping the watchdog.
|
|
744
|
-
let lastToolActivity = 0;
|
|
745
|
-
// ADR-O001 D4 (no-progress detector): the idle clock above is org-wide and
|
|
746
|
-
// says nothing at all while a hold is in force — the 8.3-hour stall looked
|
|
747
|
-
// perfectly healthy from it. Per-role last-activity is what tells a role
|
|
748
|
-
// that is nominally working but producing nothing from one that is simply
|
|
749
|
-
// waiting its turn.
|
|
750
|
-
const roleActivity = new Map();
|
|
751
|
-
const noProgressAlarmed = new Set();
|
|
752
|
-
bus.subscribe((e) => {
|
|
753
|
-
const slim = e.data?.content != null ? { ...e, data: { ...e.data, content: undefined } } : e;
|
|
754
|
-
collected.push(slim);
|
|
755
|
-
if (collected.length > MAX_COLLECTED)
|
|
756
|
-
collected.splice(0, collected.length - MAX_COLLECTED);
|
|
757
|
-
// The watchdog's own events must not count as org activity, or a hung
|
|
758
|
-
// boss would never trip the "nudge produced no activity" stop and a
|
|
759
|
-
// silent role would clear its own no-progress alarm.
|
|
760
|
-
const selfEmitted = e.reason === 'idle-nudge' || e.reason === 'no-progress' || e.reason === 'hold-expired';
|
|
761
|
-
if (!selfEmitted)
|
|
762
|
-
lastActivity = Date.now();
|
|
763
|
-
if (e.type === 'tool')
|
|
764
|
-
lastToolActivity = Date.now();
|
|
765
|
-
if (e.from && !selfEmitted) {
|
|
766
|
-
roleActivity.set(e.from, Date.now());
|
|
767
|
-
noProgressAlarmed.delete(e.from);
|
|
768
|
-
}
|
|
769
|
-
// org_complete IS the end of the run — self-stop instead of sitting
|
|
770
|
-
// "running" forever after a recorded outcome. Deferred (unref'd) so the
|
|
771
|
-
// tool call's receipt reaches the boss and its final turn text still
|
|
772
|
-
// lands on the bus before mailboxes close; stopOrg is reentrant-safe
|
|
773
|
-
// against a concurrent manual stop.
|
|
774
|
-
if (e.type === 'status' && e.reason === 'org-complete') {
|
|
775
|
-
const t = setTimeout(() => {
|
|
776
|
-
// #206: closedBy: 'org-complete' is the ONLY signal `org run`
|
|
777
|
-
// trusts to mean "the run ended cleanly, exit 0" — every other
|
|
778
|
-
// stop path (idle watchdog, boss-restart-exhausted, manual stop)
|
|
779
|
-
// leaves it unset.
|
|
780
|
-
this.stopOrg(name, { drainMs: COMPLETE_DRAIN_MS, closedBy: 'org-complete' }).catch((err) => console.error(`org ${name}: auto-stop after org_complete failed:`, err instanceof Error ? err.message : err));
|
|
781
|
-
}, 1000);
|
|
782
|
-
t.unref?.();
|
|
783
|
-
}
|
|
784
|
-
// Accumulate cost from usage events into per-role metrics
|
|
785
|
-
if (e.type === 'usage' && e.from && e.data) {
|
|
786
|
-
const runtime = running.agents.get(e.from);
|
|
787
|
-
if (runtime) {
|
|
788
|
-
const cost = Number(e.data.cost_usd ?? 0);
|
|
789
|
-
if (Number.isFinite(cost)) {
|
|
790
|
-
runtime.metrics.costUsd += cost;
|
|
791
|
-
}
|
|
792
|
-
}
|
|
793
|
-
}
|
|
794
|
-
// Bug 1 / #343: enforce the org-wide run_config.budget_tokens ceiling,
|
|
795
|
-
// hold a budget-closed role's tasks, warn near a budget.
|
|
796
|
-
onBudgetBusEvent(running, e);
|
|
797
|
-
// Track last message ID for threading responses
|
|
798
|
-
if ((e.type === 'message' || e.type === 'xorg') && e.from) {
|
|
799
|
-
const runtime = running.agents.get(e.from);
|
|
800
|
-
if (runtime) {
|
|
801
|
-
runtime.lastMessageId = e.id;
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
// Apply audit filter if configured (skip filtered tool events before forwarding)
|
|
805
|
-
if (this.opts.auditFilter && e.type === 'tool') {
|
|
806
|
-
const { tool, decision } = this.opts.auditFilter;
|
|
807
|
-
if (tool && e.tool !== tool)
|
|
808
|
-
return; // Skip: tool name doesn't match
|
|
809
|
-
if (decision && e.decision !== decision)
|
|
810
|
-
return; // Skip: decision doesn't match
|
|
811
|
-
}
|
|
812
|
-
for (const fn of this.globalSubscribers)
|
|
813
|
-
fn(e);
|
|
814
|
-
});
|
|
815
|
-
if (this.opts.forward !== false)
|
|
816
|
-
this.forwarders.set(name, attachForwarder(bus, this.opts.controlJson ?? join(this.root, '.monomind/control.json')));
|
|
817
|
-
const running = {
|
|
818
|
-
def,
|
|
819
|
-
run,
|
|
820
|
-
sessionLedger: new SessionLedger(join(dir, 'sessions.json')),
|
|
821
|
-
gates: decisionOps.readGates(this.root, name),
|
|
822
|
-
bus,
|
|
823
|
-
agents: new Map(),
|
|
824
|
-
roleSlots: new Map(),
|
|
825
|
-
bossRoleId: '', // set below, once bossRole is computed
|
|
826
|
-
glossary: [],
|
|
827
|
-
respawning: new Set(),
|
|
828
|
-
busEvents: () => [...collected],
|
|
829
|
-
workdir: cwd,
|
|
830
|
-
credential: randomUUID(),
|
|
831
|
-
};
|
|
832
|
-
this.orgs.set(name, running);
|
|
833
|
-
// ── MonoFence guardrail: pre-create per-role instances ────────────────
|
|
834
|
-
const globalFence = loadGlobalFenceConfig(this.root);
|
|
835
|
-
const orgFence = def.fence;
|
|
836
|
-
const roleFences = new Map();
|
|
837
|
-
for (const role of def.roles) {
|
|
838
|
-
const roleFenceCfg = role.policy?.fence;
|
|
839
|
-
const merged = mergeFenceConfigs(globalFence ?? undefined, orgFence, roleFenceCfg);
|
|
840
|
-
if (merged.enabled === false)
|
|
841
|
-
continue;
|
|
842
|
-
if (!globalFence && !orgFence && !roleFenceCfg)
|
|
843
|
-
continue;
|
|
844
|
-
try {
|
|
845
|
-
const instance = await createFenceForRole(merged);
|
|
846
|
-
if (instance) {
|
|
847
|
-
roleFences.set(role.id, {
|
|
848
|
-
instance,
|
|
849
|
-
abortThreshold: typeof merged.abortThreshold === 'number' ? merged.abortThreshold : 0.8,
|
|
850
|
-
scanMessages: merged.scanMessages !== false,
|
|
851
|
-
});
|
|
852
|
-
}
|
|
853
|
-
}
|
|
854
|
-
catch {
|
|
855
|
-
/* monofence-ai not installed — skip silently */
|
|
856
|
-
}
|
|
857
|
-
}
|
|
858
|
-
if (roleFences.size > 0)
|
|
859
|
-
running.fences = roleFences;
|
|
860
|
-
// Even-split budget; a role's own budget_tokens overrides it (roleTokenBudget).
|
|
861
|
-
// Bug 1: roles WITH an explicit override spend on top of the even split
|
|
862
|
-
// rather than out of it, so the roster's ceilings could sum to well over
|
|
863
|
-
// the declared org-wide budget (e.g. 4 roles @ 250k + one role overridden
|
|
864
|
-
// to 2M = 2.75M achievable against a declared 1M cap). Subtract the sum of
|
|
865
|
-
// every role's explicit override from the org-wide budget first, then
|
|
866
|
-
// split only the remainder among the roles WITHOUT an override, so the
|
|
867
|
-
// static split is honest about what's left. (Live usage is still tracked
|
|
868
|
-
// and enforced as a real ceiling above, independent of this static split.)
|
|
869
|
-
const orgBudgetTokens = def.run_config.budget_tokens ?? 1_000_000;
|
|
870
|
-
const overriddenTokenSum = def.roles.reduce((sum, r) => sum + (r.budget_tokens ?? 0), 0);
|
|
871
|
-
const unoverriddenRoleCount = agentRoles(def.roles).filter((r) => r.budget_tokens == null).length;
|
|
872
|
-
const _perRoleBudget = unoverriddenRoleCount > 0
|
|
873
|
-
? Math.max(0, Math.floor((orgBudgetTokens - overriddenTokenSum) / unoverriddenRoleCount))
|
|
874
|
-
: 0;
|
|
875
|
-
// Single boss-selection rule for kickoff AND org_complete gating — the
|
|
876
|
-
// session layer previously keyed the tool on reports_to===null while the
|
|
877
|
-
// kickoff went to (type==='boss' || reports_to===null || roles[0]), so a
|
|
878
|
-
// fallback-selected boss could be told to call org_complete without having
|
|
879
|
-
// the tool.
|
|
880
|
-
// M2: endpoint roles are never the boss.
|
|
881
|
-
const sessionRoles = agentRoles(def.roles);
|
|
882
|
-
if (sessionRoles.length === 0)
|
|
883
|
-
throw new Error(`org ${name}: no agent roles — endpoint roles cannot run an org`);
|
|
884
|
-
const bossRole = sessionRoles.find((r) => r.type === 'boss' || r.reports_to === null) ?? sessionRoles[0];
|
|
885
|
-
running.bossRoleId = bossRole.id;
|
|
886
|
-
// Canonical entity names from THIS org's KG — injected into the coordinator
|
|
887
|
-
// prompt so org_learn extractions reuse them instead of minting duplicates.
|
|
888
|
-
// Scoped: an unscoped glossary handed every org's entity names to every
|
|
889
|
-
// coordinator, which is how one org's claims got merged into another's.
|
|
890
|
-
const glossary = await (async () => {
|
|
891
|
-
try {
|
|
892
|
-
if (!(await this.orgMemoryUsable()))
|
|
893
|
-
return [];
|
|
894
|
-
const kg = await import('../memory/memory-kg.js');
|
|
895
|
-
return await kg.kgGlossary({
|
|
896
|
-
dbPath: this.orgMemoryDbPath(),
|
|
897
|
-
scope: orgMemory.orgKgScope(name),
|
|
898
|
-
});
|
|
899
|
-
}
|
|
900
|
-
catch {
|
|
901
|
-
return [];
|
|
902
|
-
}
|
|
903
|
-
})();
|
|
904
|
-
running.glossary = glossary;
|
|
905
|
-
// Resource-gated staggered spawn: check memory/process limits before each
|
|
906
|
-
// NON-BOSS agent, wait if under pressure. The boss always spawns immediately
|
|
907
|
-
// and ungated — the org has no coordinator at all without it, so gating it
|
|
908
|
-
// behind host memory pressure would make the whole org fail to start over a
|
|
909
|
-
// condition workers are specifically designed to ride out.
|
|
910
|
-
// Extracted so a role that fails its gate check can be spawned later by
|
|
911
|
-
// scheduleDeferredSpawn() once resources free up, without re-running the
|
|
912
|
-
// gate logic or duplicating the session-wiring below.
|
|
913
|
-
const spawnRole = (role, roleCheckpoint) => {
|
|
914
|
-
if (running.agents.has(role.id))
|
|
915
|
-
return;
|
|
916
|
-
if (isEndpointRole(role))
|
|
917
|
-
return; // M2: no session, mailbox or slot
|
|
918
|
-
const { runtime, abort } = this.spawnRoleIncarnation(name, running, role, roleCheckpoint?.generation ?? 0, { roleCheckpoint });
|
|
919
|
-
running.agents.set(role.id, runtime);
|
|
920
|
-
running.roleSlots.set(role.id, {
|
|
921
|
-
generation: roleCheckpoint?.generation ?? 0,
|
|
922
|
-
phase: 'running',
|
|
923
|
-
runtime,
|
|
924
|
-
abort,
|
|
925
|
-
effectiveRole: roleCheckpoint?.effectiveRoleOverrides &&
|
|
926
|
-
Object.keys(roleCheckpoint.effectiveRoleOverrides).length > 0
|
|
927
|
-
? mergeEffectiveRoleConfig(role, roleCheckpoint.effectiveRoleOverrides)
|
|
928
|
-
: role,
|
|
929
|
-
respawnCount: roleCheckpoint?.respawnCount ?? 0,
|
|
930
|
-
queuedDuringSwap: roleCheckpoint?.queuedDuringSwap ?? [],
|
|
931
|
-
retiredUsage: roleCheckpoint?.retiredUsage ?? { tokens: 0, costUsd: 0 },
|
|
932
|
-
});
|
|
933
|
-
};
|
|
934
|
-
if (options?.resume && checkpoint) {
|
|
935
|
-
const restoredRoles = new Set(Object.keys(checkpoint.roleState));
|
|
936
|
-
for (const [roleId, roleState] of Object.entries(checkpoint.roleState)) {
|
|
937
|
-
const role = def.roles.find((r) => r.id === roleId);
|
|
938
|
-
if (role)
|
|
939
|
-
spawnRole(role, roleState);
|
|
940
|
-
}
|
|
941
|
-
const pendingRoles = new Map();
|
|
942
|
-
for (const role of def.roles) {
|
|
943
|
-
if (!restoredRoles.has(role.id) && !isEndpointRole(role)) {
|
|
944
|
-
pendingRoles.set(role.id, role);
|
|
945
|
-
}
|
|
946
|
-
}
|
|
947
|
-
running.pendingRoles = pendingRoles;
|
|
948
|
-
running.spawnRole = spawnRole;
|
|
949
|
-
running.taskDag =
|
|
950
|
-
checkpoint.tasks && checkpoint.tasks.length > 0
|
|
951
|
-
? TaskDag.fromJSON(checkpoint.tasks)
|
|
952
|
-
: new TaskDag();
|
|
953
|
-
// A 'running' task's "[task:…]" message was consumed by the session that
|
|
954
|
-
// was working it. If that role's SDK session is resumed (checkpointed
|
|
955
|
-
// sessionId) the task is still in its context; otherwise — role not
|
|
956
|
-
// restored at all, or restored into a fresh session — nothing knows
|
|
957
|
-
// about the task, so put it back to 'ready' and re-dispatch.
|
|
958
|
-
for (const task of running.taskDag.all()) {
|
|
959
|
-
if (task.status !== 'running')
|
|
960
|
-
continue;
|
|
961
|
-
if (checkpoint.roleState[task.assignee]?.sessionId)
|
|
962
|
-
continue;
|
|
963
|
-
running.taskDag.requeue(task.id);
|
|
964
|
-
}
|
|
965
|
-
decisionOps.dispatchReadyTasks(this, name, running);
|
|
966
|
-
if (worktreePath)
|
|
967
|
-
running.worktreePath = worktreePath;
|
|
968
|
-
}
|
|
969
|
-
else {
|
|
970
|
-
spawnRole(bossRole); // always, ungated — see comment above
|
|
971
|
-
// Lazy spawn: register non-boss roles as pending. They spawn on first
|
|
972
|
-
// message (see deliver()), avoiding the memory gate stampede at startup.
|
|
973
|
-
const pendingRoles = new Map();
|
|
974
|
-
for (const role of def.roles) {
|
|
975
|
-
if (role.id === bossRole.id || isEndpointRole(role))
|
|
976
|
-
continue;
|
|
977
|
-
pendingRoles.set(role.id, role);
|
|
978
|
-
}
|
|
979
|
-
running.pendingRoles = pendingRoles;
|
|
980
|
-
running.spawnRole = spawnRole;
|
|
981
|
-
running.taskDag = new TaskDag();
|
|
982
|
-
if (worktreePath)
|
|
983
|
-
running.worktreePath = worktreePath;
|
|
984
|
-
}
|
|
985
|
-
// Crash cleanup: reap SDK children if this process exits abnormally.
|
|
986
|
-
// monolean: process-scoped listener — upgrade path = per-org tracking
|
|
987
|
-
const crashCleanup = () => {
|
|
988
|
-
try {
|
|
989
|
-
// Statically imported: a process 'exit' handler must be synchronous,
|
|
990
|
-
// so `await import()` is unavailable — and a bare require() throws
|
|
991
|
-
// "require is not defined" in this ESM package. Guarded by
|
|
992
|
-
// no-cjs-require-in-esm.test.ts.
|
|
993
|
-
reapOrphanedSdkProcesses(new Set(), process.pid);
|
|
994
|
-
}
|
|
995
|
-
catch {
|
|
996
|
-
/* best-effort */
|
|
997
|
-
}
|
|
998
|
-
// #301: `process.on('exit')` is the SECOND termination path — it fires
|
|
999
|
-
// for a normal stop too (finishStop's own prune above already covers
|
|
1000
|
-
// that case, so this is a harmless idempotent repeat there) but also
|
|
1001
|
-
// for every path that reaches it via an explicit `process.exit()`
|
|
1002
|
-
// (org.ts's SIGTERM/SIGINT/SIGHUP handlers, uncaughtException,
|
|
1003
|
-
// unhandledRejection) — none of which run finishStop's cleanup at all.
|
|
1004
|
-
// execFileSync is already a static top-level import (see the comment
|
|
1005
|
-
// above), so this stays synchronous-safe like the rest of this
|
|
1006
|
-
// handler. SIGKILL cannot reach here — no in-process code runs for
|
|
1007
|
-
// it — which is why prune-at-start exists as the complement.
|
|
1008
|
-
try {
|
|
1009
|
-
execFileSync('git', ['worktree', 'prune'], {
|
|
1010
|
-
cwd: this.root,
|
|
1011
|
-
stdio: 'ignore',
|
|
1012
|
-
timeout: 30_000,
|
|
1013
|
-
});
|
|
1014
|
-
}
|
|
1015
|
-
catch {
|
|
1016
|
-
/* best-effort: not a git repo, git missing, or a wedged hook */
|
|
1017
|
-
}
|
|
1018
|
-
};
|
|
1019
|
-
process.on('exit', crashCleanup);
|
|
1020
|
-
running._crashCleanup = crashCleanup;
|
|
1021
|
-
// Stale-base drift detection: if the working tree is too many commits behind
|
|
1022
|
-
// its tracking branch, warn or refuse to start. Best-effort — git may not be
|
|
1023
|
-
// available, or the repo may have no tracking branch.
|
|
1024
|
-
const staleThreshold = def.run_config.stale_base_threshold;
|
|
1025
|
-
if (staleThreshold && staleThreshold > 0 && cwd === this.root) {
|
|
1026
|
-
try {
|
|
1027
|
-
const { execSync } = await import('node:child_process');
|
|
1028
|
-
const behind = execSync('git rev-list --count HEAD..@{upstream} 2>/dev/null', {
|
|
1029
|
-
cwd,
|
|
1030
|
-
encoding: 'utf8',
|
|
1031
|
-
timeout: 10_000,
|
|
1032
|
-
}).trim();
|
|
1033
|
-
const count = parseInt(behind, 10);
|
|
1034
|
-
if (!Number.isNaN(count) && count > staleThreshold) {
|
|
1035
|
-
bus.emit({
|
|
1036
|
-
type: 'audit',
|
|
1037
|
-
reason: 'stale-base',
|
|
1038
|
-
msg: `working tree is ${count} commits behind upstream (threshold: ${staleThreshold}) — consider pulling before running`,
|
|
1039
|
-
data: { behind: count, threshold: staleThreshold },
|
|
1040
|
-
});
|
|
1041
|
-
}
|
|
1042
|
-
}
|
|
1043
|
-
catch {
|
|
1044
|
-
/* no upstream tracking or git unavailable — skip silently */
|
|
1045
|
-
}
|
|
1046
|
-
}
|
|
1047
|
-
const boss = bossRole;
|
|
1048
|
-
if (options?.resume) {
|
|
1049
|
-
if (running.agents.get(boss.id)?.mailbox.serialize().queue.length === 0) {
|
|
1050
|
-
running.agents
|
|
1051
|
-
.get(boss.id)
|
|
1052
|
-
?.mailbox.push(`Org "${name}" resumed from checkpoint (run ${run}).\nGoal: ${taskOverride ?? def.goal}\n` +
|
|
1053
|
-
`Outstanding tasks and role states have been restored. Continue coordinating your team.`);
|
|
1054
|
-
}
|
|
1055
|
-
bus.emit({
|
|
1056
|
-
type: 'status',
|
|
1057
|
-
msg: `org resumed from checkpoint (${run})`,
|
|
1058
|
-
data: { goal: taskOverride ?? def.goal },
|
|
1059
|
-
});
|
|
1060
|
-
}
|
|
1061
|
-
else {
|
|
1062
|
-
// Cross-run memory: brief the coordinator on the previous run so scheduled
|
|
1063
|
-
// orgs accumulate instead of starting cold every interval.
|
|
1064
|
-
const prev = readHistory(this.root, name).at(-1);
|
|
1065
|
-
const prevBrief = prev
|
|
1066
|
-
? `\n\nPrevious run (${prev.run}${prev.endedAt ? `, ${new Date(prev.endedAt).toISOString()}` : ''}): ` +
|
|
1067
|
-
(prev.outcome
|
|
1068
|
-
? `outcome "${prev.outcome.status}" — ${prev.outcome.summary}`
|
|
1069
|
-
: `no recorded outcome (${prev.messages} messages, ${prev.assets.length} assets${prev.crashes.length ? `, ${prev.crashes.length} crashed agent(s)` : ''})`) +
|
|
1070
|
-
`\nBuild on that work — do not redo what is already done.`
|
|
1071
|
-
: '';
|
|
1072
|
-
running.agents
|
|
1073
|
-
.get(boss.id)
|
|
1074
|
-
?.mailbox.push(`Org "${name}" started (run ${run}).\nGoal: ${taskOverride ?? def.goal}\n` +
|
|
1075
|
-
`Coordinate your team via org_send. Only when the FULL goal above is achieved (or clearly can't be) — not merely "this batch of dispatched tasks finished" — record it with org_complete, then end your turn. ` +
|
|
1076
|
-
`If a batch finishes but the goal has more scope left, dispatch the next batch instead of ending the run.${prevBrief}`);
|
|
1077
|
-
bus.emit({
|
|
1078
|
-
type: 'status',
|
|
1079
|
-
msg: `org started (${sessionRoles.length} agents)`,
|
|
1080
|
-
data: { goal: taskOverride ?? def.goal },
|
|
1081
|
-
});
|
|
1082
|
-
}
|
|
1083
|
-
this.persistState(name, 'running', run);
|
|
1084
|
-
// Idle watchdog: a hung tool call (or a run that quietly finished without
|
|
1085
|
-
// org_complete) produces no bus events, and every agent just waits. After
|
|
1086
|
-
// idle_minutes of silence, nudge the boss to complete or reassign; if the
|
|
1087
|
-
// nudge itself produces no activity (boss hung/crashed), or the org keeps
|
|
1088
|
-
// going idle after MAX_IDLE_NUDGES nudges in a row without ever recovering
|
|
1089
|
-
// (see resolvedIdleNudgeCount), stop the run instead of letting it freeze
|
|
1090
|
-
// forever. idle_minutes: 0 disables.
|
|
1091
|
-
const idleMs = (def.run_config.idle_minutes ?? 10) * 60_000;
|
|
1092
|
-
if (idleMs > 0) {
|
|
1093
|
-
const MAX_IDLE_NUDGES = 3;
|
|
1094
|
-
let nudgedAt = 0;
|
|
1095
|
-
let nudges = 0;
|
|
1096
|
-
let stopping = false;
|
|
1097
|
-
const idleStop = (msg) => {
|
|
1098
|
-
stopping = true;
|
|
1099
|
-
bus.emit({ type: 'audit', reason: 'idle-stop', msg });
|
|
1100
|
-
// #302: closedBy: 'idle-stop' — the truth gate at finishStop's
|
|
1101
|
-
// history write reads this to record what actually happened
|
|
1102
|
-
// (including any runnable work left in org_tasks) instead of
|
|
1103
|
-
// letting a null outcome default to a plain "completed".
|
|
1104
|
-
this.stopOrg(name, { closedBy: 'idle-stop' }).catch((err) => console.error(`org ${name}: idle-stop failed:`, err instanceof Error ? err.message : err));
|
|
1105
|
-
};
|
|
1106
|
-
// #296: publish the projected stop time for `org status --json`. Only
|
|
1107
|
-
// written when it changes; a failed write must not throw out of the
|
|
1108
|
-
// interval (that would reach the process crash handlers).
|
|
1109
|
-
let published = '';
|
|
1110
|
-
const publishDeadline = (hold) => {
|
|
1111
|
-
if (stopping)
|
|
1112
|
-
return;
|
|
1113
|
-
const bossRt = running.agents.get(bossRole.id);
|
|
1114
|
-
const at = hold
|
|
1115
|
-
? null
|
|
1116
|
-
: new Date(projectIdleStop({
|
|
1117
|
-
lastActivity,
|
|
1118
|
-
nudgedAt,
|
|
1119
|
-
nudges,
|
|
1120
|
-
maxNudges: MAX_IDLE_NUDGES,
|
|
1121
|
-
idleMs,
|
|
1122
|
-
bossReachable: bossRt?.status === 'running' && !bossRt.mailbox.isClosed,
|
|
1123
|
-
})).toISOString();
|
|
1124
|
-
const key = `${at}|${hold?.reason ?? null}|${hold?.until ?? null}`;
|
|
1125
|
-
if (key === published)
|
|
1126
|
-
return;
|
|
1127
|
-
try {
|
|
1128
|
-
writeIdleRecord(this.root, name, {
|
|
1129
|
-
run,
|
|
1130
|
-
idle_minutes: idleMs / 60_000,
|
|
1131
|
-
idle_stop_at: at,
|
|
1132
|
-
hold,
|
|
1133
|
-
});
|
|
1134
|
-
published = key;
|
|
1135
|
-
}
|
|
1136
|
-
catch (err) {
|
|
1137
|
-
console.error(`org ${name}: could not write the idle deadline:`, err instanceof Error ? err.message : err);
|
|
1138
|
-
}
|
|
1139
|
-
};
|
|
1140
|
-
// ADR-O001 D4 — Gas Town's "30 minutes hooked without progress" alarm.
|
|
1141
|
-
// The org-wide idle clock is silent while a hold is in force, so a role
|
|
1142
|
-
// that is nominally running and producing nothing gets its own, loud
|
|
1143
|
-
// audit event. Once per spell: the bus subscriber clears the flag as
|
|
1144
|
-
// soon as the role emits anything.
|
|
1145
|
-
const alarmNoProgress = (now) => {
|
|
1146
|
-
// Only a role with work can be stalled on it: one with no task and no
|
|
1147
|
-
// mail, parked on its mailbox (or with its process down after
|
|
1148
|
-
// session_idle_exit_ms), is waiting, not hooked.
|
|
1149
|
-
const withTask = new Set((running.taskDag?.all() ?? [])
|
|
1150
|
-
.filter((t) => t.status === 'running')
|
|
1151
|
-
.map((t) => t.assignee));
|
|
1152
|
-
const stalled = noProgressRoles([...running.agents].map(([id, rt]) => ({
|
|
1153
|
-
id,
|
|
1154
|
-
working: rt.status === 'running' &&
|
|
1155
|
-
!rt.mailbox.isClosed &&
|
|
1156
|
-
hookedOnWork({
|
|
1157
|
-
runningTask: withTask.has(id),
|
|
1158
|
-
queuedMail: rt.mailbox.peek() !== undefined,
|
|
1159
|
-
awaitingMail: rt.mailbox.awaitingMail,
|
|
1160
|
-
}),
|
|
1161
|
-
lastActivity: roleActivity.get(id) ?? lastActivity,
|
|
1162
|
-
alarmed: noProgressAlarmed.has(id),
|
|
1163
|
-
})), now);
|
|
1164
|
-
for (const role of stalled) {
|
|
1165
|
-
noProgressAlarmed.add(role.id);
|
|
1166
|
-
bus.emit({
|
|
1167
|
-
type: 'audit',
|
|
1168
|
-
from: role.id,
|
|
1169
|
-
reason: 'no-progress',
|
|
1170
|
-
msg: `role "${role.id}" has been running for ${Math.round(role.silentMs / 60_000)}m ` +
|
|
1171
|
-
`without a single bus event — it is hooked but producing nothing`,
|
|
1172
|
-
data: { role: role.id, silentMinutes: Math.round(role.silentMs / 60_000) },
|
|
1173
|
-
});
|
|
1174
|
-
}
|
|
1175
|
-
};
|
|
1176
|
-
// The legitimate waits the watchdog holds through — every one of them
|
|
1177
|
-
// with a deadline attached by advanceHold (ADR-O001 D4).
|
|
1178
|
-
const holdReason = () => {
|
|
1179
|
-
if (this.restarting.has(name))
|
|
1180
|
-
return 'restarting'; // boss auto-restart in flight
|
|
1181
|
-
// A pending gate means the org is legitimately waiting for human input
|
|
1182
|
-
const pendingGates = this.readGates(name).gates.filter((g) => g.status === 'pending');
|
|
1183
|
-
if (pendingGates.length > 0)
|
|
1184
|
-
return 'pending-gate';
|
|
1185
|
-
// Bug 3: a pending ask_human question is the same kind of legitimate
|
|
1186
|
-
// wait as a pending gate — askHuman()'s receipt tells the role to end
|
|
1187
|
-
// its turn and wait for the resolution, so a role that follows that
|
|
1188
|
-
// instruction and goes quiet looks identical to a genuinely stalled
|
|
1189
|
-
// agent. Without this check the watchdog nudges (and, after enough
|
|
1190
|
-
// nudges, idle-stops) an org that's simply waiting on a human answer
|
|
1191
|
-
// that's already on its way.
|
|
1192
|
-
//
|
|
1193
|
-
// ADR-O001 D4: only a question the asker declared BLOCKING counts. The
|
|
1194
|
-
// 8.3-hour stall was held open by a question whose own text opened
|
|
1195
|
-
// with "no answer needed for the run to continue; I am not blocking on
|
|
1196
|
-
// this" — it suppressed the watchdog exactly like a real blocker.
|
|
1197
|
-
if (questionOps.pendingBlockingQuestions(this.root, name).length > 0)
|
|
1198
|
-
return 'pending-question';
|
|
1199
|
-
// M1 (C-41): a pending tool approval is the same kind of legitimate
|
|
1200
|
-
// wait — the role was told to wait for `org approve/deny`.
|
|
1201
|
-
if ((this.approvals.get(name) ?? []).some((a) => a.approved === null))
|
|
1202
|
-
return 'pending-approval';
|
|
1203
|
-
// M2 (C-41): a delivered endpoint message whose reply is still due.
|
|
1204
|
-
if (hasActiveEndpointWait(running))
|
|
1205
|
-
return 'endpoint-reply-due';
|
|
1206
|
-
// A task blocked on a real-world time still in the future is
|
|
1207
|
-
// legitimate waiting, same as a pending gate — don't nudge about it.
|
|
1208
|
-
// Its deadline is the time the asker actually named, not the default
|
|
1209
|
-
// hold TTL, so a block set hours out is honoured exactly.
|
|
1210
|
-
const blockedUntil = running.taskDag?.activeBlockUntil(Date.now()) ?? null;
|
|
1211
|
-
if (blockedUntil !== null)
|
|
1212
|
-
return { reason: 'task-blocked', until: blockedUntil };
|
|
1213
|
-
return null;
|
|
1214
|
-
};
|
|
1215
|
-
// Auto-resume any task whose org_task_block time has passed: flip it
|
|
1216
|
-
// back to 'running' and re-push it into the assignee's mailbox, same as
|
|
1217
|
-
// a fresh dispatch. This IS real activity, so it feeds the normal
|
|
1218
|
-
// idleFor check rather than short-circuiting it — an unblocked task
|
|
1219
|
-
// should reset the idle clock, not just silently update state nobody
|
|
1220
|
-
// notices until the next nudge. Runs on every tick, a hold in force
|
|
1221
|
-
// included: a block that expires mid-wait is real work again.
|
|
1222
|
-
const resumeExpiredBlocks = () => {
|
|
1223
|
-
wakeDueBlockRechecks(running, Date.now()); // #329: every block is re-checked
|
|
1224
|
-
const unblocked = running.taskDag?.unblockExpired(Date.now()) ?? [];
|
|
1225
|
-
for (const task of unblocked) {
|
|
1226
|
-
const agent = running.agents.get(task.assignee);
|
|
1227
|
-
if (agent && !agent.mailbox.isClosed) {
|
|
1228
|
-
agent.mailbox.push(`${taskTag(task)} Block expired — resuming: ${task.title}`);
|
|
1229
|
-
}
|
|
1230
|
-
bus.emit({
|
|
1231
|
-
type: 'status',
|
|
1232
|
-
from: 'dag',
|
|
1233
|
-
reason: 'task-unblocked',
|
|
1234
|
-
msg: `task ${task.id} block expired — resumed and re-dispatched to ${task.assignee}`,
|
|
1235
|
-
data: { taskId: task.id, assignee: task.assignee },
|
|
1236
|
-
});
|
|
1237
|
-
}
|
|
1238
|
-
};
|
|
1239
|
-
// The normal idle path: nudge the boss, then stop if the nudge produced
|
|
1240
|
-
// nothing. Runs only when nothing (still) holds the watchdog.
|
|
1241
|
-
const check = () => {
|
|
1242
|
-
const idleFor = Date.now() - lastActivity;
|
|
1243
|
-
if (idleFor < idleMs) {
|
|
1244
|
-
nudges = resolvedIdleNudgeCount(nudgedAt, nudges, lastToolActivity);
|
|
1245
|
-
nudgedAt = 0;
|
|
1246
|
-
return;
|
|
1247
|
-
}
|
|
1248
|
-
if (nudgedAt === 0) {
|
|
1249
|
-
if (nudges >= MAX_IDLE_NUDGES) {
|
|
1250
|
-
idleStop(`org idle again after ${nudges} nudges — stopping run`);
|
|
1251
|
-
return;
|
|
1252
|
-
}
|
|
1253
|
-
const bossRt = running.agents.get(bossRole.id);
|
|
1254
|
-
// #205: a budget-exhausted boss closed its own mailbox on
|
|
1255
|
-
// purpose (session.ts) — that's a recoverable pause, not the
|
|
1256
|
-
// same "unreachable" condition as a crash. Name it distinctly so
|
|
1257
|
-
// the operator's remedy (raise the budget, resume) is obvious
|
|
1258
|
-
// instead of reading like the run died.
|
|
1259
|
-
const budgetReason = bossRt?.mailbox.closeReason;
|
|
1260
|
-
if (budgetReason === 'token-budget' || budgetReason === 'usd-budget') {
|
|
1261
|
-
idleStop(`org idle for ${Math.round(idleFor / 60_000)}m and boss "${bossRole.id}" is over its ` +
|
|
1262
|
-
`${budgetReason === 'token-budget' ? 'token' : 'USD'} budget — raise the role's ` +
|
|
1263
|
-
`${budgetReason === 'token-budget' ? 'budget_tokens' : 'budget_usd'} (or run_config's) and resume from checkpoint — stopping run`);
|
|
1264
|
-
return;
|
|
1265
|
-
}
|
|
1266
|
-
if (bossRt?.status !== 'running' || bossRt.mailbox.isClosed) {
|
|
1267
|
-
idleStop(`org idle for ${Math.round(idleFor / 60_000)}m and boss "${bossRole.id}" is unreachable — stopping run`);
|
|
1268
|
-
return;
|
|
1269
|
-
}
|
|
1270
|
-
nudges++;
|
|
1271
|
-
nudgedAt = Date.now();
|
|
1272
|
-
bus.emit({
|
|
1273
|
-
type: 'audit',
|
|
1274
|
-
from: bossRole.id,
|
|
1275
|
-
reason: 'idle-nudge',
|
|
1276
|
-
msg: `no org activity for ${Math.round(idleFor / 60_000)}m — nudging boss (${nudges}/${MAX_IDLE_NUDGES})`,
|
|
1277
|
-
});
|
|
1278
|
-
bossRt.mailbox.push(`[watchdog] No activity in org "${name}" for ${Math.round(idleFor / 60_000)} minute(s). ` +
|
|
1279
|
-
`Check org_tasks first, then pick ONE: (1) the org's full stated goal is achieved or clearly cannot be — call org_complete now (this ends the run for good, not just this batch); ` +
|
|
1280
|
-
`(2) someone has stalled or unstarted work — check on your team via org_send and reassign it; ` +
|
|
1281
|
-
`(3) the current task batch is done but the goal has more scope left — do NOT call org_complete for this case, instead dispatch the next batch of work with org_task/createTask so the org keeps making progress; ` +
|
|
1282
|
-
`(4) a task is stuck 'running' only because it's genuinely waiting on a real-world time (a scheduled process, a deadline) and there is nothing else to dispatch right now — do NOT just leave it and re-confirm this every time you get nudged, call org_task_block(taskId, untilIso, reason) instead so this watchdog stops nudging you about it and auto-resumes the task when the time arrives.`);
|
|
1283
|
-
}
|
|
1284
|
-
else if (Date.now() - nudgedAt >= idleMs) {
|
|
1285
|
-
idleStop(`nudge produced no activity for another ${Math.round(idleMs / 60_000)}m — boss appears hung, stopping run`);
|
|
1286
|
-
}
|
|
1287
|
-
};
|
|
1288
|
-
// One watchdog tick. The hold is resolved FIRST and with a deadline, so
|
|
1289
|
-
// a wait that outlives its deadline hands the run back to the idle path
|
|
1290
|
-
// instead of suppressing it forever (ADR-O001 D4).
|
|
1291
|
-
let holdTrack = null;
|
|
1292
|
-
const tick = () => {
|
|
1293
|
-
const now = Date.now();
|
|
1294
|
-
alarmNoProgress(now);
|
|
1295
|
-
resumeExpiredBlocks();
|
|
1296
|
-
const step = advanceHold(holdTrack, holdReason(), now);
|
|
1297
|
-
holdTrack = step.track;
|
|
1298
|
-
if (step.expired) {
|
|
1299
|
-
bus.emit({
|
|
1300
|
-
type: 'audit',
|
|
1301
|
-
reason: 'hold-expired',
|
|
1302
|
-
msg: `the "${step.expired}" hold on org "${name}" outlived its deadline — the idle ` +
|
|
1303
|
-
`watchdog is running again and will nudge, then stop the run if nothing happens`,
|
|
1304
|
-
data: { hold: step.expired },
|
|
1305
|
-
});
|
|
1306
|
-
}
|
|
1307
|
-
if (step.hold)
|
|
1308
|
-
return step.hold;
|
|
1309
|
-
check();
|
|
1310
|
-
return null;
|
|
1311
|
-
};
|
|
1312
|
-
const initialHold = advanceHold(null, holdReason(), Date.now());
|
|
1313
|
-
holdTrack = initialHold.track;
|
|
1314
|
-
publishDeadline(initialHold.hold);
|
|
1315
|
-
const wd = setInterval(() => publishDeadline(tick()), Math.max(200, Math.min(idleMs / 2, 30_000)));
|
|
1316
|
-
wd.unref?.();
|
|
1317
|
-
this.watchdogs.set(name, wd);
|
|
1318
|
-
}
|
|
1319
|
-
else {
|
|
1320
|
-
try {
|
|
1321
|
-
writeIdleRecord(this.root, name, {
|
|
1322
|
-
run,
|
|
1323
|
-
idle_minutes: 0,
|
|
1324
|
-
idle_stop_at: null,
|
|
1325
|
-
hold: { reason: 'disabled', until: null },
|
|
1326
|
-
});
|
|
1327
|
-
}
|
|
1328
|
-
catch (err) {
|
|
1329
|
-
console.error(`org ${name}: could not write the idle deadline:`, err instanceof Error ? err.message : err);
|
|
1330
|
-
}
|
|
1331
|
-
}
|
|
1332
|
-
if (this.opts.crossProcess && this.opts.inboxUrl) {
|
|
1333
|
-
const operatorCred = normalizeCredential(this.opts.operatorCredential);
|
|
1334
|
-
const lease = new BrokerLease(name, this.opts.inboxUrl, this.opts.brokerDir, undefined, running.credential, operatorCred ? { credential: operatorCred, dir: this.opts.operatorDir } : undefined, this.root);
|
|
1335
|
-
lease.start();
|
|
1336
|
-
this.leases.set(name, lease);
|
|
1337
|
-
}
|
|
1338
|
-
// Drain any messages that arrived while the org was offline
|
|
1339
|
-
const queued = drainInbox(this.root, name);
|
|
1340
|
-
// M2: messages for endpoint roles are delivered by POST, not into a mailbox —
|
|
1341
|
-
// put them back (flagged) and let the endpoint retry path send them.
|
|
1342
|
-
const endpointQueued = new Set();
|
|
1343
|
-
for (const msg of queued) {
|
|
1344
|
-
if (!isEndpointRole(def.roles.find((r) => r.id === msg.toRole)))
|
|
1345
|
-
continue;
|
|
1346
|
-
const messageId = msg.messageId ?? newMessageId();
|
|
1347
|
-
endpointQueued.add(messageId);
|
|
1348
|
-
queueMessage(this.root, name, { ...msg, messageId, endpoint: true });
|
|
1349
|
-
}
|
|
1350
|
-
startEndpointRetryLoop(this, name);
|
|
1351
|
-
if (endpointQueued.size > 0)
|
|
1352
|
-
void retryQueuedEndpoints(this, name, (m) => endpointQueued.has(m.messageId ?? '')).catch(() => {
|
|
1353
|
-
/* stays queued — the periodic sweep retries */
|
|
1354
|
-
});
|
|
1355
|
-
for (const msg of queued) {
|
|
1356
|
-
if (isEndpointRole(def.roles.find((r) => r.id === msg.toRole)))
|
|
1357
|
-
continue;
|
|
1358
|
-
// Spawn a lazy target before delivering. These messages were queued while
|
|
1359
|
-
// the org was offline — a human's answer, or another org's request — and
|
|
1360
|
-
// the whole point of draining is that they arrive. Skipping a role merely
|
|
1361
|
-
// because it has not spawned yet discarded them permanently, after
|
|
1362
|
-
// queueMessage had already reported them accepted.
|
|
1363
|
-
if (!running.agents.has(msg.toRole) && running.pendingRoles?.has(msg.toRole)) {
|
|
1364
|
-
const pending = running.pendingRoles.get(msg.toRole);
|
|
1365
|
-
// Bug 4: don't spawn past run_config.max_concurrent_agents. Requeue
|
|
1366
|
-
// this message (queueMessage, not a silent drop) and defer the spawn
|
|
1367
|
-
// the same way a concurrency-gated lazy spawn defers elsewhere.
|
|
1368
|
-
const concurrencyLimit = def.run_config.max_concurrent_agents;
|
|
1369
|
-
if (concurrencyLimit != null && activeRoleCount(running) >= concurrencyLimit) {
|
|
1370
|
-
running.pendingRoles.delete(msg.toRole);
|
|
1371
|
-
queueMessage(this.root, name, msg);
|
|
1372
|
-
this.scheduleConcurrencyDeferredSpawn(name, running, pending, running.spawnRole);
|
|
1373
|
-
}
|
|
1374
|
-
else {
|
|
1375
|
-
running.pendingRoles.delete(msg.toRole);
|
|
1376
|
-
running.spawnRole?.(pending);
|
|
1377
|
-
}
|
|
1378
|
-
}
|
|
1379
|
-
const agent = running.agents.get(msg.toRole);
|
|
1380
|
-
if (agent && !agent.mailbox.isClosed) {
|
|
1381
|
-
bus.emit({
|
|
1382
|
-
type: 'xorg',
|
|
1383
|
-
from: msg.fromQualified,
|
|
1384
|
-
to: `${name}:${msg.toRole}`,
|
|
1385
|
-
subject: msg.subject,
|
|
1386
|
-
msg: msg.body,
|
|
1387
|
-
data: { messageId: msg.messageId ?? newMessageId() },
|
|
1388
|
-
});
|
|
1389
|
-
await crossOrg.pushMessage(this, name, running, msg.toRole, msg.fromQualified, msg.subject, msg.body, `inbox-${msg.ts}-${Math.random().toString(36).slice(2, 8)}`);
|
|
1390
|
-
}
|
|
1391
|
-
}
|
|
1392
|
-
if (queued.length)
|
|
1393
|
-
bus.emit({ type: 'status', msg: `drained ${queued.length} queued message(s) from inbox` });
|
|
1394
|
-
return running;
|
|
179
|
+
return orgStart.startOrg(this, name, taskOverride, options);
|
|
1395
180
|
}
|
|
1396
181
|
/** Build one role incarnation: mailbox, policy, AgentRuntime, sessionOpts,
|
|
1397
182
|
* and the supervised crash-retry loop. Used by BOTH the startup lazy-spawn
|
|
@@ -1399,896 +184,13 @@ export class OrgDaemon {
|
|
|
1399
184
|
* and respawnRole() (generation N+1). Does not touch running.agents or
|
|
1400
185
|
* running.roleSlots — callers publish the result themselves. */
|
|
1401
186
|
spawnRoleIncarnation(name, running, role, generation, opts = {}) {
|
|
1402
|
-
|
|
1403
|
-
const abort = opts.abort ?? new AbortController();
|
|
1404
|
-
const { def, bus, run } = running;
|
|
1405
|
-
const cwd = running.workdir;
|
|
1406
|
-
const ws = this.workspaceSetting(def);
|
|
1407
|
-
const perRoleBudget = opts.budgetTokensOverride ?? computeReplacementBudget(def, role.id);
|
|
1408
|
-
let roleCwd = cwd;
|
|
1409
|
-
const existingSlot = running.roleSlots.get(role.id);
|
|
1410
|
-
if (ws === 'worktree-per-role' && role.id !== running.bossRoleId) {
|
|
1411
|
-
const wtPath = join(this.root, ORG_DIR, name, `worktree-${role.id}`);
|
|
1412
|
-
if (existingSlot?.runtime?.worktreePath === wtPath && existsSync(wtPath)) {
|
|
1413
|
-
// A replacement (generation > 0) reuses the SAME worktree path —
|
|
1414
|
-
// recreating it here would delete any uncommitted work the old
|
|
1415
|
-
// incarnation left behind (design constraint #5).
|
|
1416
|
-
roleCwd = wtPath;
|
|
1417
|
-
}
|
|
1418
|
-
else {
|
|
1419
|
-
try {
|
|
1420
|
-
// Q7: top-level `import { execFileSync }` replaces the inlined
|
|
1421
|
-
// `require('node:child_process')` that broke ESM at runtime —
|
|
1422
|
-
// vitest's CJS shim masked it in tests but the built package
|
|
1423
|
-
// threw "require is not defined" in real Node ESM execution.
|
|
1424
|
-
// SEC-5: argv-array form, no shell.
|
|
1425
|
-
if (existsSync(wtPath)) {
|
|
1426
|
-
try {
|
|
1427
|
-
execFileSync('git', ['worktree', 'remove', '--force', wtPath], {
|
|
1428
|
-
cwd: this.root,
|
|
1429
|
-
stdio: 'ignore',
|
|
1430
|
-
timeout: 30_000,
|
|
1431
|
-
});
|
|
1432
|
-
}
|
|
1433
|
-
catch {
|
|
1434
|
-
/* best-effort */
|
|
1435
|
-
}
|
|
1436
|
-
}
|
|
1437
|
-
execFileSync('git', ['worktree', 'add', wtPath, 'HEAD', '--detach'], {
|
|
1438
|
-
cwd: this.root,
|
|
1439
|
-
stdio: 'ignore',
|
|
1440
|
-
timeout: 30_000,
|
|
1441
|
-
});
|
|
1442
|
-
roleCwd = wtPath;
|
|
1443
|
-
}
|
|
1444
|
-
catch {
|
|
1445
|
-
/* fallback to shared cwd if git worktree fails */
|
|
1446
|
-
}
|
|
1447
|
-
}
|
|
1448
|
-
}
|
|
1449
|
-
const mailbox = new Mailbox();
|
|
1450
|
-
if (roleCheckpoint?.mailboxQueue?.length) {
|
|
1451
|
-
restoreMailboxQueue({ mailbox }, roleCheckpoint.mailboxQueue);
|
|
1452
|
-
}
|
|
1453
|
-
// A recoverable close (budget exhaustion) is left open on resume — see
|
|
1454
|
-
// isRecoverableCloseReason's doc comment. Re-closing it here would
|
|
1455
|
-
// make the idle watchdog's "raise the budget and resume" remedy a
|
|
1456
|
-
// no-op, since nothing in this codebase ever reopens a closed mailbox.
|
|
1457
|
-
if (roleCheckpoint?.mailboxClosed &&
|
|
1458
|
-
!isRecoverableCloseReason(roleCheckpoint.mailboxCloseReason)) {
|
|
1459
|
-
mailbox.close(roleCheckpoint.mailboxCloseReason);
|
|
1460
|
-
}
|
|
1461
|
-
const policy = new PolicyEngine(role.id, {
|
|
1462
|
-
maxTokens: role.budget_tokens ?? perRoleBudget,
|
|
1463
|
-
// ADR-O001 D1: which basis that ceiling is enforced on. Defaults to
|
|
1464
|
-
// the historical uncached basis so the honest (cache-aware) meter
|
|
1465
|
-
// introduced alongside it cannot exhaust an existing budget_tokens —
|
|
1466
|
-
// including the schema's 1M default — roughly 100x early.
|
|
1467
|
-
maxTokensBasis: def.run_config.budget_tokens_basis ?? 'uncached',
|
|
1468
|
-
maxUsd: role.budget_usd,
|
|
1469
|
-
...(role.policy ?? {}),
|
|
1470
|
-
}, bus, roleCwd,
|
|
1471
|
-
// #303: the file tools (Read/Write/Edit/Glob/Grep) get the same extra
|
|
1472
|
-
// roots the Bash sandbox already treats as writable (role-sandbox.ts) —
|
|
1473
|
-
// $TMPDIR, the org root, and any policy.sandbox.allowWrite entries.
|
|
1474
|
-
// $HOME is deliberately excluded; see file-roots.ts.
|
|
1475
|
-
fileToolRoots({ cwd: roleCwd, orgRoot: this.root }, role.policy?.sandbox));
|
|
1476
|
-
policy.setToolContext({
|
|
1477
|
-
providerPrefixes: () => roleProviderPrefixes({ tool_providers: effectiveToolProviders(role, this.root) }),
|
|
1478
|
-
trace: () => this.roleTrace(name, role.id),
|
|
1479
|
-
});
|
|
1480
|
-
// ADR-O001 D1: prefer the persisted four-quantity breakdown; a checkpoint
|
|
1481
|
-
// written before it existed still resumes via the scalar, on the uncached
|
|
1482
|
-
// basis it was recorded on.
|
|
1483
|
-
if (roleCheckpoint?.tokenUsage) {
|
|
1484
|
-
policy.setTokenUsage(roleCheckpoint.tokenUsage);
|
|
1485
|
-
}
|
|
1486
|
-
else if (roleCheckpoint?.tokensUsed) {
|
|
1487
|
-
policy.setUsage(roleCheckpoint.tokensUsed);
|
|
1488
|
-
}
|
|
1489
|
-
// ORG-7: restore accumulated USD spend across resume so a stop/resume
|
|
1490
|
-
// cycle can't reset a role's USD budget back to zero.
|
|
1491
|
-
if (roleCheckpoint?.costUsd) {
|
|
1492
|
-
policy.setUsageUsd(roleCheckpoint.costUsd);
|
|
1493
|
-
}
|
|
1494
|
-
// ADR-O001 D7: the loadout this incarnation's session is built with, fixed
|
|
1495
|
-
// for its life. A checkpointed role keeps what it had (its SDK session was
|
|
1496
|
-
// built with it); a replacement keeps its predecessor's; a new role takes
|
|
1497
|
-
// the loadout of the first ready task it is being spawned for.
|
|
1498
|
-
const loadoutName = roleCheckpoint
|
|
1499
|
-
? roleCheckpoint.loadout
|
|
1500
|
-
: (existingSlot?.runtime?.loadout ?? sessionLoadoutFor(running.taskDag, role.id));
|
|
1501
|
-
let loadout;
|
|
1502
|
-
if (loadoutName) {
|
|
1503
|
-
try {
|
|
1504
|
-
loadout = resolveLoadout(def, loadoutName, this.root);
|
|
1505
|
-
}
|
|
1506
|
-
catch (err) {
|
|
1507
|
-
// Validated at start, so only a config hot-reload or a deleted
|
|
1508
|
-
// instructions_file lands here. Spawn without it, loudly: a role that
|
|
1509
|
-
// fails to spawn would strand its task, which is worse.
|
|
1510
|
-
bus.emit({
|
|
1511
|
-
type: 'audit',
|
|
1512
|
-
from: role.id,
|
|
1513
|
-
reason: 'loadout-unresolvable',
|
|
1514
|
-
msg: `role "${role.id}" spawned without loadout "${loadoutName}": ${err instanceof Error ? err.message : err}`,
|
|
1515
|
-
data: { loadout: loadoutName },
|
|
1516
|
-
});
|
|
1517
|
-
}
|
|
1518
|
-
}
|
|
1519
|
-
const runtime = {
|
|
1520
|
-
mailbox,
|
|
1521
|
-
policy,
|
|
1522
|
-
status: restoredRoleStatus(roleCheckpoint),
|
|
1523
|
-
done: Promise.resolve(),
|
|
1524
|
-
metrics: { tokens: roleCheckpoint?.tokensUsed ?? 0, costUsd: roleCheckpoint?.costUsd ?? 0 },
|
|
1525
|
-
lastMessageId: roleCheckpoint?.lastMessageId,
|
|
1526
|
-
error: roleCheckpoint?.error,
|
|
1527
|
-
sessionId: roleCheckpoint?.sessionId,
|
|
1528
|
-
worktreePath: roleCwd !== cwd ? roleCwd : undefined,
|
|
1529
|
-
scrollback: new ScrollbackBuffer(),
|
|
1530
|
-
...(loadout ? { loadout: loadout.name } : {}),
|
|
1531
|
-
taskProcesses: new TaskProcesses(),
|
|
1532
|
-
};
|
|
1533
|
-
if (roleCheckpoint?.scrollback?.length) {
|
|
1534
|
-
for (const line of roleCheckpoint.scrollback)
|
|
1535
|
-
runtime.scrollback.push(line);
|
|
1536
|
-
}
|
|
1537
|
-
if (roleCheckpoint?.turns && !running.turns?.has(role.id)) {
|
|
1538
|
-
(running.turns ??= new Map()).set(role.id, roleCheckpoint.turns);
|
|
1539
|
-
}
|
|
1540
|
-
const sessionOpts = {
|
|
1541
|
-
org: name,
|
|
1542
|
-
role,
|
|
1543
|
-
bus,
|
|
1544
|
-
policy,
|
|
1545
|
-
mailbox,
|
|
1546
|
-
taskProcesses: runtime.taskProcesses,
|
|
1547
|
-
cwd: roleCwd,
|
|
1548
|
-
def,
|
|
1549
|
-
// Pass the org state directory so runners that persist per-role state
|
|
1550
|
-
// (VercelAgentRunner session files) write under .monomind/orgs/<name>
|
|
1551
|
-
// instead of polluting the workspace cwd.
|
|
1552
|
-
orgDir: join(this.root, ORG_DIR, name),
|
|
1553
|
-
// Project root for named-provider (`adapter_config.provider`) config
|
|
1554
|
-
// lookup — role cwd may be an isolated workspace with no config file.
|
|
1555
|
-
orgRoot: this.root,
|
|
1556
|
-
run,
|
|
1557
|
-
// M1: role tool providers — listed at session start, processes spawned
|
|
1558
|
-
// lazily on first call and killed when the session ends.
|
|
1559
|
-
buildProviderTools: async () => {
|
|
1560
|
-
const providers = effectiveToolProviders(role, this.root);
|
|
1561
|
-
if (providers.length === 0)
|
|
1562
|
-
return undefined;
|
|
1563
|
-
return this.toolProviders.buildRoleTools({
|
|
1564
|
-
ctx: { org: name, run, role: role.id, root: this.root },
|
|
1565
|
-
providers,
|
|
1566
|
-
trace: () => this.roleTrace(name, role.id),
|
|
1567
|
-
bus,
|
|
1568
|
-
cwd: roleCwd,
|
|
1569
|
-
});
|
|
1570
|
-
},
|
|
1571
|
-
maxTurns: role.max_turns_per_message ?? def.run_config.max_turns_per_message,
|
|
1572
|
-
resumeSessionId: roleCheckpoint?.sessionId,
|
|
1573
|
-
sessionLedger: running.sessionLedger,
|
|
1574
|
-
// ADR-O001 D3 x D7: a task-scoped session is built with its task's own
|
|
1575
|
-
// recorded loadout. Unresolvable → no loadout, loudly (as at spawn).
|
|
1576
|
-
loadoutFor: (taskId) => {
|
|
1577
|
-
const name = running.taskDag?.get(taskId)?.loadout;
|
|
1578
|
-
if (!name)
|
|
1579
|
-
return undefined;
|
|
1580
|
-
try {
|
|
1581
|
-
return resolveLoadout(def, name, this.root);
|
|
1582
|
-
}
|
|
1583
|
-
catch (err) {
|
|
1584
|
-
bus.emit({
|
|
1585
|
-
type: 'audit',
|
|
1586
|
-
from: role.id,
|
|
1587
|
-
reason: 'loadout-unresolvable',
|
|
1588
|
-
msg: `task ${taskId} session built without loadout "${name}": ${err instanceof Error ? err.message : err}`,
|
|
1589
|
-
data: { loadout: name, taskId },
|
|
1590
|
-
});
|
|
1591
|
-
return undefined;
|
|
1592
|
-
}
|
|
1593
|
-
},
|
|
1594
|
-
lastMessageId: () => runtime.lastMessageId,
|
|
1595
|
-
onOutput: (line) => runtime.scrollback.push(line),
|
|
1596
|
-
onSessionId: (id) => {
|
|
1597
|
-
runtime.sessionId = id;
|
|
1598
|
-
},
|
|
1599
|
-
onTurnEnd: () => {
|
|
1600
|
-
endTurn(running, role.id);
|
|
1601
|
-
decisionOps.nudgeOpenTasksAtTurnEnd(running, role.id);
|
|
1602
|
-
},
|
|
1603
|
-
// #327: the role's org_send mail carries its chain at the next hop.
|
|
1604
|
-
deliver: (from, to, subject, body) => this.deliver(name, from, to, subject, withTrace(body, this.roleTrace(name, role.id))),
|
|
1605
|
-
askHuman: (r, question, blocking) => this.askHuman(name, r, question, blocking),
|
|
1606
|
-
onGate: (r, gateName, gateDesc) => this.createGate(name, r, gateName, gateDesc),
|
|
1607
|
-
circuitBreaker: (() => {
|
|
1608
|
-
const cb = def.run_config.circuit_breaker;
|
|
1609
|
-
if (!cb)
|
|
1610
|
-
return undefined;
|
|
1611
|
-
return { threshold: cb.failure_threshold ?? 5, state: { failures: 0, tripped: false } };
|
|
1612
|
-
})(),
|
|
1613
|
-
beforeTool: (r, toolName, input) => this.checkApproval(name, r, toolName, input),
|
|
1614
|
-
fence: running.fences?.get(role.id),
|
|
1615
|
-
// ORG-1: gatedCanUseTool denials are a natural decision point — record them so
|
|
1616
|
-
// `org decisions` shows real traces instead of always reporting none.
|
|
1617
|
-
onDecision: (r, toolName, message, kind) => {
|
|
1618
|
-
this.recordDecision(name, r, {
|
|
1619
|
-
type: 'tool',
|
|
1620
|
-
kind,
|
|
1621
|
-
context: `tool call: ${toolName}`,
|
|
1622
|
-
reasoning: message,
|
|
1623
|
-
outcome: 'denied',
|
|
1624
|
-
});
|
|
1625
|
-
},
|
|
1626
|
-
// ORG-9: decision gates are documented as "hard-blocking" — make that
|
|
1627
|
-
// true by actually denying tool use while this role has a pending gate,
|
|
1628
|
-
// the same way pending approvals already do.
|
|
1629
|
-
hasPendingGate: () => this.listGates(name, 'pending').some((g) => g.roleId === role.id),
|
|
1630
|
-
// #302: refuse an unsatisfiable org_complete BEFORE the 'org-complete'
|
|
1631
|
-
// status event ever exists — the bus subscriber at the top of this
|
|
1632
|
-
// function auto-stops the run on that event alone, so a refusal that
|
|
1633
|
-
// still emitted it would be undone by the very next tick regardless of
|
|
1634
|
-
// what this function returns to the tool handler.
|
|
1635
|
-
onComplete: role.id === running.bossRoleId
|
|
1636
|
-
? (r, outcome, summary, blocker, blockerDetail) => {
|
|
1637
|
-
let maxBudgetFraction = 0;
|
|
1638
|
-
for (const rt of running.agents.values()) {
|
|
1639
|
-
const p = rt.policy;
|
|
1640
|
-
if (p.policy.maxTokens)
|
|
1641
|
-
maxBudgetFraction = Math.max(maxBudgetFraction, p.usage / p.policy.maxTokens);
|
|
1642
|
-
if (p.policy.maxUsd)
|
|
1643
|
-
maxBudgetFraction = Math.max(maxBudgetFraction, p.usageUsd / p.policy.maxUsd);
|
|
1644
|
-
}
|
|
1645
|
-
// Same predicates the idle watchdog uses for its own
|
|
1646
|
-
// legitimate-wait check (:1316-1328) — a pending gate or an
|
|
1647
|
-
// unanswered question is the same "genuinely waiting on a
|
|
1648
|
-
// human" fact either way.
|
|
1649
|
-
const pendingHumanWaits = this.listGates(name, 'pending').length +
|
|
1650
|
-
questionOps.pendingBlockingQuestions(this.root, name).length;
|
|
1651
|
-
return resolveOrgComplete(bus, r, outcome, summary, blocker, blockerDetail, {
|
|
1652
|
-
mode: def.run_config.completion ?? 'boss',
|
|
1653
|
-
maxBudgetFraction,
|
|
1654
|
-
pendingHumanWaits,
|
|
1655
|
-
hasActiveBlock: running.taskDag?.hasActiveBlock(Date.now()) ?? false,
|
|
1656
|
-
hasPendingWork: running.taskDag?.hasPendingWork() ?? false,
|
|
1657
|
-
});
|
|
1658
|
-
}
|
|
1659
|
-
: undefined,
|
|
1660
|
-
// #11: a boss that overflows its context window isn't a crash (it keeps
|
|
1661
|
-
// returning +0-token errors forever), so without this the idle watchdog
|
|
1662
|
-
// just nudges it for ~30 min before idle-stopping. Restart the whole org
|
|
1663
|
-
// with fresh sessions instead — bounded by MAX_BOSS_RESTARTS.
|
|
1664
|
-
onContextLimit: role.id === running.bossRoleId ? () => this.scheduleBossRestart(name) : undefined,
|
|
1665
|
-
onListRuntimeOptions: role.id === running.bossRoleId && (def.run_config.max_role_respawns ?? 0) > 0
|
|
1666
|
-
? () => this.listRuntimeOptions()
|
|
1667
|
-
: undefined,
|
|
1668
|
-
onRespawnRole: role.id === running.bossRoleId && (def.run_config.max_role_respawns ?? 0) > 0
|
|
1669
|
-
? (callerId, args) => this.respawnRole(name, callerId, args)
|
|
1670
|
-
: undefined,
|
|
1671
|
-
recall: async (r, q) => {
|
|
1672
|
-
const answer = await this.recallOrgMemory(name, def, q, r);
|
|
1673
|
-
bus.emit({
|
|
1674
|
-
type: 'status',
|
|
1675
|
-
from: r,
|
|
1676
|
-
reason: 'org-recall',
|
|
1677
|
-
msg: `recall: ${q.slice(0, 80)}`,
|
|
1678
|
-
data: { hits: answer.hits },
|
|
1679
|
-
});
|
|
1680
|
-
return answer.text;
|
|
1681
|
-
},
|
|
1682
|
-
searchKnowledge: async (r, q) => {
|
|
1683
|
-
const answer = await this.searchProjectKnowledge(q);
|
|
1684
|
-
bus.emit({
|
|
1685
|
-
type: 'status',
|
|
1686
|
-
from: r,
|
|
1687
|
-
reason: 'knowledge-search',
|
|
1688
|
-
msg: `knowledge: ${q.slice(0, 80)}`,
|
|
1689
|
-
data: { hits: answer.hits },
|
|
1690
|
-
});
|
|
1691
|
-
return answer.text;
|
|
1692
|
-
},
|
|
1693
|
-
glossary: running.glossary,
|
|
1694
|
-
remember: async (r, content, scope) => {
|
|
1695
|
-
const text = await this.rememberOrgMemory(name, def, r, content, scope, run);
|
|
1696
|
-
bus.emit({
|
|
1697
|
-
type: 'status',
|
|
1698
|
-
from: r,
|
|
1699
|
-
reason: 'org-remember',
|
|
1700
|
-
msg: `remember (${scope}): ${content.slice(0, 80)}`,
|
|
1701
|
-
data: { scope },
|
|
1702
|
-
});
|
|
1703
|
-
return text;
|
|
1704
|
-
},
|
|
1705
|
-
learn: async (r, payload) => {
|
|
1706
|
-
const text = await this.learnOrgKnowledge(name, run, payload);
|
|
1707
|
-
bus.emit({
|
|
1708
|
-
type: 'status',
|
|
1709
|
-
from: r,
|
|
1710
|
-
reason: 'org-learn',
|
|
1711
|
-
msg: `learn: ${text.slice(0, 120)}`,
|
|
1712
|
-
data: {
|
|
1713
|
-
nodes: payload.nodes?.length ?? 0,
|
|
1714
|
-
edges: payload.edges?.length ?? 0,
|
|
1715
|
-
rules: payload.rules?.length ?? 0,
|
|
1716
|
-
},
|
|
1717
|
-
});
|
|
1718
|
-
return text;
|
|
1719
|
-
},
|
|
1720
|
-
createTask: (r, title, assignee, deps, loadout, brief, pick) => {
|
|
1721
|
-
return this.dagCreateTask(name, r, title, assignee, deps, loadout, brief, pick);
|
|
1722
|
-
},
|
|
1723
|
-
pickAssignee: resolveAutoAssignee(def, (id) => openTaskCount(this.orgs.get(name), id), () => this.orgs.get(name)?.taskDag?.all() ?? []),
|
|
1724
|
-
onSkillLoad: (r, skill) => decisionOps.recordSkillLoad(this.orgs.get(name), r, skill),
|
|
1725
|
-
// ADR-O001 D7: only an org with a catalog gets the `loadout` argument;
|
|
1726
|
-
// the session itself is built with the loadout frozen above.
|
|
1727
|
-
loadoutCatalog: loadoutCatalog(def),
|
|
1728
|
-
loadout,
|
|
1729
|
-
completeTask: (r, taskId, result, evidence) => {
|
|
1730
|
-
return this.dagCompleteTask(name, r, taskId, result, evidence);
|
|
1731
|
-
},
|
|
1732
|
-
// ADR-O001 D5: only an org that opted in advertises the evidence
|
|
1733
|
-
// argument, so every other org's tool list stays byte-identical.
|
|
1734
|
-
requireTaskEvidence: def.run_config.completion_evidence === true && role.deliberative !== true,
|
|
1735
|
-
// ADR-O001 D6: only an org with an artifact-only reviewer gets org_review,
|
|
1736
|
-
// so every other org's tool list stays byte-identical.
|
|
1737
|
-
requestReview: def.roles.some((r) => r.review_input === 'artifact-only')
|
|
1738
|
-
? (r, taskId, reviewer, base) => this.dagRequestReview(name, r, taskId, reviewer, base)
|
|
1739
|
-
: undefined,
|
|
1740
|
-
listTasks: (taskId) => decisionOps.dagListTasks(this, name, taskId),
|
|
1741
|
-
splitTask: (r, parentId, children) => {
|
|
1742
|
-
return this.dagSplitTask(name, r, parentId, children);
|
|
1743
|
-
},
|
|
1744
|
-
mergeTask: (r, sourceId, targetId) => {
|
|
1745
|
-
return this.dagMergeTask(name, r, sourceId, targetId);
|
|
1746
|
-
},
|
|
1747
|
-
cancelTask: (r, taskId, reason) => {
|
|
1748
|
-
return this.dagCancelTask(name, r, taskId, reason);
|
|
1749
|
-
},
|
|
1750
|
-
blockTask: (r, taskId, untilIso, reason, every) => {
|
|
1751
|
-
return this.dagBlockTask(name, r, taskId, untilIso, reason, every);
|
|
1752
|
-
},
|
|
1753
|
-
planGraph: (r, specs) => {
|
|
1754
|
-
return this.dagPlanGraph(name, r, specs);
|
|
1755
|
-
},
|
|
1756
|
-
queryFn: this.opts.queryFn,
|
|
1757
|
-
// Runner resolution: explicit opts.runner > role `runtime` field >
|
|
1758
|
-
// org def `runtime` field > MONOMIND_RUNTIME env (opencode/kimicode) >
|
|
1759
|
-
// undefined (session.ts falls back to ClaudeAgentRunner via queryFn).
|
|
1760
|
-
// Leaving it undefined for the default path is what keeps
|
|
1761
|
-
// Claude/Antigravity orgs byte-for-byte unchanged. Session opts are
|
|
1762
|
-
// built per role here, so each role gets its own runner.
|
|
1763
|
-
runner: this.opts.runner ??
|
|
1764
|
-
resolveRoleRunner(role.runtime, def.runtime, role.provider?.kind, undefined, role.provider),
|
|
1765
|
-
// Lets respawnRole force-stop THIS specific incarnation (mid-run role
|
|
1766
|
-
// replacement's forced-stop step) without reaching into runAgentSession's
|
|
1767
|
-
// internals.
|
|
1768
|
-
externalAbort: abort,
|
|
1769
|
-
silentSessionMs: this.opts.silentSessionMs,
|
|
1770
|
-
};
|
|
1771
|
-
// Supervised session: transient crashes (provider blips, network) restart
|
|
1772
|
-
// with backoff; a crash with the mailbox already closed, or one that
|
|
1773
|
-
// exhausts the retry budget, is terminal. runAgentSession already emits a
|
|
1774
|
-
// 'status' event for the raw error; the terminal 'audit' event is for
|
|
1775
|
-
// dashboards/alerts that filter on actionable failures (not routine
|
|
1776
|
-
// status chatter) so a dead agent surfaces instead of a run that
|
|
1777
|
-
// silently never progresses.
|
|
1778
|
-
const BACKOFFS_MS = this.opts.crashBackoffsMs ?? [1000, 5000, 15000];
|
|
1779
|
-
const myGeneration = generation;
|
|
1780
|
-
const isStaleGeneration = () => (running.roleSlots.get(role.id)?.generation ?? 0) !== myGeneration;
|
|
1781
|
-
if (!mailbox.isClosed && runtime.status !== 'crashed') {
|
|
1782
|
-
runtime.done = (async () => {
|
|
1783
|
-
for (let attempt = 0;; attempt++) {
|
|
1784
|
-
try {
|
|
1785
|
-
await runAgentSession(sessionOpts);
|
|
1786
|
-
runtime.status = 'ended';
|
|
1787
|
-
return;
|
|
1788
|
-
}
|
|
1789
|
-
catch (err) {
|
|
1790
|
-
// A deliberate respawn (see respawnRole) bumps the slot's
|
|
1791
|
-
// generation and force-stops this incarnation via its
|
|
1792
|
-
// externalAbort - that abort makes runAgentSession reject here
|
|
1793
|
-
// exactly like a real crash would. Recognize supersession
|
|
1794
|
-
// FIRST: this generation's retry loop must never restart,
|
|
1795
|
-
// never run terminal crash handling, and never notify the
|
|
1796
|
-
// boss - the replacement (a new generation, spawned
|
|
1797
|
-
// separately) already owns this role id.
|
|
1798
|
-
if (isStaleGeneration())
|
|
1799
|
-
return;
|
|
1800
|
-
// Drop the crashed session's stale waker immediately: a push()
|
|
1801
|
-
// during the backoff window must queue for the NEXT session, not
|
|
1802
|
-
// wake the dead generator to swallow it.
|
|
1803
|
-
mailbox.detach();
|
|
1804
|
-
// #203: if the crashed session's mailbox generator was abandoned
|
|
1805
|
-
// mid-yield (message already shift()ed for it, turn never
|
|
1806
|
-
// finished), put that message back on the queue — otherwise the
|
|
1807
|
-
// replacement session's stream() finds an empty queue and parks
|
|
1808
|
-
// forever, since the "delivered" message is gone for good.
|
|
1809
|
-
mailbox.reclaimInFlight();
|
|
1810
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1811
|
-
const isTurnLimit = /Reached maximum number of turns|error_max_turns/i.test(message);
|
|
1812
|
-
// Bounded like every other recovery: attempt counts every pass
|
|
1813
|
-
// through this loop, so a role that keeps surfacing max-turns
|
|
1814
|
-
// errors here (session.ts already swallows the normal ones)
|
|
1815
|
-
// falls through to crash handling instead of looping forever.
|
|
1816
|
-
if (isTurnLimit && !mailbox.isClosed && attempt < BACKOFFS_MS.length) {
|
|
1817
|
-
sessionOpts.resumeSessionId = undefined;
|
|
1818
|
-
mailbox.push(`${Mailbox.CONTINUE_PREFIX} You reached the turn limit on your task. Continue your in-progress work from where you left off; if finished, end your turn.`);
|
|
1819
|
-
bus.emit({
|
|
1820
|
-
type: 'status',
|
|
1821
|
-
from: role.id,
|
|
1822
|
-
reason: 'turn-limit-recover',
|
|
1823
|
-
msg: `agent "${role.id}" hit turn limit error — continuing with fresh session`,
|
|
1824
|
-
});
|
|
1825
|
-
continue;
|
|
1826
|
-
}
|
|
1827
|
-
// Exit 143 = SIGTERM. If the mailbox is already closed, we
|
|
1828
|
-
// sent the signal ourselves during stop — not a crash.
|
|
1829
|
-
const killedByStop = mailbox.isClosed && /exit(?:ed)? with code 143/.test(message);
|
|
1830
|
-
// #251: the org's own stop/complete (finishStop) removes this run
|
|
1831
|
-
// from this.orgs, closes every mailbox and aborts every session —
|
|
1832
|
-
// an idle one then rejects with the abort ("Operation aborted",
|
|
1833
|
-
// "Claude Code process aborted by user"). That is a shutdown, not a
|
|
1834
|
-
// crash. A non-abort error surfacing during the stop, or a crash
|
|
1835
|
-
// already backing off when the stop landed, stays a crash.
|
|
1836
|
-
const abortedByStop = mailbox.isClosed &&
|
|
1837
|
-
this.orgs.get(name) !== running &&
|
|
1838
|
-
(err?.name === 'AbortError' ||
|
|
1839
|
-
/\baborted\b/i.test(message));
|
|
1840
|
-
const crash = () => {
|
|
1841
|
-
if (killedByStop) {
|
|
1842
|
-
runtime.status = 'ended';
|
|
1843
|
-
bus.emit({
|
|
1844
|
-
type: 'status',
|
|
1845
|
-
from: role.id,
|
|
1846
|
-
msg: `agent "${role.id}" terminated by stop (was still working when drain window expired)`,
|
|
1847
|
-
reason: 'terminated-by-stop',
|
|
1848
|
-
});
|
|
1849
|
-
return;
|
|
1850
|
-
}
|
|
1851
|
-
if (abortedByStop) {
|
|
1852
|
-
runtime.status = 'ended';
|
|
1853
|
-
// #304: `message` here is always an abort string (see abortedByStop
|
|
1854
|
-
// above) — either "Operation aborted" or the SDK's "Claude Code
|
|
1855
|
-
// process aborted by user". Echoing it made a planned stop read as
|
|
1856
|
-
// a human interruption, and made roles of the same run read
|
|
1857
|
-
// differently. Report WHY the org stopped instead; the raw string
|
|
1858
|
-
// stays in `data` for debugging.
|
|
1859
|
-
const why = running.closedBy === 'org-complete' ? 'org_complete' : 'stop requested';
|
|
1860
|
-
bus.emit({
|
|
1861
|
-
type: 'status',
|
|
1862
|
-
from: role.id,
|
|
1863
|
-
msg: `agent "${role.id}" stopped with the org (${why})`,
|
|
1864
|
-
reason: 'agent-stopped',
|
|
1865
|
-
data: { agentId: role.id, error: message },
|
|
1866
|
-
});
|
|
1867
|
-
return;
|
|
1868
|
-
}
|
|
1869
|
-
runtime.status = 'crashed';
|
|
1870
|
-
runtime.error = message;
|
|
1871
|
-
// Close the mailbox so deliver()/receiveRemote() report a real
|
|
1872
|
-
// error instead of pushing into a queue no session will read
|
|
1873
|
-
// (and returning a false "delivered" receipt to the sender).
|
|
1874
|
-
mailbox.close();
|
|
1875
|
-
const isContextLimit = OrgDaemon.CONTEXT_LIMIT_RE.test(message);
|
|
1876
|
-
bus.emit({
|
|
1877
|
-
type: 'audit',
|
|
1878
|
-
from: role.id,
|
|
1879
|
-
msg: `agent "${role.id}" crashed: ${message}`,
|
|
1880
|
-
reason: isContextLimit ? 'agent-context-limit' : 'agent-session-crash',
|
|
1881
|
-
data: {
|
|
1882
|
-
agentId: role.id,
|
|
1883
|
-
error: message,
|
|
1884
|
-
restarts: attempt,
|
|
1885
|
-
contextLimit: isContextLimit,
|
|
1886
|
-
},
|
|
1887
|
-
});
|
|
1888
|
-
if (role.id !== running.bossRoleId) {
|
|
1889
|
-
// #2/#3: a worker is gone for the rest of this run. Without this
|
|
1890
|
-
// notice the coordinator keeps messaging a corpse (observed: four
|
|
1891
|
-
// unanswered org_send calls to a developer that had crashed on a
|
|
1892
|
-
// context-window limit). Tell the boss to reassign — and if the
|
|
1893
|
-
// crash was a context overflow, tell it to chunk smaller, since
|
|
1894
|
-
// re-dispatching the same task verbatim fails the same way.
|
|
1895
|
-
const bossRt = running.agents.get(running.bossRoleId);
|
|
1896
|
-
if (bossRt && !bossRt.mailbox.isClosed) {
|
|
1897
|
-
const guidance = isContextLimit
|
|
1898
|
-
? ' This was a context-window overflow — re-dispatching the same task verbatim will fail identically. Break the work into smaller pieces (one file or section at a time) and do not paste large file contents in a single message.'
|
|
1899
|
-
: '';
|
|
1900
|
-
bossRt.mailbox.push(`[system] Worker "${role.id}" crashed and will not recover this run (${message}). It can no longer receive messages — stop messaging it. Reassign its outstanding work to another agent or take it on yourself.${guidance}`);
|
|
1901
|
-
bus.emit({
|
|
1902
|
-
type: 'audit',
|
|
1903
|
-
from: running.bossRoleId,
|
|
1904
|
-
reason: 'worker-crashed',
|
|
1905
|
-
msg: `worker "${role.id}" crashed (contextLimit=${isContextLimit}); coordinator notified to reassign`,
|
|
1906
|
-
});
|
|
1907
|
-
}
|
|
1908
|
-
}
|
|
1909
|
-
else {
|
|
1910
|
-
// #4: the coordinator itself died. Don't go silent and wait for a
|
|
1911
|
-
// human — attempt a bounded whole-org restart with fresh sessions
|
|
1912
|
-
// (which also sheds whatever bloated context caused the crash).
|
|
1913
|
-
this.scheduleBossRestart(name);
|
|
1914
|
-
}
|
|
1915
|
-
};
|
|
1916
|
-
// Fatal errors (provider auth/quota/billing — tagged with
|
|
1917
|
-
// err.fatal by the runner) can NEVER be fixed by a restart: the
|
|
1918
|
-
// same call fails identically or hangs. Skip the backoff loop
|
|
1919
|
-
// and go straight to terminal crash handling instead of burning
|
|
1920
|
-
// the retry budget and wall-clock on a guaranteed failure.
|
|
1921
|
-
const fatal = err?.fatal === true;
|
|
1922
|
-
if (fatal) {
|
|
1923
|
-
bus.emit({
|
|
1924
|
-
type: 'status',
|
|
1925
|
-
from: role.id,
|
|
1926
|
-
reason: 'agent-fatal',
|
|
1927
|
-
msg: `agent "${role.id}" hit a fatal (non-retryable) error — not restarting`,
|
|
1928
|
-
});
|
|
1929
|
-
crash();
|
|
1930
|
-
return;
|
|
1931
|
-
}
|
|
1932
|
-
if (mailbox.isClosed || attempt >= BACKOFFS_MS.length) {
|
|
1933
|
-
crash();
|
|
1934
|
-
return;
|
|
1935
|
-
}
|
|
1936
|
-
bus.emit({
|
|
1937
|
-
type: 'status',
|
|
1938
|
-
from: role.id,
|
|
1939
|
-
reason: 'agent-restart',
|
|
1940
|
-
msg: `agent "${role.id}" crashed (${message}) — restarting in ${BACKOFFS_MS[attempt]}ms (attempt ${attempt + 1}/${BACKOFFS_MS.length})`,
|
|
1941
|
-
});
|
|
1942
|
-
await new Promise((r) => {
|
|
1943
|
-
const t = setTimeout(r, BACKOFFS_MS[attempt]);
|
|
1944
|
-
t.unref?.();
|
|
1945
|
-
// Org stop (finishStop) aborts every active slot's controller —
|
|
1946
|
-
// without racing it here, this wait wouldn't notice for up to
|
|
1947
|
-
// BACKOFFS_MS[attempt] (default up to 15s), well past finishStop's
|
|
1948
|
-
// own bounded drain window. That let this loop's crash() —
|
|
1949
|
-
// and the bus.emit() it triggers — fire AFTER finishStop had
|
|
1950
|
-
// already declared the org stopped and returned, capable of
|
|
1951
|
-
// recreating files in a run directory a caller was already
|
|
1952
|
-
// deleting.
|
|
1953
|
-
if (abort.signal.aborted) {
|
|
1954
|
-
clearTimeout(t);
|
|
1955
|
-
r();
|
|
1956
|
-
return;
|
|
1957
|
-
}
|
|
1958
|
-
abort.signal.addEventListener('abort', () => {
|
|
1959
|
-
clearTimeout(t);
|
|
1960
|
-
r();
|
|
1961
|
-
}, { once: true });
|
|
1962
|
-
});
|
|
1963
|
-
if (isStaleGeneration())
|
|
1964
|
-
return; // superseded during the backoff wait
|
|
1965
|
-
if (mailbox.isClosed) {
|
|
1966
|
-
crash();
|
|
1967
|
-
return;
|
|
1968
|
-
} // org stopped during backoff — never recovered
|
|
1969
|
-
// #247: continue the crashed conversation (briefing, task context,
|
|
1970
|
-
// finished work) instead of starting cold. runAgentSession falls
|
|
1971
|
-
// back to one fresh session if this id can't be resumed.
|
|
1972
|
-
sessionOpts.resumeSessionId = runtime.sessionId;
|
|
1973
|
-
}
|
|
1974
|
-
}
|
|
1975
|
-
})();
|
|
1976
|
-
}
|
|
1977
|
-
return { runtime, abort };
|
|
187
|
+
return roleIncarnation.spawnRoleIncarnation(this, name, running, role, generation, opts);
|
|
1978
188
|
}
|
|
1979
189
|
/** org_respawn_role's daemon-owned implementation. See the design doc's
|
|
1980
190
|
* "Replacement algorithm" (13 steps) — this method's body follows those
|
|
1981
191
|
* steps in order, numbered in comments. */
|
|
1982
192
|
async respawnRole(name, callerId, rawInput) {
|
|
1983
|
-
|
|
1984
|
-
if (!running) {
|
|
1985
|
-
return {
|
|
1986
|
-
success: false,
|
|
1987
|
-
roleId: '',
|
|
1988
|
-
generation: 0,
|
|
1989
|
-
respawnCount: 0,
|
|
1990
|
-
respawnsRemaining: 0,
|
|
1991
|
-
error: `org "${name}" is not running`,
|
|
1992
|
-
};
|
|
1993
|
-
}
|
|
1994
|
-
// Step 1: authorize (defense in depth — buildOrgTools only ever wires
|
|
1995
|
-
// onRespawnRole for the selected coordinator, but re-check here too).
|
|
1996
|
-
if (callerId !== running.bossRoleId) {
|
|
1997
|
-
return {
|
|
1998
|
-
success: false,
|
|
1999
|
-
roleId: '',
|
|
2000
|
-
generation: 0,
|
|
2001
|
-
respawnCount: 0,
|
|
2002
|
-
respawnsRemaining: 0,
|
|
2003
|
-
error: 'only the selected coordinator may call org_respawn_role',
|
|
2004
|
-
};
|
|
2005
|
-
}
|
|
2006
|
-
if (this.stopping.has(name)) {
|
|
2007
|
-
return {
|
|
2008
|
-
success: false,
|
|
2009
|
-
roleId: '',
|
|
2010
|
-
generation: 0,
|
|
2011
|
-
respawnCount: 0,
|
|
2012
|
-
respawnsRemaining: 0,
|
|
2013
|
-
error: `org "${name}" is stopping`,
|
|
2014
|
-
};
|
|
2015
|
-
}
|
|
2016
|
-
const validated = validateRespawnInput(rawInput);
|
|
2017
|
-
if (!validated.ok) {
|
|
2018
|
-
return {
|
|
2019
|
-
success: false,
|
|
2020
|
-
roleId: '',
|
|
2021
|
-
generation: 0,
|
|
2022
|
-
respawnCount: 0,
|
|
2023
|
-
respawnsRemaining: 0,
|
|
2024
|
-
error: validated.error,
|
|
2025
|
-
};
|
|
2026
|
-
}
|
|
2027
|
-
const input = validated.value;
|
|
2028
|
-
if (input.roleId === running.bossRoleId) {
|
|
2029
|
-
return {
|
|
2030
|
-
success: false,
|
|
2031
|
-
roleId: input.roleId,
|
|
2032
|
-
generation: 0,
|
|
2033
|
-
respawnCount: 0,
|
|
2034
|
-
respawnsRemaining: 0,
|
|
2035
|
-
error: 'cannot replace the selected coordinator',
|
|
2036
|
-
};
|
|
2037
|
-
}
|
|
2038
|
-
const slot = running.roleSlots.get(input.roleId);
|
|
2039
|
-
if (!slot) {
|
|
2040
|
-
return {
|
|
2041
|
-
success: false,
|
|
2042
|
-
roleId: input.roleId,
|
|
2043
|
-
generation: 0,
|
|
2044
|
-
respawnCount: 0,
|
|
2045
|
-
respawnsRemaining: 0,
|
|
2046
|
-
error: `unknown or not-yet-started role "${input.roleId}"`,
|
|
2047
|
-
};
|
|
2048
|
-
}
|
|
2049
|
-
const maxRespawns = running.def.run_config.max_role_respawns ?? 0;
|
|
2050
|
-
if (slot.phase === 'removed') {
|
|
2051
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2052
|
-
roleId: input.roleId,
|
|
2053
|
-
error: `role "${input.roleId}" was removed from this org`,
|
|
2054
|
-
});
|
|
2055
|
-
}
|
|
2056
|
-
// Step 2: acquire the role slot (reject a concurrent replacement).
|
|
2057
|
-
if (running.respawning.has(input.roleId) || slot.respawnPromise) {
|
|
2058
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2059
|
-
roleId: input.roleId,
|
|
2060
|
-
error: `role "${input.roleId}" is already undergoing replacement`,
|
|
2061
|
-
});
|
|
2062
|
-
}
|
|
2063
|
-
if (slot.respawnCount >= maxRespawns) {
|
|
2064
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2065
|
-
roleId: input.roleId,
|
|
2066
|
-
error: `role "${input.roleId}" has reached its respawn limit (${slot.respawnCount}/${maxRespawns}) for this run`,
|
|
2067
|
-
});
|
|
2068
|
-
}
|
|
2069
|
-
// Step 3: resolve the candidate configuration.
|
|
2070
|
-
if (input.providerName !== undefined && slot.effectiveRole.provider) {
|
|
2071
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2072
|
-
roleId: input.roleId,
|
|
2073
|
-
error: `role "${input.roleId}" has an inline provider, which always takes precedence over adapter_config.provider — replacing an inline provider is a separate design`,
|
|
2074
|
-
});
|
|
2075
|
-
}
|
|
2076
|
-
const candidateRole = mergeEffectiveRoleConfig(slot.effectiveRole, {
|
|
2077
|
-
runtime: input.runtime,
|
|
2078
|
-
model: input.model,
|
|
2079
|
-
providerName: input.providerName,
|
|
2080
|
-
});
|
|
2081
|
-
const budgetTokens = input.budgetTokens ?? computeReplacementBudget(running.def, input.roleId);
|
|
2082
|
-
// Step 4: preflight — must not mutate the old runtime.
|
|
2083
|
-
try {
|
|
2084
|
-
resolveRoleProvider(candidateRole, this.root);
|
|
2085
|
-
}
|
|
2086
|
-
catch (err) {
|
|
2087
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2088
|
-
roleId: input.roleId,
|
|
2089
|
-
error: `preflight failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
2090
|
-
});
|
|
2091
|
-
}
|
|
2092
|
-
// resolveRoleRunner's undefined return is the valid Claude default, not
|
|
2093
|
-
// an error — nothing further to validate for the runtime dimension here.
|
|
2094
|
-
resolveRoleRunner(candidateRole.runtime, running.def.runtime, candidateRole.provider?.kind, undefined, candidateRole.provider);
|
|
2095
|
-
// Step 5: consume one attempt — only after validation/preflight succeed.
|
|
2096
|
-
running.respawning.add(input.roleId);
|
|
2097
|
-
slot.respawnCount++;
|
|
2098
|
-
running.bus.emit({
|
|
2099
|
-
type: 'audit',
|
|
2100
|
-
from: callerId,
|
|
2101
|
-
reason: 'role-respawn-started',
|
|
2102
|
-
msg: `replacing role "${input.roleId}": ${input.reason}`,
|
|
2103
|
-
data: {
|
|
2104
|
-
roleId: input.roleId,
|
|
2105
|
-
from: redactRoleConfig(slot.effectiveRole),
|
|
2106
|
-
to: redactRoleConfig(candidateRole),
|
|
2107
|
-
generation: slot.generation,
|
|
2108
|
-
caller: callerId,
|
|
2109
|
-
},
|
|
2110
|
-
});
|
|
2111
|
-
// Step 6: quiesce the old incarnation. Bump the generation NOW, before
|
|
2112
|
-
// draining starts — not at the final publish (step 11-13) — so the OLD
|
|
2113
|
-
// generation's crash-retry loop (spawnRoleIncarnation's isStaleGeneration
|
|
2114
|
-
// check) recognizes supersession immediately. Without this, a backoff
|
|
2115
|
-
// timer firing during the drain/force-stop window, or the forced abort's
|
|
2116
|
-
// own rejection, would still see itself as the current generation:
|
|
2117
|
-
// the abort's rejection doesn't match killedByStop's SIGTERM-only regex,
|
|
2118
|
-
// so it would run full terminal crash handling — a duplicate live runner
|
|
2119
|
-
// (mid-backoff restart) or a false worker-crashed notification, exactly
|
|
2120
|
-
// what the guard exists to prevent.
|
|
2121
|
-
const newGeneration = slot.generation + 1;
|
|
2122
|
-
slot.generation = newGeneration;
|
|
2123
|
-
slot.phase = 'draining';
|
|
2124
|
-
// Every await from here on can race a stop/restart of this org — verify
|
|
2125
|
-
// ownership before EVERY subsequent step, not just once before the final
|
|
2126
|
-
// publish, so a stale operation can never mutate accounting, force-stop
|
|
2127
|
-
// a runtime, or spawn into an org that's no longer the live one.
|
|
2128
|
-
const stillOwned = () => this.orgs.get(name) === running && running.roleSlots.get(input.roleId) === slot;
|
|
2129
|
-
const abandonedReceipt = () => {
|
|
2130
|
-
running.respawning.delete(input.roleId);
|
|
2131
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2132
|
-
roleId: input.roleId,
|
|
2133
|
-
error: `org "${name}" stopped or restarted during replacement`,
|
|
2134
|
-
});
|
|
2135
|
-
};
|
|
2136
|
-
const oldRuntime = slot.runtime;
|
|
2137
|
-
const sweptQueue = oldRuntime.mailbox.beginDrain();
|
|
2138
|
-
slot.queuedDuringSwap.push(...sweptQueue);
|
|
2139
|
-
const drainTimeoutMs = running.def.run_config.respawn_drain_timeout_ms ?? 30_000;
|
|
2140
|
-
const drained = await Promise.race([
|
|
2141
|
-
oldRuntime.done.then(() => true),
|
|
2142
|
-
new Promise((r) => setTimeout(() => r(false), drainTimeoutMs)),
|
|
2143
|
-
]);
|
|
2144
|
-
if (!stillOwned())
|
|
2145
|
-
return abandonedReceipt();
|
|
2146
|
-
let drainTimedOut = false;
|
|
2147
|
-
if (!drained) {
|
|
2148
|
-
drainTimedOut = true;
|
|
2149
|
-
// Step 7: force stop.
|
|
2150
|
-
slot.abort?.abort();
|
|
2151
|
-
const forceStopMs = running.def.run_config.respawn_force_stop_timeout_ms ?? 5_000;
|
|
2152
|
-
const stopped = await Promise.race([
|
|
2153
|
-
oldRuntime.done.then(() => true).catch(() => true),
|
|
2154
|
-
new Promise((r) => setTimeout(() => r(false), forceStopMs)),
|
|
2155
|
-
]);
|
|
2156
|
-
if (!stillOwned())
|
|
2157
|
-
return abandonedReceipt();
|
|
2158
|
-
if (!stopped) {
|
|
2159
|
-
slot.phase = 'stuck';
|
|
2160
|
-
running.respawning.delete(input.roleId);
|
|
2161
|
-
running.bus.emit({
|
|
2162
|
-
type: 'audit',
|
|
2163
|
-
from: callerId,
|
|
2164
|
-
reason: 'role-respawn-failed',
|
|
2165
|
-
msg: `role "${input.roleId}" forced stop did not confirm termination — refusing to spawn a replacement`,
|
|
2166
|
-
});
|
|
2167
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2168
|
-
roleId: input.roleId,
|
|
2169
|
-
error: `role "${input.roleId}" could not be confirmed stopped; not replaced`,
|
|
2170
|
-
});
|
|
2171
|
-
}
|
|
2172
|
-
}
|
|
2173
|
-
// Step 8: preserve durable role state (worktree path, task ownership, and
|
|
2174
|
-
// the DAG survive untouched — they live outside AgentRuntime/Mailbox
|
|
2175
|
-
// entirely, keyed by role.id, which never changes). Reclaim any message
|
|
2176
|
-
// abandoned mid-yield by a forced stop for at-least-once redelivery.
|
|
2177
|
-
oldRuntime.mailbox.reclaimInFlight();
|
|
2178
|
-
const reclaimedQueue = oldRuntime.mailbox.serialize().queue;
|
|
2179
|
-
slot.queuedDuringSwap.push(...reclaimedQueue);
|
|
2180
|
-
// Step 9: retire accounting BEFORE replacing the runtime.
|
|
2181
|
-
slot.retiredUsage = {
|
|
2182
|
-
// Budgeted basis: this total is summed with live policy.budgetedUsage
|
|
2183
|
-
// against the org-wide budget_tokens ceiling (ADR-O001 D1), so the two
|
|
2184
|
-
// terms must share a basis.
|
|
2185
|
-
tokens: slot.retiredUsage.tokens + oldRuntime.policy.budgetedUsage,
|
|
2186
|
-
costUsd: slot.retiredUsage.costUsd + oldRuntime.metrics.costUsd,
|
|
2187
|
-
};
|
|
2188
|
-
// Step 10: spawn generation N+1 (generation already bumped in step 6).
|
|
2189
|
-
const { runtime: newRuntime, abort: newAbort } = this.spawnRoleIncarnation(name, running, candidateRole, newGeneration, { budgetTokensOverride: budgetTokens });
|
|
2190
|
-
// Seed the new mailbox with everything swapped/reclaimed, delivered
|
|
2191
|
-
// FIFO, plus a delimited coordinator briefing appended last so it reads
|
|
2192
|
-
// as the newest context once the replacement starts its first turn.
|
|
2193
|
-
for (const queued of slot.queuedDuringSwap)
|
|
2194
|
-
newRuntime.mailbox.push(queued);
|
|
2195
|
-
newRuntime.mailbox.push(`[system: role replacement briefing — not a system prompt] You are a fresh session replacing the previous incarnation of role "${input.roleId}". Reason: ${input.reason}\n\n${input.briefing}`);
|
|
2196
|
-
// Seed USD accounting from retained totals so a respawn cannot reset
|
|
2197
|
-
// role.budget_usd.
|
|
2198
|
-
if (running.def.roles.find((r) => r.id === input.roleId)?.budget_usd !== undefined) {
|
|
2199
|
-
newRuntime.policy.setUsageUsd(slot.retiredUsage.costUsd);
|
|
2200
|
-
}
|
|
2201
|
-
// "Ready" here means "did not crash within the startup window" — a
|
|
2202
|
-
// silent-but-healthy runner (one that never emits a chat/tool/usage
|
|
2203
|
-
// event, e.g. because it hasn't finished its first turn yet) must not be
|
|
2204
|
-
// misreported as a startup failure, so this does NOT wait for a positive
|
|
2205
|
-
// signal. It races the new incarnation's own crash-retry loop (which
|
|
2206
|
-
// shares this generation, so it is NOT superseded and behaves normally)
|
|
2207
|
-
// against the timeout: a config that fails immediately (bad model,
|
|
2208
|
-
// missing runtime binary, auth failure) crashes fast and newRuntime.done
|
|
2209
|
-
// resolves with status 'crashed' well before startTimeoutMs, correctly
|
|
2210
|
-
// failing readiness and triggering rollback.
|
|
2211
|
-
const startTimeoutMs = running.def.run_config.respawn_start_timeout_ms ?? 60_000;
|
|
2212
|
-
const ready = await Promise.race([
|
|
2213
|
-
newRuntime.done.then(() => newRuntime.status !== 'crashed'),
|
|
2214
|
-
new Promise((r) => setTimeout(() => r(true), startTimeoutMs)),
|
|
2215
|
-
]);
|
|
2216
|
-
// Step 11: publish atomically — verify ownership is still current.
|
|
2217
|
-
if (!stillOwned()) {
|
|
2218
|
-
newAbort.abort();
|
|
2219
|
-
return abandonedReceipt();
|
|
2220
|
-
}
|
|
2221
|
-
if (!ready) {
|
|
2222
|
-
// Step 12: rollback — one attempt with the prior effective config.
|
|
2223
|
-
newAbort.abort();
|
|
2224
|
-
running.bus.emit({
|
|
2225
|
-
type: 'audit',
|
|
2226
|
-
from: callerId,
|
|
2227
|
-
reason: 'role-respawn-failed',
|
|
2228
|
-
msg: `role "${input.roleId}" replacement did not become ready within ${startTimeoutMs}ms — attempting rollback`,
|
|
2229
|
-
});
|
|
2230
|
-
try {
|
|
2231
|
-
const { runtime: rolledBack, abort: rolledBackAbort } = this.spawnRoleIncarnation(name, running, slot.effectiveRole, newGeneration + 1, {});
|
|
2232
|
-
for (const queued of slot.queuedDuringSwap)
|
|
2233
|
-
rolledBack.mailbox.push(queued);
|
|
2234
|
-
running.agents.set(input.roleId, rolledBack);
|
|
2235
|
-
slot.runtime = rolledBack;
|
|
2236
|
-
slot.abort = rolledBackAbort;
|
|
2237
|
-
slot.generation = newGeneration + 1;
|
|
2238
|
-
slot.phase = 'running';
|
|
2239
|
-
slot.queuedDuringSwap = [];
|
|
2240
|
-
running.respawning.delete(input.roleId);
|
|
2241
|
-
running.bus.emit({
|
|
2242
|
-
type: 'audit',
|
|
2243
|
-
from: callerId,
|
|
2244
|
-
reason: 'role-respawn-failed',
|
|
2245
|
-
msg: `role "${input.roleId}" replacement failed; rolled back to prior config`,
|
|
2246
|
-
});
|
|
2247
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2248
|
-
roleId: input.roleId,
|
|
2249
|
-
drainTimedOut,
|
|
2250
|
-
error: `replacement failed to start; rolled back to prior configuration`,
|
|
2251
|
-
});
|
|
2252
|
-
}
|
|
2253
|
-
catch (rollbackErr) {
|
|
2254
|
-
slot.phase = 'crashed';
|
|
2255
|
-
running.respawning.delete(input.roleId);
|
|
2256
|
-
running.bus.emit({
|
|
2257
|
-
type: 'audit',
|
|
2258
|
-
from: callerId,
|
|
2259
|
-
reason: 'role-respawn-rollback-failed',
|
|
2260
|
-
msg: `role "${input.roleId}" replacement AND rollback both failed: ${rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)}`,
|
|
2261
|
-
});
|
|
2262
|
-
return buildRespawnReceipt(slot, maxRespawns, false, {
|
|
2263
|
-
roleId: input.roleId,
|
|
2264
|
-
drainTimedOut,
|
|
2265
|
-
error: `replacement and rollback both failed; role "${input.roleId}" is unavailable`,
|
|
2266
|
-
});
|
|
2267
|
-
}
|
|
2268
|
-
}
|
|
2269
|
-
running.agents.set(input.roleId, newRuntime);
|
|
2270
|
-
slot.runtime = newRuntime;
|
|
2271
|
-
slot.abort = newAbort;
|
|
2272
|
-
slot.generation = newGeneration;
|
|
2273
|
-
slot.effectiveRole = candidateRole;
|
|
2274
|
-
slot.phase = 'running';
|
|
2275
|
-
slot.queuedDuringSwap = [];
|
|
2276
|
-
running.respawning.delete(input.roleId);
|
|
2277
|
-
// Step 13: audit and persist.
|
|
2278
|
-
running.bus.emit({
|
|
2279
|
-
type: 'audit',
|
|
2280
|
-
from: callerId,
|
|
2281
|
-
reason: 'role-respawned',
|
|
2282
|
-
msg: `role "${input.roleId}" replaced (generation ${newGeneration})`,
|
|
2283
|
-
data: {
|
|
2284
|
-
roleId: input.roleId,
|
|
2285
|
-
generation: newGeneration,
|
|
2286
|
-
respawnCount: slot.respawnCount,
|
|
2287
|
-
drainTimedOut,
|
|
2288
|
-
},
|
|
2289
|
-
});
|
|
2290
|
-
this.persistState(name, 'running', running.run);
|
|
2291
|
-
return buildRespawnReceipt(slot, maxRespawns, true, { roleId: input.roleId, drainTimedOut });
|
|
193
|
+
return roleRespawn.respawnRole(this, name, callerId, rawInput);
|
|
2292
194
|
}
|
|
2293
195
|
/** @internal */
|
|
2294
196
|
hasOrgDef(name) {
|
|
@@ -2305,325 +207,7 @@ export class OrgDaemon {
|
|
|
2305
207
|
* watchdog, boss-restart-exhausted, manual `org stop`) and exit non-zero
|
|
2306
208
|
* for the latter. Only the org_complete auto-stop path passes this. */
|
|
2307
209
|
async stopOrg(name, opts) {
|
|
2308
|
-
|
|
2309
|
-
// (org_complete, idle watchdog) run detached, and a caller like
|
|
2310
|
-
// `org run`'s final stopAll() must not resolve — letting the process
|
|
2311
|
-
// exit — while that stop is still flushing the bus and writing
|
|
2312
|
-
// history/runtime.json.
|
|
2313
|
-
const inflight = this.stopping.get(name);
|
|
2314
|
-
if (inflight)
|
|
2315
|
-
return inflight;
|
|
2316
|
-
const org = this.orgs.get(name);
|
|
2317
|
-
if (!org)
|
|
2318
|
-
return; // already stopped
|
|
2319
|
-
org.pendingRoles?.clear(); // prevent lazy spawns after stop
|
|
2320
|
-
this.spawning.delete(name); // clean up spawning tracking for this org
|
|
2321
|
-
// #304: set before the delete below, since that delete is what makes
|
|
2322
|
-
// abortedByStop (role loop) true — the role loop reads it off this same
|
|
2323
|
-
// object reference, not a fresh lookup (the org is gone from the map by then).
|
|
2324
|
-
org.closedBy = opts?.closedBy;
|
|
2325
|
-
// Remove immediately (not at the end) so a concurrent stopOrg(name) call —
|
|
2326
|
-
// e.g. stopAll() racing a scheduler-triggered stop on SIGINT — joins this
|
|
2327
|
-
// shutdown via `stopping` instead of re-running the whole sequence and
|
|
2328
|
-
// double-emitting 'org stopped' (duplicate org:complete/session:complete).
|
|
2329
|
-
this.orgs.delete(name);
|
|
2330
|
-
const p = this.finishStop(name, org, opts?.drainMs, opts?.closedBy);
|
|
2331
|
-
this.stopping.set(name, p);
|
|
2332
|
-
try {
|
|
2333
|
-
await p;
|
|
2334
|
-
}
|
|
2335
|
-
finally {
|
|
2336
|
-
this.stopping.delete(name);
|
|
2337
|
-
}
|
|
2338
|
-
}
|
|
2339
|
-
async finishStop(name, org, drainMs, closedBy) {
|
|
2340
|
-
// Process- and daemon-level handles come off FIRST, before anything that
|
|
2341
|
-
// can throw. These used to be removed after captureCheckpoint(), so a
|
|
2342
|
-
// throw there — which a half-started org can provoke, since it may be
|
|
2343
|
-
// missing state a checkpoint expects — aborted the whole stop and left a
|
|
2344
|
-
// process 'exit' listener, an interval and a broker lease behind for a run
|
|
2345
|
-
// that no longer exists. startOrg()'s teardown-on-failure path swallows a
|
|
2346
|
-
// rejecting stopOrg (it has its own error to report), so the leak was
|
|
2347
|
-
// silent.
|
|
2348
|
-
const cleanup = org._crashCleanup;
|
|
2349
|
-
if (cleanup)
|
|
2350
|
-
process.removeListener('exit', cleanup);
|
|
2351
|
-
const wd = this.watchdogs.get(name);
|
|
2352
|
-
if (wd) {
|
|
2353
|
-
clearInterval(wd);
|
|
2354
|
-
this.watchdogs.delete(name);
|
|
2355
|
-
}
|
|
2356
|
-
clearIdleRecord(this.root, name);
|
|
2357
|
-
// The run's gates are authoritative; put them back over whatever the file
|
|
2358
|
-
// holds now (a role may have rewritten it).
|
|
2359
|
-
if (org.gates) {
|
|
2360
|
-
try {
|
|
2361
|
-
decisionOps.writeGates(this.root, name, org.gates);
|
|
2362
|
-
}
|
|
2363
|
-
catch {
|
|
2364
|
-
/* the next start reads the last write-through */
|
|
2365
|
-
}
|
|
2366
|
-
}
|
|
2367
|
-
this.leases.get(name)?.stop();
|
|
2368
|
-
this.leases.delete(name);
|
|
2369
|
-
// Capture THIS run's forwarder now: an autoWake-restart of the same org
|
|
2370
|
-
// during the long tail below (agent wait, flush, history write) would
|
|
2371
|
-
// register a NEW forwarder under the same name — settling/unsubscribing
|
|
2372
|
-
// that one would sever the new run's dashboard stream.
|
|
2373
|
-
const forwarder = this.forwarders.get(name);
|
|
2374
|
-
// Snapshot checkpoint BEFORE closing mailboxes / draining sessions — the
|
|
2375
|
-
// queue is emptied during the drain, so capturing afterwards loses all
|
|
2376
|
-
// unconsumed messages (the whole point of checkpoint-resume). Best-effort:
|
|
2377
|
-
// a run that cannot be checkpointed must still be stopped and cleaned up.
|
|
2378
|
-
let stopCheckpoint;
|
|
2379
|
-
try {
|
|
2380
|
-
stopCheckpoint = captureCheckpoint(org, 'stopped');
|
|
2381
|
-
}
|
|
2382
|
-
catch (err) {
|
|
2383
|
-
console.error(`org ${name}: could not capture the stop checkpoint:`, err instanceof Error ? err.message : err);
|
|
2384
|
-
}
|
|
2385
|
-
// #275: drop task dispatches still inside their coalescing window — the
|
|
2386
|
-
// mailboxes they target are closed on the next line anyway.
|
|
2387
|
-
for (const held of org.pendingDispatch?.values() ?? [])
|
|
2388
|
-
clearTimeout(held.timer);
|
|
2389
|
-
org.pendingDispatch?.clear();
|
|
2390
|
-
for (const a of org.agents.values())
|
|
2391
|
-
a.mailbox.close();
|
|
2392
|
-
// Closing the mailbox stops new work being handed to a session, but does
|
|
2393
|
-
// NOT cancel a turn already in flight (e.g. mid provider call) — that
|
|
2394
|
-
// session can keep running, and eventually crash/finish, well past this
|
|
2395
|
-
// function's own bounded drain below. Abort each slot's live incarnation
|
|
2396
|
-
// too, reusing respawnRole's existing force-stop handle, so in-flight
|
|
2397
|
-
// work is told to stop now instead of merely being denied new input.
|
|
2398
|
-
for (const slot of org.roleSlots.values())
|
|
2399
|
-
slot.abort?.abort();
|
|
2400
|
-
// M1: kill every tool-provider process of this org's sessions.
|
|
2401
|
-
this.toolProviders.closeOrg(name);
|
|
2402
|
-
// M2: stop endpoint retry timers (queued entries stay queued).
|
|
2403
|
-
stopEndpointRetries(this, name);
|
|
2404
|
-
// Bounded: a genuinely hung agent session (stuck mid-tool-call, not just
|
|
2405
|
-
// idle) must not make stopOrg() hang forever — callers like the scheduler
|
|
2406
|
-
// already race their own timeout around a run, and this wait re-blocking
|
|
2407
|
-
// unboundedly on the same never-resolving promises defeated that bound.
|
|
2408
|
-
// A planned completion is not an abort. The boss declaring the cycle done
|
|
2409
|
-
// says nothing about its siblings: they are routinely mid-build or mid-edit
|
|
2410
|
-
// when it fires, and a 15s window SIGTERM'd them (exit 143, reported as
|
|
2411
|
-
// "crashed") and threw the work away. allSettled resolves as soon as every
|
|
2412
|
-
// session ends, so a long drain is a ceiling, not a delay.
|
|
2413
|
-
const stopWaitMs = drainMs ?? this.opts.stopWaitMs ?? 15_000;
|
|
2414
|
-
const allDone = Promise.allSettled([...org.agents.values()].map((a) => a.done)).then(() => false);
|
|
2415
|
-
// Clear the ceiling timer once the sessions win the race: left pending, a
|
|
2416
|
-
// COMPLETE_DRAIN_MS stop kept `org run` (which returns without
|
|
2417
|
-
// process.exit on a clean completion) alive for up to five minutes after
|
|
2418
|
-
// every session had already ended. Deliberately NOT unref'd — on the
|
|
2419
|
-
// timed-out path this timer may be the only thing keeping the loop alive
|
|
2420
|
-
// long enough to write 'stopped' to runtime.json and flush the bus.
|
|
2421
|
-
let drainTimer;
|
|
2422
|
-
const timedOut = await Promise.race([
|
|
2423
|
-
allDone,
|
|
2424
|
-
new Promise((r) => {
|
|
2425
|
-
drainTimer = setTimeout(() => r(true), stopWaitMs);
|
|
2426
|
-
}),
|
|
2427
|
-
]);
|
|
2428
|
-
clearTimeout(drainTimer);
|
|
2429
|
-
if (timedOut) {
|
|
2430
|
-
// #152: "proceeding anyway" alone didn't say WHO got cut off — a run
|
|
2431
|
-
// reviewer had no way to tell whether real, in-progress work (a
|
|
2432
|
-
// mid-build, a mid-write) was force-stopped, or the drain window
|
|
2433
|
-
// simply outlived a handful of already-idle sessions. status is only
|
|
2434
|
-
// 'ended'/'crashed' once a role's session promise has actually
|
|
2435
|
-
// settled; still 'running' here means it was mid-turn when the
|
|
2436
|
-
// ceiling hit, not merely idle-but-not-yet-reaped.
|
|
2437
|
-
const stillActive = [...org.agents.entries()]
|
|
2438
|
-
.filter(([, a]) => a.status === 'running')
|
|
2439
|
-
.map(([roleId]) => roleId);
|
|
2440
|
-
const rosterSuffix = stillActive.length ? ` — still active: ${stillActive.join(', ')}` : '';
|
|
2441
|
-
org.bus.emit({
|
|
2442
|
-
type: 'audit',
|
|
2443
|
-
msg: `org stop timed out after ${stopWaitMs}ms waiting for agent sessions to finish — proceeding anyway${rosterSuffix}`,
|
|
2444
|
-
reason: 'stop-timeout',
|
|
2445
|
-
data: { stillActive },
|
|
2446
|
-
});
|
|
2447
|
-
// Reap only SDK processes spawned by THIS node process — ownerPid filter
|
|
2448
|
-
// ensures other `monomind org run` daemons' agents are untouched.
|
|
2449
|
-
try {
|
|
2450
|
-
const reaped = reapOrphanedSdkProcesses(new Set(), process.pid);
|
|
2451
|
-
if (reaped > 0)
|
|
2452
|
-
org.bus.emit({
|
|
2453
|
-
type: 'audit',
|
|
2454
|
-
reason: 'orphan-reap',
|
|
2455
|
-
msg: `reaped ${reaped} orphaned SDK process(es) after stop timeout`,
|
|
2456
|
-
});
|
|
2457
|
-
}
|
|
2458
|
-
catch {
|
|
2459
|
-
/* best-effort */
|
|
2460
|
-
}
|
|
2461
|
-
}
|
|
2462
|
-
// The run's sessions are gone: take down the sandbox stubs it held.
|
|
2463
|
-
sandboxStubs.release(`${name}:${org.run}`);
|
|
2464
|
-
// #302 truth gate: every stop path funnels through here, so this is the
|
|
2465
|
-
// one place that can record how the run ACTUALLY ended, regardless of
|
|
2466
|
-
// which of the five paths triggered it. `closedBy` is undefined only for
|
|
2467
|
-
// a bare manual `stopOrg(name)` (CLI `org stop`, shutdown) — every
|
|
2468
|
-
// automated path above now tags its own real cause. reporting.ts reads
|
|
2469
|
-
// this event (reason: 'org-stopped') to decide whether the run's outcome
|
|
2470
|
-
// may be rendered as a boss-attributed 'partial'/'achieved' at all: only
|
|
2471
|
-
// closedBy === 'org-complete' may be.
|
|
2472
|
-
const runnableTasks = org.taskDag?.pendingTaskCount() ?? 0;
|
|
2473
|
-
// Rendered, not just recorded (#302 AC6, same reasoning as the
|
|
2474
|
-
// blockerSuffix above): `org logs` prints `msg` verbatim.
|
|
2475
|
-
const stopSuffix = closedBy && closedBy !== 'org-complete'
|
|
2476
|
-
? ` (${closedBy}${runnableTasks > 0 ? `, ${runnableTasks} task(s) left` : ''})`
|
|
2477
|
-
: '';
|
|
2478
|
-
org.bus.emit({
|
|
2479
|
-
type: 'status',
|
|
2480
|
-
reason: 'org-stopped',
|
|
2481
|
-
msg: `org stopped${stopSuffix}`,
|
|
2482
|
-
data: { closedBy, runnableTasks },
|
|
2483
|
-
});
|
|
2484
|
-
await org.bus.flush();
|
|
2485
|
-
// Append this run's summary to <org>/history.jsonl — read back from the
|
|
2486
|
-
// flushed bus.jsonl (the full durable record) rather than the bounded
|
|
2487
|
-
// in-memory buffer, so long runs summarize completely.
|
|
2488
|
-
//
|
|
2489
|
-
// This block runs BEFORE the seal below (#293): storeRunMemory emits an
|
|
2490
|
-
// audit event when the run's memory could not be stored, and a sealed bus
|
|
2491
|
-
// fans out to in-memory listeners without ever reaching bus.jsonl — an
|
|
2492
|
-
// event the live view shows and the durable record does not, which is both
|
|
2493
|
-
// the divergence test-loop's `persisted` check exists to catch and useless
|
|
2494
|
-
// to whoever reads the run back later. Sealing after it keeps every emitted
|
|
2495
|
-
// event durable. The seal still closes before this function returns, which
|
|
2496
|
-
// is what its own contract (below) is about.
|
|
2497
|
-
try {
|
|
2498
|
-
const events = readRunEvents(this.root, name, org.run);
|
|
2499
|
-
if (events.length) {
|
|
2500
|
-
const summary = summarizeRun(events);
|
|
2501
|
-
const { appendFileSync } = await import('node:fs');
|
|
2502
|
-
appendFileSync(historyFile(this.root, name), `${JSON.stringify(summary)}\n`, 'utf8');
|
|
2503
|
-
// Cross-run memory: make this run's outcome recallable by meaning.
|
|
2504
|
-
// #293: the result is CHECKED — a store that silently did nothing used
|
|
2505
|
-
// to be indistinguishable from one that worked, and the symptom
|
|
2506
|
-
// (org_recall always empty) showed up runs later with no trail. The
|
|
2507
|
-
// reason is stashed for persistState() below so runtime.json — and
|
|
2508
|
-
// therefore `org status` — carries it after the bus event and the
|
|
2509
|
-
// stderr warning have scrolled away.
|
|
2510
|
-
const memory = await this.storeRunMemory(name, org.def, org.run, summary, org.bus);
|
|
2511
|
-
if (memory.stored)
|
|
2512
|
-
this.memoryErrors.delete(name);
|
|
2513
|
-
else
|
|
2514
|
-
this.memoryErrors.set(name, memory.reason ?? 'unknown');
|
|
2515
|
-
}
|
|
2516
|
-
}
|
|
2517
|
-
catch (err) {
|
|
2518
|
-
console.error(`org ${name}: could not write run history:`, err instanceof Error ? err.message : err);
|
|
2519
|
-
}
|
|
2520
|
-
finally {
|
|
2521
|
-
this.recallUsage.delete(name);
|
|
2522
|
-
this.orgLearnedRuns.delete(`${name}:${org.run}`);
|
|
2523
|
-
}
|
|
2524
|
-
// flush() only awaits a snapshot of writes queued at call time (see its
|
|
2525
|
-
// own doc comment) — it has no visibility into a session that crashes
|
|
2526
|
-
// after the abort signal above but before this function returns. Seal
|
|
2527
|
-
// the bus now so any such late bus.emit() still reaches in-memory
|
|
2528
|
-
// listeners but can never schedule a new disk write into a run
|
|
2529
|
-
// directory a caller (e.g. a test's afterEach) may already be deleting.
|
|
2530
|
-
// seal() awaits the pending writes first, so the audit event the block
|
|
2531
|
-
// above may have emitted is on disk before the bus closes.
|
|
2532
|
-
await org.bus.seal();
|
|
2533
|
-
// the "org stopped" event above triggers the forwarder's final org:complete /
|
|
2534
|
-
// session:complete POST — without waiting for it here, the CLI process can exit
|
|
2535
|
-
// (and kill the in-flight fetch) before that last event reaches the dashboard,
|
|
2536
|
-
// leaving the run stuck showing "running" forever. Bounded: a stalled
|
|
2537
|
-
// dashboard must not hang org shutdown indefinitely.
|
|
2538
|
-
if (forwarder) {
|
|
2539
|
-
await Promise.race([
|
|
2540
|
-
forwarder.settle(),
|
|
2541
|
-
new Promise((r) => {
|
|
2542
|
-
const t = setTimeout(r, 5_000);
|
|
2543
|
-
t.unref?.();
|
|
2544
|
-
}),
|
|
2545
|
-
]);
|
|
2546
|
-
forwarder.unsubscribe();
|
|
2547
|
-
// Only remove from the map if it's still OURS — an autoWake-restart may
|
|
2548
|
-
// have registered the new run's forwarder under this name meanwhile.
|
|
2549
|
-
if (this.forwarders.get(name) === forwarder)
|
|
2550
|
-
this.forwarders.delete(name);
|
|
2551
|
-
}
|
|
2552
|
-
// Same guard for runtime.json: if a new run started during shutdown, its
|
|
2553
|
-
// 'running' record must not be overwritten with this old run's 'stopped'.
|
|
2554
|
-
// Pass the org directly since we already removed it from the map.
|
|
2555
|
-
if (!this.orgs.has(name))
|
|
2556
|
-
this.persistState(name, 'stopped', org.run, org, stopCheckpoint, closedBy);
|
|
2557
|
-
// Clean up git worktrees — shared (workspace: 'worktree') and per-role.
|
|
2558
|
-
try {
|
|
2559
|
-
const { execFileSync } = await import('node:child_process');
|
|
2560
|
-
if (org.worktreePath) {
|
|
2561
|
-
try {
|
|
2562
|
-
execFileSync('git', ['worktree', 'remove', '--force', org.worktreePath], {
|
|
2563
|
-
cwd: this.root,
|
|
2564
|
-
stdio: 'ignore',
|
|
2565
|
-
timeout: 30_000,
|
|
2566
|
-
});
|
|
2567
|
-
}
|
|
2568
|
-
catch {
|
|
2569
|
-
/* best-effort */
|
|
2570
|
-
}
|
|
2571
|
-
}
|
|
2572
|
-
for (const agent of org.agents.values()) {
|
|
2573
|
-
if (agent.worktreePath) {
|
|
2574
|
-
try {
|
|
2575
|
-
execFileSync('git', ['worktree', 'remove', '--force', agent.worktreePath], {
|
|
2576
|
-
cwd: this.root,
|
|
2577
|
-
stdio: 'ignore',
|
|
2578
|
-
timeout: 30_000,
|
|
2579
|
-
});
|
|
2580
|
-
}
|
|
2581
|
-
catch {
|
|
2582
|
-
/* best-effort */
|
|
2583
|
-
}
|
|
2584
|
-
}
|
|
2585
|
-
}
|
|
2586
|
-
// #301: roles create their own linked worktrees with Bash (paths the
|
|
2587
|
-
// daemon never recorded — org.worktreePath/agent.worktreePath above are
|
|
2588
|
-
// only ever set for workspace: 'worktree'/'worktree-per-role', empty
|
|
2589
|
-
// for the common workspace: 'repo' shape), and deleting the working
|
|
2590
|
-
// directory from inside a role sandbox leaves .git/worktrees/<name>
|
|
2591
|
-
// behind — `git worktree list` then hides it, and it never gets
|
|
2592
|
-
// cleaned up. Unconditional on purpose: gating this on
|
|
2593
|
-
// org/agent.worktreePath would skip exactly the runs that hit the bug.
|
|
2594
|
-
// prune only drops metadata whose worktree directory is already gone,
|
|
2595
|
-
// so a live worktree — including the owner's — is never touched; it is
|
|
2596
|
-
// idempotent; and the two removals just above already run `git
|
|
2597
|
-
// worktree remove --force` against this same repo from this same cwd,
|
|
2598
|
-
// so this is strictly less invasive than what already ships. Run after
|
|
2599
|
-
// both removal loops so a worktree just removed is also pruned.
|
|
2600
|
-
//
|
|
2601
|
-
// Bounded race, measured rather than assumed (same treatment as the
|
|
2602
|
-
// SIGKILL case above): an entry whose `gitdir` file is absent is
|
|
2603
|
-
// pruned unconditionally, and `--expire` cannot protect it — measured
|
|
2604
|
-
// across every window from `--expire=now` to `--expire=3.months.ago`,
|
|
2605
|
-
// a fresh no-gitdir entry is removed regardless, while `--expire` also
|
|
2606
|
-
// makes an already-deleted worktree SURVIVE, breaking the "a run
|
|
2607
|
-
// always begins clean" guarantee this fix exists to provide. So a
|
|
2608
|
-
// concurrent `git worktree add` by another process in this repo is
|
|
2609
|
-
// vulnerable for the microseconds between its `mkdir` and its
|
|
2610
|
-
// `gitdir` write. A mid-creation state cannot persist longer than
|
|
2611
|
-
// that, so an entry found in that state is dead metadata, not a live
|
|
2612
|
-
// worktree in progress.
|
|
2613
|
-
try {
|
|
2614
|
-
execFileSync('git', ['worktree', 'prune'], {
|
|
2615
|
-
cwd: this.root,
|
|
2616
|
-
stdio: 'ignore',
|
|
2617
|
-
timeout: 30_000,
|
|
2618
|
-
});
|
|
2619
|
-
}
|
|
2620
|
-
catch {
|
|
2621
|
-
/* best-effort: not a git repo, git missing, or a wedged hook */
|
|
2622
|
-
}
|
|
2623
|
-
}
|
|
2624
|
-
catch {
|
|
2625
|
-
/* node:child_process unavailable — skip */
|
|
2626
|
-
}
|
|
210
|
+
return orgStop.stopOrg(this, name, opts);
|
|
2627
211
|
}
|
|
2628
212
|
async stopAll() {
|
|
2629
213
|
await Promise.all([
|
|
@@ -2638,52 +222,7 @@ export class OrgDaemon {
|
|
|
2638
222
|
* stop`). Mirrors persistCrashStateAll()'s existing closedBy: 'crash-handler'
|
|
2639
223
|
* for the process-crash path, which org.ts already reads. */
|
|
2640
224
|
persistState(name, status, run, org, checkpointOverride, closedBy) {
|
|
2641
|
-
|
|
2642
|
-
const missing = [...(this.abandoned.get(name) ?? [])];
|
|
2643
|
-
const memoryError = this.memoryErrors.get(name);
|
|
2644
|
-
const running = org ?? this.orgs.get(name);
|
|
2645
|
-
const validStatus = status === 'stopped' || status === 'crashed' ? status : 'running';
|
|
2646
|
-
// Pattern 3: Capture full checkpoint state for resume. On stop, finishStop
|
|
2647
|
-
// passes a snapshot captured BEFORE mailboxes close and sessions drain —
|
|
2648
|
-
// otherwise the queue is always empty by persist time.
|
|
2649
|
-
let checkpoint = checkpointOverride ?? null;
|
|
2650
|
-
if (!checkpoint && running) {
|
|
2651
|
-
// Best-effort, like the snapshot in finishStop: persisting the run's
|
|
2652
|
-
// state matters more than the resume checkpoint inside it, and a stop
|
|
2653
|
-
// must not fail because a checkpoint could not be built.
|
|
2654
|
-
try {
|
|
2655
|
-
checkpoint = captureCheckpoint(running, validStatus);
|
|
2656
|
-
}
|
|
2657
|
-
catch (err) {
|
|
2658
|
-
console.error(`org ${name}: could not capture the ${validStatus} checkpoint:`, err instanceof Error ? err.message : err);
|
|
2659
|
-
checkpoint = null;
|
|
2660
|
-
}
|
|
2661
|
-
}
|
|
2662
|
-
else if (checkpoint && checkpoint.status !== validStatus) {
|
|
2663
|
-
const { checksum: _, ...state } = checkpoint;
|
|
2664
|
-
checkpoint = {
|
|
2665
|
-
...state,
|
|
2666
|
-
status: validStatus,
|
|
2667
|
-
checksum: generateChecksum({
|
|
2668
|
-
...state,
|
|
2669
|
-
status: validStatus,
|
|
2670
|
-
}),
|
|
2671
|
-
};
|
|
2672
|
-
}
|
|
2673
|
-
// C4: writeJsonFileAtomic (tmp + rename) — a direct writeFileSync here
|
|
2674
|
-
// could leave runtime.json truncated on Ctrl-C during `org stop`, which
|
|
2675
|
-
// would brick every subsequent `org status` / isOrgRunning / scheduler
|
|
2676
|
-
// call. The state files in 6 other daemon paths already use this helper.
|
|
2677
|
-
writeJsonFileAtomic(p, {
|
|
2678
|
-
status,
|
|
2679
|
-
run,
|
|
2680
|
-
pid: process.pid,
|
|
2681
|
-
updated: new Date().toISOString(),
|
|
2682
|
-
...(missing.length ? { abandonedRoles: missing } : {}),
|
|
2683
|
-
...(memoryError ? { memoryError } : {}),
|
|
2684
|
-
...(checkpoint ? { checkpoint } : {}),
|
|
2685
|
-
...(closedBy ? { closedBy } : {}),
|
|
2686
|
-
});
|
|
225
|
+
orgStateFile.persistState(this, name, status, run, org, checkpointOverride, closedBy);
|
|
2687
226
|
}
|
|
2688
227
|
/** Mark every currently-running org as crashed in runtime.json.
|
|
2689
228
|
* Called from process-level crash handlers — must be synchronous and best-effort.
|
|
@@ -2691,68 +230,19 @@ export class OrgDaemon {
|
|
|
2691
230
|
* this, `runOutcomeResult` (org.ts)'s "crashed: <error>" message always
|
|
2692
231
|
* read "crashed: unknown error" regardless of what actually happened. */
|
|
2693
232
|
persistCrashStateAll(error) {
|
|
2694
|
-
|
|
2695
|
-
try {
|
|
2696
|
-
const p = join(this.root, ORG_DIR, name, 'runtime.json');
|
|
2697
|
-
// Capture separately from the write below: a throw here (e.g. a
|
|
2698
|
-
// cyclic structure in roleState reaching generateChecksum) must not
|
|
2699
|
-
// suppress the base crash record, which is the actually-important
|
|
2700
|
-
// best-effort write this method exists for.
|
|
2701
|
-
let checkpoint;
|
|
2702
|
-
try {
|
|
2703
|
-
checkpoint = captureCheckpoint(org, 'crashed');
|
|
2704
|
-
}
|
|
2705
|
-
catch {
|
|
2706
|
-
/* best effort — proceed without a checkpoint */
|
|
2707
|
-
}
|
|
2708
|
-
// C4: atomic write — crash handler is the most likely place to hit
|
|
2709
|
-
// a partial write since the process is mid-teardown.
|
|
2710
|
-
writeJsonFileAtomic(p, {
|
|
2711
|
-
status: 'crashed',
|
|
2712
|
-
run: org.run,
|
|
2713
|
-
pid: process.pid,
|
|
2714
|
-
updated: new Date().toISOString(),
|
|
2715
|
-
closedBy: 'crash-handler',
|
|
2716
|
-
...(checkpoint ? { checkpoint } : {}),
|
|
2717
|
-
...(error ? { error } : {}),
|
|
2718
|
-
});
|
|
2719
|
-
}
|
|
2720
|
-
catch {
|
|
2721
|
-
/* best effort — filesystem may be unavailable */
|
|
2722
|
-
}
|
|
2723
|
-
}
|
|
2724
|
-
}
|
|
2725
|
-
heartbeatPath() {
|
|
2726
|
-
return join(this.root, '.monomind', 'serve-heartbeat.json');
|
|
233
|
+
orgStateFile.persistCrashStateAll(this, error);
|
|
2727
234
|
}
|
|
2728
235
|
/** Write a heartbeat file so `org status` can distinguish "daemon alive" from
|
|
2729
236
|
* "daemon gone" even when runtime.json still says running. */
|
|
2730
237
|
writeHeartbeat() {
|
|
2731
|
-
|
|
2732
|
-
const p = this.heartbeatPath();
|
|
2733
|
-
mkdirSync(join(this.root, '.monomind'), { recursive: true });
|
|
2734
|
-
// C4: atomic write — heartbeat corruption is how `org status` reports
|
|
2735
|
-
// a phantom daemon after a crash.
|
|
2736
|
-
writeJsonFileAtomic(p, {
|
|
2737
|
-
pid: process.pid,
|
|
2738
|
-
updatedAt: new Date().toISOString(),
|
|
2739
|
-
running: this.listRunning(),
|
|
2740
|
-
});
|
|
2741
|
-
}
|
|
2742
|
-
catch {
|
|
2743
|
-
/* best effort */
|
|
2744
|
-
}
|
|
238
|
+
orgStateFile.writeHeartbeat(this);
|
|
2745
239
|
}
|
|
2746
240
|
clearHeartbeat() {
|
|
2747
|
-
|
|
2748
|
-
unlinkSync(this.heartbeatPath());
|
|
2749
|
-
}
|
|
2750
|
-
catch {
|
|
2751
|
-
/* already gone or never written */
|
|
2752
|
-
}
|
|
241
|
+
orgStateFile.clearHeartbeat(this);
|
|
2753
242
|
}
|
|
2754
243
|
// ── Delegated methods — extracted to focused modules ──────────────────
|
|
2755
244
|
// approvals.ts
|
|
245
|
+
/** @internal */
|
|
2756
246
|
checkApproval(org, role, action, input) {
|
|
2757
247
|
return approvalOps.checkApproval(this, org, role, action, input);
|
|
2758
248
|
}
|
|
@@ -2767,6 +257,7 @@ export class OrgDaemon {
|
|
|
2767
257
|
return questionOps.answerQuestion(this, org, role, questionId, answer, resolvedBy);
|
|
2768
258
|
}
|
|
2769
259
|
// decisions.ts
|
|
260
|
+
/** @internal */
|
|
2770
261
|
readGates(org) {
|
|
2771
262
|
return decisionOps.gatesFor(this, org);
|
|
2772
263
|
}
|
|
@@ -2779,9 +270,11 @@ export class OrgDaemon {
|
|
|
2779
270
|
listGates(org, status) {
|
|
2780
271
|
return decisionOps.listGates(this, org, status);
|
|
2781
272
|
}
|
|
273
|
+
/** @internal */
|
|
2782
274
|
dagCreateTask(org, role, title, assignee, deps, loadout, brief, pick) {
|
|
2783
275
|
return decisionOps.dagCreateTask(this, org, role, title, assignee, deps, loadout, brief, pick);
|
|
2784
276
|
}
|
|
277
|
+
/** @internal */
|
|
2785
278
|
dagCompleteTask(org, role, taskId, result, evidence) {
|
|
2786
279
|
return decisionOps.dagCompleteTask(this, org, role, taskId, result, evidence);
|
|
2787
280
|
}
|
|
@@ -2789,18 +282,23 @@ export class OrgDaemon {
|
|
|
2789
282
|
dagRequestReview(org, role, taskId, reviewer, base) {
|
|
2790
283
|
return decisionOps.dagRequestReview(this, org, role, taskId, reviewer, base);
|
|
2791
284
|
}
|
|
285
|
+
/** @internal */
|
|
2792
286
|
dagSplitTask(org, role, parentId, children) {
|
|
2793
287
|
return decisionOps.dagSplitTask(this, org, role, parentId, children);
|
|
2794
288
|
}
|
|
289
|
+
/** @internal */
|
|
2795
290
|
dagMergeTask(org, role, sourceId, targetId) {
|
|
2796
291
|
return decisionOps.dagMergeTask(this, org, role, sourceId, targetId);
|
|
2797
292
|
}
|
|
293
|
+
/** @internal */
|
|
2798
294
|
dagCancelTask(org, role, taskId, reason) {
|
|
2799
295
|
return decisionOps.dagCancelTask(this, org, role, taskId, reason);
|
|
2800
296
|
}
|
|
297
|
+
/** @internal */
|
|
2801
298
|
dagBlockTask(org, role, taskId, untilIso, reason, recheckAfterMinutes) {
|
|
2802
299
|
return decisionOps.dagBlockTask(this, org, role, taskId, untilIso, reason, recheckAfterMinutes);
|
|
2803
300
|
}
|
|
301
|
+
/** @internal */
|
|
2804
302
|
dagPlanGraph(org, role, specs) {
|
|
2805
303
|
return decisionOps.dagPlanGraph(this, org, role, specs);
|
|
2806
304
|
}
|
|
@@ -2823,6 +321,7 @@ export class OrgDaemon {
|
|
|
2823
321
|
autoWake(name) {
|
|
2824
322
|
scheduler.autoWake(this, name);
|
|
2825
323
|
}
|
|
324
|
+
/** @internal */
|
|
2826
325
|
scheduleBossRestart(name) {
|
|
2827
326
|
scheduler.scheduleBossRestart(this, name);
|
|
2828
327
|
}
|
|
@@ -2841,24 +340,30 @@ export class OrgDaemon {
|
|
|
2841
340
|
orgMemoryNamespace(name, def) {
|
|
2842
341
|
return orgMemory.orgMemoryNamespace(name, def);
|
|
2843
342
|
}
|
|
343
|
+
/** @internal */
|
|
2844
344
|
orgMemoryDbPath() {
|
|
2845
345
|
return orgMemory.orgMemoryDbPath(this.root);
|
|
2846
346
|
}
|
|
347
|
+
/** @internal */
|
|
2847
348
|
orgMemoryUsable() {
|
|
2848
349
|
return orgMemory.orgMemoryUsable(this.root);
|
|
2849
350
|
}
|
|
351
|
+
/** @internal */
|
|
2850
352
|
async rememberOrgMemory(name, def, role, content, scope, run) {
|
|
2851
353
|
return orgMemory.rememberOrgMemory(this.root, name, def, role, content, scope, run);
|
|
2852
354
|
}
|
|
355
|
+
/** @internal */
|
|
2853
356
|
async recallOrgMemory(name, def, query, role) {
|
|
2854
357
|
return orgMemory.recallOrgMemory(this, name, def, query, role);
|
|
2855
358
|
}
|
|
2856
359
|
async searchProjectKnowledge(query) {
|
|
2857
360
|
return orgMemory.searchProjectKnowledge(this.root, query);
|
|
2858
361
|
}
|
|
362
|
+
/** @internal */
|
|
2859
363
|
async learnOrgKnowledge(name, run, payload) {
|
|
2860
364
|
return orgMemory.learnOrgKnowledge(this, name, run, payload);
|
|
2861
365
|
}
|
|
366
|
+
/** @internal */
|
|
2862
367
|
async storeRunMemory(name, def, run, summary, bus) {
|
|
2863
368
|
return orgMemory.storeRunMemory(this, name, def, run, summary, bus);
|
|
2864
369
|
}
|