@ag-eco/agentplate-cli 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +462 -0
- package/agents/ap-co-creation.md +90 -0
- package/agents/builder.md +144 -0
- package/agents/coordinator.md +377 -0
- package/agents/lead.md +435 -0
- package/agents/merger.md +164 -0
- package/agents/monitor.md +214 -0
- package/agents/orchestrator.md +239 -0
- package/agents/reviewer.md +140 -0
- package/agents/scout.md +125 -0
- package/agents/supervisor.md +427 -0
- package/package.json +66 -0
- package/src/agents/capabilities.test.ts +85 -0
- package/src/agents/capabilities.ts +125 -0
- package/src/agents/checkpoint.test.ts +88 -0
- package/src/agents/checkpoint.ts +101 -0
- package/src/agents/copilot-hooks-deployer.test.ts +162 -0
- package/src/agents/copilot-hooks-deployer.ts +93 -0
- package/src/agents/guard-rules.test.ts +372 -0
- package/src/agents/guard-rules.ts +97 -0
- package/src/agents/headless-mail-injector.test.ts +709 -0
- package/src/agents/headless-mail-injector.ts +377 -0
- package/src/agents/headless-prompt.test.ts +102 -0
- package/src/agents/headless-prompt.ts +68 -0
- package/src/agents/hooks-deployer.test.ts +3119 -0
- package/src/agents/hooks-deployer.ts +804 -0
- package/src/agents/identity.test.ts +604 -0
- package/src/agents/identity.ts +384 -0
- package/src/agents/lifecycle.test.ts +196 -0
- package/src/agents/lifecycle.ts +183 -0
- package/src/agents/mail-poll-detect.test.ts +153 -0
- package/src/agents/mail-poll-detect.ts +73 -0
- package/src/agents/manifest.test.ts +1026 -0
- package/src/agents/manifest.ts +376 -0
- package/src/agents/overlay.test.ts +1058 -0
- package/src/agents/overlay.ts +490 -0
- package/src/agents/scope-detect.test.ts +190 -0
- package/src/agents/scope-detect.ts +146 -0
- package/src/agents/turn-lock.test.ts +181 -0
- package/src/agents/turn-lock.ts +235 -0
- package/src/agents/turn-runner-dispatch.test.ts +182 -0
- package/src/agents/turn-runner-dispatch.ts +105 -0
- package/src/agents/turn-runner.test.ts +2312 -0
- package/src/agents/turn-runner.ts +1383 -0
- package/src/beads/client.test.ts +217 -0
- package/src/beads/client.ts +230 -0
- package/src/beads/molecules.test.ts +338 -0
- package/src/beads/molecules.ts +198 -0
- package/src/commands/agents.test.ts +328 -0
- package/src/commands/agents.ts +299 -0
- package/src/commands/clean.test.ts +797 -0
- package/src/commands/clean.ts +791 -0
- package/src/commands/completions.test.ts +348 -0
- package/src/commands/completions.ts +981 -0
- package/src/commands/coordinator.test.ts +2975 -0
- package/src/commands/coordinator.ts +1841 -0
- package/src/commands/costs.test.ts +1183 -0
- package/src/commands/costs.ts +599 -0
- package/src/commands/dashboard.test.ts +954 -0
- package/src/commands/dashboard.ts +1212 -0
- package/src/commands/discover.test.ts +288 -0
- package/src/commands/discover.ts +202 -0
- package/src/commands/doctor.test.ts +303 -0
- package/src/commands/doctor.ts +311 -0
- package/src/commands/ecosystem.test.ts +226 -0
- package/src/commands/ecosystem.ts +248 -0
- package/src/commands/errors.test.ts +654 -0
- package/src/commands/errors.ts +197 -0
- package/src/commands/feed.test.ts +709 -0
- package/src/commands/feed.ts +260 -0
- package/src/commands/group.test.ts +475 -0
- package/src/commands/group.ts +546 -0
- package/src/commands/hooks.test.ts +458 -0
- package/src/commands/hooks.ts +263 -0
- package/src/commands/init.test.ts +1011 -0
- package/src/commands/init.ts +967 -0
- package/src/commands/inspect.test.ts +1239 -0
- package/src/commands/inspect.ts +648 -0
- package/src/commands/log.test.ts +1913 -0
- package/src/commands/log.ts +958 -0
- package/src/commands/logs.test.ts +801 -0
- package/src/commands/logs.ts +483 -0
- package/src/commands/mail.test.ts +1501 -0
- package/src/commands/mail.ts +848 -0
- package/src/commands/merge.test.ts +864 -0
- package/src/commands/merge.ts +381 -0
- package/src/commands/metrics.test.ts +458 -0
- package/src/commands/metrics.ts +129 -0
- package/src/commands/monitor.test.ts +191 -0
- package/src/commands/monitor.ts +409 -0
- package/src/commands/nudge.test.ts +579 -0
- package/src/commands/nudge.ts +646 -0
- package/src/commands/orchestrator.ts +42 -0
- package/src/commands/prime.test.ts +612 -0
- package/src/commands/prime.ts +359 -0
- package/src/commands/replay.test.ts +757 -0
- package/src/commands/replay.ts +231 -0
- package/src/commands/run.test.ts +469 -0
- package/src/commands/run.ts +353 -0
- package/src/commands/serve/agent-actions.test.ts +210 -0
- package/src/commands/serve/agent-actions.ts +192 -0
- package/src/commands/serve/build.test.ts +202 -0
- package/src/commands/serve/build.ts +206 -0
- package/src/commands/serve/coordinator-actions.test.ts +339 -0
- package/src/commands/serve/coordinator-actions.ts +410 -0
- package/src/commands/serve/dev.test.ts +168 -0
- package/src/commands/serve/dev.ts +117 -0
- package/src/commands/serve/mail-actions.test.ts +312 -0
- package/src/commands/serve/mail-actions.ts +167 -0
- package/src/commands/serve/rest.test.ts +1680 -0
- package/src/commands/serve/rest.ts +1130 -0
- package/src/commands/serve/static.ts +51 -0
- package/src/commands/serve/ws.test.ts +361 -0
- package/src/commands/serve/ws.ts +332 -0
- package/src/commands/serve.test.ts +459 -0
- package/src/commands/serve.ts +654 -0
- package/src/commands/sling.test.ts +1583 -0
- package/src/commands/sling.ts +1351 -0
- package/src/commands/spec.test.ts +179 -0
- package/src/commands/spec.ts +105 -0
- package/src/commands/status.test.ts +614 -0
- package/src/commands/status.ts +403 -0
- package/src/commands/stop.test.ts +964 -0
- package/src/commands/stop.ts +319 -0
- package/src/commands/supervisor.test.ts +185 -0
- package/src/commands/supervisor.ts +537 -0
- package/src/commands/trace.test.ts +762 -0
- package/src/commands/trace.ts +205 -0
- package/src/commands/update.test.ts +466 -0
- package/src/commands/update.ts +263 -0
- package/src/commands/upgrade.test.ts +48 -0
- package/src/commands/upgrade.ts +240 -0
- package/src/commands/watch.test.ts +257 -0
- package/src/commands/watch.ts +308 -0
- package/src/commands/worktree.test.ts +1297 -0
- package/src/commands/worktree.ts +451 -0
- package/src/config.test.ts +1535 -0
- package/src/config.ts +1064 -0
- package/src/doctor/agents.test.ts +523 -0
- package/src/doctor/agents.ts +399 -0
- package/src/doctor/config-check.test.ts +191 -0
- package/src/doctor/config-check.ts +183 -0
- package/src/doctor/consistency.test.ts +807 -0
- package/src/doctor/consistency.ts +347 -0
- package/src/doctor/databases.test.ts +350 -0
- package/src/doctor/databases.ts +243 -0
- package/src/doctor/dependencies.test.ts +296 -0
- package/src/doctor/dependencies.ts +272 -0
- package/src/doctor/ecosystem.test.ts +308 -0
- package/src/doctor/ecosystem.ts +156 -0
- package/src/doctor/logs.test.ts +253 -0
- package/src/doctor/logs.ts +295 -0
- package/src/doctor/merge-queue.test.ts +315 -0
- package/src/doctor/merge-queue.ts +167 -0
- package/src/doctor/providers.test.ts +409 -0
- package/src/doctor/providers.ts +250 -0
- package/src/doctor/serve.test.ts +95 -0
- package/src/doctor/serve.ts +86 -0
- package/src/doctor/structure.test.ts +423 -0
- package/src/doctor/structure.ts +285 -0
- package/src/doctor/types.ts +43 -0
- package/src/doctor/version.test.ts +241 -0
- package/src/doctor/version.ts +132 -0
- package/src/doctor/watchdog.test.ts +167 -0
- package/src/doctor/watchdog.ts +214 -0
- package/src/e2e/init-sling-lifecycle.test.ts +283 -0
- package/src/errors.test.ts +350 -0
- package/src/errors.ts +217 -0
- package/src/events/store.test.ts +660 -0
- package/src/events/store.ts +369 -0
- package/src/events/tailer.test.ts +719 -0
- package/src/events/tailer.ts +332 -0
- package/src/events/tool-filter.test.ts +330 -0
- package/src/events/tool-filter.ts +126 -0
- package/src/index.ts +533 -0
- package/src/insights/analyzer.test.ts +466 -0
- package/src/insights/analyzer.ts +203 -0
- package/src/insights/quality-gates.test.ts +141 -0
- package/src/insights/quality-gates.ts +156 -0
- package/src/json.test.ts +72 -0
- package/src/json.ts +53 -0
- package/src/loam/client.test.ts +752 -0
- package/src/loam/client.ts +664 -0
- package/src/logging/color.test.ts +252 -0
- package/src/logging/color.ts +105 -0
- package/src/logging/format.test.ts +110 -0
- package/src/logging/format.ts +255 -0
- package/src/logging/logger.test.ts +814 -0
- package/src/logging/logger.ts +266 -0
- package/src/logging/reporter.test.ts +259 -0
- package/src/logging/reporter.ts +110 -0
- package/src/logging/sanitizer.test.ts +190 -0
- package/src/logging/sanitizer.ts +57 -0
- package/src/logging/theme.ts +140 -0
- package/src/mail/broadcast.test.ts +204 -0
- package/src/mail/broadcast.ts +92 -0
- package/src/mail/client.test.ts +774 -0
- package/src/mail/client.ts +236 -0
- package/src/mail/store.test.ts +898 -0
- package/src/mail/store.ts +425 -0
- package/src/merge/lock.test.ts +149 -0
- package/src/merge/lock.ts +140 -0
- package/src/merge/predict.test.ts +387 -0
- package/src/merge/predict.ts +249 -0
- package/src/merge/queue.test.ts +426 -0
- package/src/merge/queue.ts +246 -0
- package/src/merge/resolver.test.ts +1993 -0
- package/src/merge/resolver.ts +926 -0
- package/src/metrics/pricing.test.ts +258 -0
- package/src/metrics/pricing.ts +135 -0
- package/src/metrics/store.test.ts +978 -0
- package/src/metrics/store.ts +501 -0
- package/src/metrics/summary.test.ts +398 -0
- package/src/metrics/summary.ts +178 -0
- package/src/metrics/transcript.test.ts +483 -0
- package/src/metrics/transcript.ts +114 -0
- package/src/runtimes/__fixtures__/claude-stream-fixture.ts +22 -0
- package/src/runtimes/aider.test.ts +124 -0
- package/src/runtimes/aider.ts +147 -0
- package/src/runtimes/amp.test.ts +164 -0
- package/src/runtimes/amp.ts +154 -0
- package/src/runtimes/claude.test.ts +1474 -0
- package/src/runtimes/claude.ts +579 -0
- package/src/runtimes/codex.test.ts +805 -0
- package/src/runtimes/codex.ts +273 -0
- package/src/runtimes/connections.test.ts +214 -0
- package/src/runtimes/connections.ts +103 -0
- package/src/runtimes/copilot.test.ts +707 -0
- package/src/runtimes/copilot.ts +316 -0
- package/src/runtimes/cursor.test.ts +497 -0
- package/src/runtimes/cursor.ts +205 -0
- package/src/runtimes/gemini.test.ts +537 -0
- package/src/runtimes/gemini.ts +243 -0
- package/src/runtimes/goose.test.ts +133 -0
- package/src/runtimes/goose.ts +157 -0
- package/src/runtimes/headless-connection.test.ts +264 -0
- package/src/runtimes/headless-connection.ts +158 -0
- package/src/runtimes/opencode.test.ts +325 -0
- package/src/runtimes/opencode.ts +188 -0
- package/src/runtimes/pi-guards.test.ts +486 -0
- package/src/runtimes/pi-guards.ts +367 -0
- package/src/runtimes/pi.test.ts +789 -0
- package/src/runtimes/pi.ts +305 -0
- package/src/runtimes/registry.test.ts +196 -0
- package/src/runtimes/registry.ts +99 -0
- package/src/runtimes/sapling.test.ts +1267 -0
- package/src/runtimes/sapling.ts +710 -0
- package/src/runtimes/types.ts +266 -0
- package/src/schema-consistency.test.ts +246 -0
- package/src/sessions/compat.test.ts +281 -0
- package/src/sessions/compat.ts +105 -0
- package/src/sessions/store.test.ts +1748 -0
- package/src/sessions/store.ts +858 -0
- package/src/test-helpers.test.ts +124 -0
- package/src/test-helpers.ts +145 -0
- package/src/test-setup.test.ts +31 -0
- package/src/test-setup.ts +28 -0
- package/src/tools/loam/api.ts +368 -0
- package/src/tools/loam/cli.ts +278 -0
- package/src/tools/loam/commands/add.ts +52 -0
- package/src/tools/loam/commands/archive.ts +214 -0
- package/src/tools/loam/commands/audit.ts +276 -0
- package/src/tools/loam/commands/compact.ts +1062 -0
- package/src/tools/loam/commands/completions.ts +79 -0
- package/src/tools/loam/commands/config.ts +381 -0
- package/src/tools/loam/commands/delete-domain.ts +121 -0
- package/src/tools/loam/commands/delete.ts +316 -0
- package/src/tools/loam/commands/diff.ts +200 -0
- package/src/tools/loam/commands/doctor.ts +1113 -0
- package/src/tools/loam/commands/edit.ts +226 -0
- package/src/tools/loam/commands/init.ts +31 -0
- package/src/tools/loam/commands/learn.ts +179 -0
- package/src/tools/loam/commands/move.ts +323 -0
- package/src/tools/loam/commands/onboard.ts +374 -0
- package/src/tools/loam/commands/outcome.ts +185 -0
- package/src/tools/loam/commands/prime.ts +688 -0
- package/src/tools/loam/commands/prune.ts +614 -0
- package/src/tools/loam/commands/query.ts +218 -0
- package/src/tools/loam/commands/rank.ts +180 -0
- package/src/tools/loam/commands/ready.ts +189 -0
- package/src/tools/loam/commands/record.ts +1210 -0
- package/src/tools/loam/commands/restore.ts +166 -0
- package/src/tools/loam/commands/search.ts +327 -0
- package/src/tools/loam/commands/setup.ts +887 -0
- package/src/tools/loam/commands/status.ts +103 -0
- package/src/tools/loam/commands/sync.ts +298 -0
- package/src/tools/loam/commands/update.ts +19 -0
- package/src/tools/loam/commands/upgrade.ts +93 -0
- package/src/tools/loam/commands/validate.ts +190 -0
- package/src/tools/loam/index.ts +62 -0
- package/src/tools/loam/log.ts +127 -0
- package/src/tools/loam/registry/builtins.ts +409 -0
- package/src/tools/loam/registry/custom.ts +431 -0
- package/src/tools/loam/registry/init.ts +55 -0
- package/src/tools/loam/registry/template.ts +40 -0
- package/src/tools/loam/registry/type-registry.ts +113 -0
- package/src/tools/loam/schemas/config-schema.ts +489 -0
- package/src/tools/loam/schemas/config.ts +245 -0
- package/src/tools/loam/schemas/index.ts +18 -0
- package/src/tools/loam/schemas/record-schema.ts +191 -0
- package/src/tools/loam/schemas/record.ts +115 -0
- package/src/tools/loam/utils/active-work.ts +205 -0
- package/src/tools/loam/utils/anchor-validity.ts +80 -0
- package/src/tools/loam/utils/archive.ts +146 -0
- package/src/tools/loam/utils/audit.ts +667 -0
- package/src/tools/loam/utils/bm25.ts +238 -0
- package/src/tools/loam/utils/budget.ts +142 -0
- package/src/tools/loam/utils/config.ts +344 -0
- package/src/tools/loam/utils/dir-anchors.ts +62 -0
- package/src/tools/loam/utils/domain-rules.ts +114 -0
- package/src/tools/loam/utils/expertise.ts +393 -0
- package/src/tools/loam/utils/format-helpers.ts +96 -0
- package/src/tools/loam/utils/format.ts +1234 -0
- package/src/tools/loam/utils/git-context.ts +50 -0
- package/src/tools/loam/utils/git.ts +183 -0
- package/src/tools/loam/utils/hooks.ts +299 -0
- package/src/tools/loam/utils/index.ts +52 -0
- package/src/tools/loam/utils/json-output.ts +13 -0
- package/src/tools/loam/utils/lock.ts +76 -0
- package/src/tools/loam/utils/markers.ts +48 -0
- package/src/tools/loam/utils/numeric-flags.ts +20 -0
- package/src/tools/loam/utils/palette.ts +44 -0
- package/src/tools/loam/utils/prime-ranking.ts +135 -0
- package/src/tools/loam/utils/recipe-discovery.ts +195 -0
- package/src/tools/loam/utils/runtime-flags.ts +28 -0
- package/src/tools/loam/utils/scoring.ts +94 -0
- package/src/tools/loam/utils/version.ts +116 -0
- package/src/tools/sprout/commands/block.ts +64 -0
- package/src/tools/sprout/commands/blocked.ts +86 -0
- package/src/tools/sprout/commands/close.ts +129 -0
- package/src/tools/sprout/commands/completions.ts +198 -0
- package/src/tools/sprout/commands/config.ts +238 -0
- package/src/tools/sprout/commands/create.ts +164 -0
- package/src/tools/sprout/commands/dep.ts +148 -0
- package/src/tools/sprout/commands/doctor.ts +979 -0
- package/src/tools/sprout/commands/init.ts +83 -0
- package/src/tools/sprout/commands/label.ts +178 -0
- package/src/tools/sprout/commands/list.ts +210 -0
- package/src/tools/sprout/commands/migrate.ts +133 -0
- package/src/tools/sprout/commands/onboard.ts +207 -0
- package/src/tools/sprout/commands/plan-show.ts +278 -0
- package/src/tools/sprout/commands/plan.ts +2526 -0
- package/src/tools/sprout/commands/prime.ts +399 -0
- package/src/tools/sprout/commands/ready.ts +245 -0
- package/src/tools/sprout/commands/search.ts +221 -0
- package/src/tools/sprout/commands/show.ts +277 -0
- package/src/tools/sprout/commands/stats.ts +146 -0
- package/src/tools/sprout/commands/sync.ts +134 -0
- package/src/tools/sprout/commands/tpl.ts +364 -0
- package/src/tools/sprout/commands/unblock.ts +115 -0
- package/src/tools/sprout/commands/update.ts +257 -0
- package/src/tools/sprout/commands/upgrade.ts +91 -0
- package/src/tools/sprout/config-schema.ts +152 -0
- package/src/tools/sprout/config.ts +355 -0
- package/src/tools/sprout/filter.ts +107 -0
- package/src/tools/sprout/format.ts +43 -0
- package/src/tools/sprout/id.ts +22 -0
- package/src/tools/sprout/index.ts +204 -0
- package/src/tools/sprout/log.ts +76 -0
- package/src/tools/sprout/markers.ts +22 -0
- package/src/tools/sprout/output.ts +121 -0
- package/src/tools/sprout/plan-backref.ts +93 -0
- package/src/tools/sprout/plan-context.ts +81 -0
- package/src/tools/sprout/plan-domain.ts +139 -0
- package/src/tools/sprout/plan-lifecycle.ts +65 -0
- package/src/tools/sprout/plan-loam.ts +207 -0
- package/src/tools/sprout/plan-schema.ts +209 -0
- package/src/tools/sprout/sort.ts +31 -0
- package/src/tools/sprout/store.ts +172 -0
- package/src/tools/sprout/types.ts +118 -0
- package/src/tools/sprout/validation.ts +119 -0
- package/src/tools/sprout/version.ts +1 -0
- package/src/tools/sprout/yaml.ts +387 -0
- package/src/tools/trellis/commands/archive.ts +87 -0
- package/src/tools/trellis/commands/completions.ts +610 -0
- package/src/tools/trellis/commands/config.ts +382 -0
- package/src/tools/trellis/commands/create.ts +252 -0
- package/src/tools/trellis/commands/diff.ts +150 -0
- package/src/tools/trellis/commands/doctor.ts +771 -0
- package/src/tools/trellis/commands/emit.ts +365 -0
- package/src/tools/trellis/commands/history.ts +83 -0
- package/src/tools/trellis/commands/import.ts +198 -0
- package/src/tools/trellis/commands/init.ts +81 -0
- package/src/tools/trellis/commands/list.ts +103 -0
- package/src/tools/trellis/commands/onboard.ts +156 -0
- package/src/tools/trellis/commands/pin.ts +172 -0
- package/src/tools/trellis/commands/prime.ts +193 -0
- package/src/tools/trellis/commands/render.ts +122 -0
- package/src/tools/trellis/commands/schema.ts +353 -0
- package/src/tools/trellis/commands/show.ts +115 -0
- package/src/tools/trellis/commands/stats.ts +65 -0
- package/src/tools/trellis/commands/sync.ts +112 -0
- package/src/tools/trellis/commands/tree.ts +123 -0
- package/src/tools/trellis/commands/update.ts +330 -0
- package/src/tools/trellis/commands/upgrade.ts +95 -0
- package/src/tools/trellis/commands/validate.ts +166 -0
- package/src/tools/trellis/config-schema.ts +81 -0
- package/src/tools/trellis/config.ts +108 -0
- package/src/tools/trellis/frontmatter.ts +348 -0
- package/src/tools/trellis/id.ts +24 -0
- package/src/tools/trellis/index.ts +209 -0
- package/src/tools/trellis/markers.ts +28 -0
- package/src/tools/trellis/output.ts +84 -0
- package/src/tools/trellis/render.ts +212 -0
- package/src/tools/trellis/store.ts +144 -0
- package/src/tools/trellis/types.ts +82 -0
- package/src/tools/trellis/validate.ts +199 -0
- package/src/tools/trellis/yaml.ts +309 -0
- package/src/tracker/beads.test.ts +454 -0
- package/src/tracker/beads.ts +56 -0
- package/src/tracker/factory.test.ts +90 -0
- package/src/tracker/factory.ts +65 -0
- package/src/tracker/sprout.test.ts +461 -0
- package/src/tracker/sprout.ts +182 -0
- package/src/tracker/types.ts +52 -0
- package/src/trellis/client.test.ts +107 -0
- package/src/trellis/client.ts +179 -0
- package/src/types.ts +970 -0
- package/src/utils/bin.test.ts +10 -0
- package/src/utils/bin.ts +37 -0
- package/src/utils/browser.test.ts +49 -0
- package/src/utils/browser.ts +48 -0
- package/src/utils/fs.test.ts +119 -0
- package/src/utils/fs.ts +62 -0
- package/src/utils/pid.test.ts +152 -0
- package/src/utils/pid.ts +130 -0
- package/src/utils/process-scan.test.ts +53 -0
- package/src/utils/process-scan.ts +76 -0
- package/src/utils/time.test.ts +43 -0
- package/src/utils/time.ts +37 -0
- package/src/utils/version.test.ts +33 -0
- package/src/utils/version.ts +70 -0
- package/src/version.ts +5 -0
- package/src/watchdog/daemon.test.ts +3721 -0
- package/src/watchdog/daemon.ts +1257 -0
- package/src/watchdog/health.test.ts +830 -0
- package/src/watchdog/health.ts +434 -0
- package/src/watchdog/triage.test.ts +205 -0
- package/src/watchdog/triage.ts +205 -0
- package/src/worktree/manager.test.ts +720 -0
- package/src/worktree/manager.ts +405 -0
- package/src/worktree/process.test.ts +172 -0
- package/src/worktree/process.ts +131 -0
- package/src/worktree/tmux.test.ts +1616 -0
- package/src/worktree/tmux.ts +721 -0
- package/templates/CLAUDE.md.tmpl +100 -0
- package/templates/copilot-hooks.json.tmpl +13 -0
- package/templates/hooks.json.tmpl +109 -0
- package/templates/overlay.md.tmpl +88 -0
- package/ui/dist/apple-touch-icon-bdy6teep.png +0 -0
- package/ui/dist/chunk-8s31f05k.css +1 -0
- package/ui/dist/chunk-vm5rz679.js +300 -0
- package/ui/dist/favicon-nzb39vza.svg +4 -0
- package/ui/dist/index.html +17 -0
|
@@ -0,0 +1,3721 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Integration tests for the watchdog daemon tick loop.
|
|
3
|
+
*
|
|
4
|
+
* Uses real filesystem (temp directories via mkdtemp) and real SessionStore
|
|
5
|
+
* (bun:sqlite) for session persistence, plus real health evaluation logic.
|
|
6
|
+
*
|
|
7
|
+
* Only tmux operations (isSessionAlive, killSession), triage, and nudge are
|
|
8
|
+
* mocked via dependency injection (_tmux, _triage, _nudge params) because:
|
|
9
|
+
* - Real tmux interferes with developer sessions and is fragile in CI.
|
|
10
|
+
* - Real triage spawns Claude CLI which has cost and latency.
|
|
11
|
+
* - Real nudge requires active tmux sessions.
|
|
12
|
+
*
|
|
13
|
+
* Does NOT use mock.module() — it leaks across test files. See loam record
|
|
14
|
+
* mx-56558b for background.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
|
|
18
|
+
import { mkdir, mkdtemp } from "node:fs/promises";
|
|
19
|
+
import { tmpdir } from "node:os";
|
|
20
|
+
import { join } from "node:path";
|
|
21
|
+
import { createEventStore } from "../events/store.ts";
|
|
22
|
+
import { createMailStore } from "../mail/store.ts";
|
|
23
|
+
import { createRunStore, createSessionStore } from "../sessions/store.ts";
|
|
24
|
+
import { cleanupTempDir } from "../test-helpers.ts";
|
|
25
|
+
import type { AgentSession, HealthCheck, StoredEvent, WorkerDiedPayload } from "../types.ts";
|
|
26
|
+
import {
|
|
27
|
+
buildCompletionMessage,
|
|
28
|
+
type RunIdWarnState,
|
|
29
|
+
runDaemonTick,
|
|
30
|
+
startDaemon,
|
|
31
|
+
} from "./daemon.ts";
|
|
32
|
+
|
|
33
|
+
// === Test constants ===
|
|
34
|
+
|
|
35
|
+
const THRESHOLDS = {
|
|
36
|
+
staleThresholdMs: 30_000,
|
|
37
|
+
zombieThresholdMs: 120_000,
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
// === Helpers ===
|
|
41
|
+
|
|
42
|
+
/** Create a temp directory with .agentplate/ subdirectory, ready for sessions.db. */
|
|
43
|
+
async function createTempRoot(): Promise<string> {
|
|
44
|
+
const dir = await mkdtemp(join(tmpdir(), "agentplate-daemon-test-"));
|
|
45
|
+
await mkdir(join(dir, ".agentplate"), { recursive: true });
|
|
46
|
+
return dir;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Write sessions to the SessionStore (sessions.db) at the given root. */
|
|
50
|
+
function writeSessionsToStore(root: string, sessions: AgentSession[]): void {
|
|
51
|
+
const dbPath = join(root, ".agentplate", "sessions.db");
|
|
52
|
+
const store = createSessionStore(dbPath);
|
|
53
|
+
for (const session of sessions) {
|
|
54
|
+
store.upsert(session);
|
|
55
|
+
}
|
|
56
|
+
store.close();
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Mark a run as active: write current-run.txt AND insert a row in the runs
|
|
61
|
+
* table (sessions.db). The watchdog now validates the id against the runs
|
|
62
|
+
* table before running the run-completion check (agentplate-87bf), so tests
|
|
63
|
+
* must seed both surfaces to mirror production reality.
|
|
64
|
+
*/
|
|
65
|
+
async function setActiveRun(root: string, runId: string): Promise<void> {
|
|
66
|
+
await Bun.write(join(root, ".agentplate", "current-run.txt"), runId);
|
|
67
|
+
const runStore = createRunStore(join(root, ".agentplate", "sessions.db"));
|
|
68
|
+
try {
|
|
69
|
+
runStore.createRun({
|
|
70
|
+
id: runId,
|
|
71
|
+
startedAt: new Date().toISOString(),
|
|
72
|
+
coordinatorSessionId: null,
|
|
73
|
+
status: "active",
|
|
74
|
+
});
|
|
75
|
+
} catch {
|
|
76
|
+
// Row may already exist (re-seeding within one test) — non-fatal.
|
|
77
|
+
} finally {
|
|
78
|
+
runStore.close();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Build a fresh, isolated RunIdWarnState for tests (agentplate-87bf). */
|
|
83
|
+
function freshRunIdWarnState(): RunIdWarnState {
|
|
84
|
+
return { missingFileWarned: false, unknownIds: new Set() };
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Read sessions from the SessionStore (sessions.db) at the given root. */
|
|
88
|
+
function readSessionsFromStore(root: string): AgentSession[] {
|
|
89
|
+
const dbPath = join(root, ".agentplate", "sessions.db");
|
|
90
|
+
const store = createSessionStore(dbPath);
|
|
91
|
+
const sessions = store.getAll();
|
|
92
|
+
store.close();
|
|
93
|
+
return sessions;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Build a test AgentSession with sensible defaults. */
|
|
97
|
+
function makeSession(overrides: Partial<AgentSession> = {}): AgentSession {
|
|
98
|
+
return {
|
|
99
|
+
id: "session-test",
|
|
100
|
+
agentName: "test-agent",
|
|
101
|
+
capability: "builder",
|
|
102
|
+
worktreePath: "/tmp/test",
|
|
103
|
+
branchName: "agentplate/test-agent/test-task",
|
|
104
|
+
taskId: "test-task",
|
|
105
|
+
tmuxSession: "agentplate-test-agent",
|
|
106
|
+
state: "working",
|
|
107
|
+
pid: process.pid, // Use our own PID so isProcessRunning returns true
|
|
108
|
+
parentAgent: null,
|
|
109
|
+
depth: 0,
|
|
110
|
+
runId: null,
|
|
111
|
+
escalationLevel: 0,
|
|
112
|
+
stalledSince: null,
|
|
113
|
+
transcriptPath: null,
|
|
114
|
+
startedAt: new Date().toISOString(),
|
|
115
|
+
lastActivity: new Date().toISOString(),
|
|
116
|
+
...overrides,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Create a fake _tmux dependency where all sessions are alive. */
|
|
121
|
+
function tmuxAllAlive(): {
|
|
122
|
+
isSessionAlive: (name: string) => Promise<boolean>;
|
|
123
|
+
killSession: (name: string) => Promise<void>;
|
|
124
|
+
} {
|
|
125
|
+
return {
|
|
126
|
+
isSessionAlive: async () => true,
|
|
127
|
+
killSession: async () => {},
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Create a fake _tmux dependency where all sessions are dead. */
|
|
132
|
+
function tmuxAllDead(): {
|
|
133
|
+
isSessionAlive: (name: string) => Promise<boolean>;
|
|
134
|
+
killSession: (name: string) => Promise<void>;
|
|
135
|
+
} {
|
|
136
|
+
return {
|
|
137
|
+
isSessionAlive: async () => false,
|
|
138
|
+
killSession: async () => {},
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Create a fake _tmux dependency with per-session liveness control.
|
|
144
|
+
* Also tracks killSession calls for assertions.
|
|
145
|
+
*/
|
|
146
|
+
function tmuxWithLiveness(aliveMap: Record<string, boolean>): {
|
|
147
|
+
isSessionAlive: (name: string) => Promise<boolean>;
|
|
148
|
+
killSession: (name: string) => Promise<void>;
|
|
149
|
+
killed: string[];
|
|
150
|
+
} {
|
|
151
|
+
const killed: string[] = [];
|
|
152
|
+
return {
|
|
153
|
+
isSessionAlive: async (name: string) => aliveMap[name] ?? false,
|
|
154
|
+
killSession: async (name: string) => {
|
|
155
|
+
killed.push(name);
|
|
156
|
+
},
|
|
157
|
+
killed,
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Create a fake _triage that always returns the given verdict. */
|
|
162
|
+
function triageAlways(
|
|
163
|
+
verdict: "retry" | "terminate" | "extend",
|
|
164
|
+
): (options: {
|
|
165
|
+
agentName: string;
|
|
166
|
+
root: string;
|
|
167
|
+
lastActivity: string;
|
|
168
|
+
}) => Promise<"retry" | "terminate" | "extend"> {
|
|
169
|
+
return async () => verdict;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Create a fake _nudge that tracks calls and always succeeds. */
|
|
173
|
+
function nudgeTracker(): {
|
|
174
|
+
nudge: (
|
|
175
|
+
projectRoot: string,
|
|
176
|
+
agentName: string,
|
|
177
|
+
message: string,
|
|
178
|
+
force: boolean,
|
|
179
|
+
) => Promise<{ delivered: boolean; reason?: string }>;
|
|
180
|
+
calls: Array<{ agentName: string; message: string }>;
|
|
181
|
+
} {
|
|
182
|
+
const calls: Array<{ agentName: string; message: string }> = [];
|
|
183
|
+
return {
|
|
184
|
+
nudge: async (_projectRoot: string, agentName: string, message: string, _force: boolean) => {
|
|
185
|
+
calls.push({ agentName, message });
|
|
186
|
+
return { delivered: true };
|
|
187
|
+
},
|
|
188
|
+
calls,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// === Tests ===
|
|
193
|
+
|
|
194
|
+
let tempRoot: string;
|
|
195
|
+
|
|
196
|
+
beforeEach(async () => {
|
|
197
|
+
tempRoot = await createTempRoot();
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
afterEach(async () => {
|
|
201
|
+
await cleanupTempDir(tempRoot);
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
describe("daemon tick", () => {
|
|
205
|
+
// --- Test 1: tick with no sessions file ---
|
|
206
|
+
|
|
207
|
+
test("tick with no sessions is a graceful no-op", async () => {
|
|
208
|
+
// No sessions in the store — daemon should not crash
|
|
209
|
+
const checks: HealthCheck[] = [];
|
|
210
|
+
|
|
211
|
+
await runDaemonTick({
|
|
212
|
+
root: tempRoot,
|
|
213
|
+
...THRESHOLDS,
|
|
214
|
+
onHealthCheck: (c) => checks.push(c),
|
|
215
|
+
_tmux: tmuxAllAlive(),
|
|
216
|
+
_triage: triageAlways("extend"),
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
// No health checks should have been produced (no sessions to check)
|
|
220
|
+
expect(checks).toHaveLength(0);
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
// --- Test 2: tick with healthy sessions ---
|
|
224
|
+
|
|
225
|
+
test("tick with healthy sessions produces no state changes", async () => {
|
|
226
|
+
const session = makeSession({
|
|
227
|
+
state: "working",
|
|
228
|
+
lastActivity: new Date().toISOString(),
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
232
|
+
|
|
233
|
+
const checks: HealthCheck[] = [];
|
|
234
|
+
|
|
235
|
+
await runDaemonTick({
|
|
236
|
+
root: tempRoot,
|
|
237
|
+
...THRESHOLDS,
|
|
238
|
+
onHealthCheck: (c) => checks.push(c),
|
|
239
|
+
_tmux: tmuxAllAlive(),
|
|
240
|
+
_triage: triageAlways("extend"),
|
|
241
|
+
});
|
|
242
|
+
|
|
243
|
+
expect(checks).toHaveLength(1);
|
|
244
|
+
const check = checks[0];
|
|
245
|
+
expect(check).toBeDefined();
|
|
246
|
+
expect(check?.state).toBe("working");
|
|
247
|
+
expect(check?.action).toBe("none");
|
|
248
|
+
|
|
249
|
+
// Session state should be unchanged because state didn't change.
|
|
250
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
251
|
+
expect(reloaded).toHaveLength(1);
|
|
252
|
+
expect(reloaded[0]?.state).toBe("working");
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
// --- Test 3: tick with dead tmux -> zombie transition ---
|
|
256
|
+
|
|
257
|
+
test("tick with dead tmux transitions session to zombie and fires terminate", async () => {
|
|
258
|
+
const session = makeSession({
|
|
259
|
+
agentName: "dead-agent",
|
|
260
|
+
tmuxSession: "agentplate-dead-agent",
|
|
261
|
+
state: "working",
|
|
262
|
+
lastActivity: new Date().toISOString(),
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
266
|
+
|
|
267
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-dead-agent": false });
|
|
268
|
+
const checks: HealthCheck[] = [];
|
|
269
|
+
|
|
270
|
+
await runDaemonTick({
|
|
271
|
+
root: tempRoot,
|
|
272
|
+
...THRESHOLDS,
|
|
273
|
+
onHealthCheck: (c) => checks.push(c),
|
|
274
|
+
_tmux: tmuxMock,
|
|
275
|
+
_triage: triageAlways("extend"),
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
// Health check should detect zombie with terminate action
|
|
279
|
+
expect(checks).toHaveLength(1);
|
|
280
|
+
expect(checks[0]?.state).toBe("zombie");
|
|
281
|
+
expect(checks[0]?.action).toBe("terminate");
|
|
282
|
+
|
|
283
|
+
// tmux is dead so killSession should NOT be called (only kills if tmuxAlive)
|
|
284
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
285
|
+
|
|
286
|
+
// Session state should be persisted as zombie
|
|
287
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
288
|
+
expect(reloaded).toHaveLength(1);
|
|
289
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
test("tick with alive tmux but zombie-old activity calls killSession", async () => {
|
|
293
|
+
// tmux IS alive but time-based zombie threshold is exceeded,
|
|
294
|
+
// causing a terminate action — killSession SHOULD be called.
|
|
295
|
+
const oldActivity = new Date(Date.now() - 200_000).toISOString();
|
|
296
|
+
const session = makeSession({
|
|
297
|
+
agentName: "zombie-agent",
|
|
298
|
+
tmuxSession: "agentplate-zombie-agent",
|
|
299
|
+
state: "working",
|
|
300
|
+
lastActivity: oldActivity,
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
304
|
+
|
|
305
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-zombie-agent": true });
|
|
306
|
+
const checks: HealthCheck[] = [];
|
|
307
|
+
|
|
308
|
+
await runDaemonTick({
|
|
309
|
+
root: tempRoot,
|
|
310
|
+
...THRESHOLDS,
|
|
311
|
+
onHealthCheck: (c) => checks.push(c),
|
|
312
|
+
_tmux: tmuxMock,
|
|
313
|
+
_triage: triageAlways("extend"),
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
expect(checks).toHaveLength(1);
|
|
317
|
+
expect(checks[0]?.action).toBe("terminate");
|
|
318
|
+
|
|
319
|
+
// tmux was alive, so killSession SHOULD have been called
|
|
320
|
+
expect(tmuxMock.killed).toContain("agentplate-zombie-agent");
|
|
321
|
+
|
|
322
|
+
// Session persisted as zombie
|
|
323
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
324
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
// --- Test 4: progressive nudging for stalled agents ---
|
|
328
|
+
|
|
329
|
+
test("first tick with stalled agent sets stalledSince and stays at level 0 (warn)", async () => {
|
|
330
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
331
|
+
const session = makeSession({
|
|
332
|
+
agentName: "stalled-agent",
|
|
333
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
334
|
+
state: "working",
|
|
335
|
+
lastActivity: staleActivity,
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
339
|
+
|
|
340
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-stalled-agent": true });
|
|
341
|
+
const checks: HealthCheck[] = [];
|
|
342
|
+
const nudgeMock = nudgeTracker();
|
|
343
|
+
|
|
344
|
+
await runDaemonTick({
|
|
345
|
+
root: tempRoot,
|
|
346
|
+
...THRESHOLDS,
|
|
347
|
+
nudgeIntervalMs: 60_000,
|
|
348
|
+
onHealthCheck: (c) => checks.push(c),
|
|
349
|
+
_tmux: tmuxMock,
|
|
350
|
+
_triage: triageAlways("extend"),
|
|
351
|
+
_nudge: nudgeMock.nudge,
|
|
352
|
+
});
|
|
353
|
+
|
|
354
|
+
expect(checks).toHaveLength(1);
|
|
355
|
+
expect(checks[0]?.action).toBe("escalate");
|
|
356
|
+
|
|
357
|
+
// No kill at level 0
|
|
358
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
359
|
+
|
|
360
|
+
// No nudge at level 0 (warn only)
|
|
361
|
+
expect(nudgeMock.calls).toHaveLength(0);
|
|
362
|
+
|
|
363
|
+
// Session should be stalled with stalledSince set and escalationLevel 0
|
|
364
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
365
|
+
expect(reloaded[0]?.state).toBe("stalled");
|
|
366
|
+
expect(reloaded[0]?.escalationLevel).toBe(0);
|
|
367
|
+
expect(reloaded[0]?.stalledSince).not.toBeNull();
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
test("stalled agent at level 1 sends nudge", async () => {
|
|
371
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
372
|
+
// Pre-set stalledSince to > nudgeIntervalMs ago so level advances to 1
|
|
373
|
+
const stalledSince = new Date(Date.now() - 70_000).toISOString();
|
|
374
|
+
const session = makeSession({
|
|
375
|
+
agentName: "stalled-agent",
|
|
376
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
377
|
+
state: "stalled",
|
|
378
|
+
lastActivity: staleActivity,
|
|
379
|
+
escalationLevel: 0,
|
|
380
|
+
stalledSince,
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
384
|
+
|
|
385
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-stalled-agent": true });
|
|
386
|
+
const nudgeMock = nudgeTracker();
|
|
387
|
+
|
|
388
|
+
await runDaemonTick({
|
|
389
|
+
root: tempRoot,
|
|
390
|
+
...THRESHOLDS,
|
|
391
|
+
nudgeIntervalMs: 60_000,
|
|
392
|
+
_tmux: tmuxMock,
|
|
393
|
+
_triage: triageAlways("extend"),
|
|
394
|
+
_nudge: nudgeMock.nudge,
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
// Level should advance to 1 and nudge should be sent
|
|
398
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
399
|
+
expect(reloaded[0]?.escalationLevel).toBe(1);
|
|
400
|
+
expect(nudgeMock.calls).toHaveLength(1);
|
|
401
|
+
expect(nudgeMock.calls[0]?.agentName).toBe("stalled-agent");
|
|
402
|
+
expect(nudgeMock.calls[0]?.message).toContain("WATCHDOG");
|
|
403
|
+
|
|
404
|
+
// No kill
|
|
405
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
test("stalled agent at level 2 calls triage when tier1Enabled", async () => {
|
|
409
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
410
|
+
// Pre-set stalledSince to > 2*nudgeIntervalMs ago so level advances to 2
|
|
411
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
412
|
+
const session = makeSession({
|
|
413
|
+
agentName: "stalled-agent",
|
|
414
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
415
|
+
state: "stalled",
|
|
416
|
+
lastActivity: staleActivity,
|
|
417
|
+
escalationLevel: 1,
|
|
418
|
+
stalledSince,
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
422
|
+
|
|
423
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-stalled-agent": true });
|
|
424
|
+
let triageCalled = false;
|
|
425
|
+
|
|
426
|
+
const triageMock = async (opts: {
|
|
427
|
+
agentName: string;
|
|
428
|
+
root: string;
|
|
429
|
+
lastActivity: string;
|
|
430
|
+
}): Promise<"retry" | "terminate" | "extend"> => {
|
|
431
|
+
triageCalled = true;
|
|
432
|
+
expect(opts.agentName).toBe("stalled-agent");
|
|
433
|
+
return "terminate";
|
|
434
|
+
};
|
|
435
|
+
|
|
436
|
+
await runDaemonTick({
|
|
437
|
+
root: tempRoot,
|
|
438
|
+
...THRESHOLDS,
|
|
439
|
+
nudgeIntervalMs: 60_000,
|
|
440
|
+
tier1Enabled: true,
|
|
441
|
+
_tmux: tmuxMock,
|
|
442
|
+
_triage: triageMock,
|
|
443
|
+
_nudge: nudgeTracker().nudge,
|
|
444
|
+
});
|
|
445
|
+
|
|
446
|
+
expect(triageCalled).toBe(true);
|
|
447
|
+
|
|
448
|
+
// Triage returned terminate — session should be zombie
|
|
449
|
+
expect(tmuxMock.killed).toContain("agentplate-stalled-agent");
|
|
450
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
451
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
452
|
+
});
|
|
453
|
+
|
|
454
|
+
test("stalled agent at level 2 skips triage when tier1Enabled is false", async () => {
|
|
455
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
456
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
457
|
+
const session = makeSession({
|
|
458
|
+
agentName: "stalled-agent",
|
|
459
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
460
|
+
state: "stalled",
|
|
461
|
+
lastActivity: staleActivity,
|
|
462
|
+
escalationLevel: 1,
|
|
463
|
+
stalledSince,
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
467
|
+
|
|
468
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-stalled-agent": true });
|
|
469
|
+
let triageCalled = false;
|
|
470
|
+
|
|
471
|
+
const triageMock = async (): Promise<"retry" | "terminate" | "extend"> => {
|
|
472
|
+
triageCalled = true;
|
|
473
|
+
return "terminate";
|
|
474
|
+
};
|
|
475
|
+
|
|
476
|
+
await runDaemonTick({
|
|
477
|
+
root: tempRoot,
|
|
478
|
+
...THRESHOLDS,
|
|
479
|
+
nudgeIntervalMs: 60_000,
|
|
480
|
+
tier1Enabled: false, // Triage disabled
|
|
481
|
+
_tmux: tmuxMock,
|
|
482
|
+
_triage: triageMock,
|
|
483
|
+
_nudge: nudgeTracker().nudge,
|
|
484
|
+
});
|
|
485
|
+
|
|
486
|
+
// Triage should NOT have been called
|
|
487
|
+
expect(triageCalled).toBe(false);
|
|
488
|
+
|
|
489
|
+
// No kill — level 2 with tier1 disabled just skips
|
|
490
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
491
|
+
|
|
492
|
+
// Session stays stalled at level 2
|
|
493
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
494
|
+
expect(reloaded[0]?.state).toBe("stalled");
|
|
495
|
+
expect(reloaded[0]?.escalationLevel).toBe(2);
|
|
496
|
+
});
|
|
497
|
+
|
|
498
|
+
test("stalled agent at level 3 is terminated", async () => {
|
|
499
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
500
|
+
// Pre-set stalledSince to > 3*nudgeIntervalMs ago so level advances to 3
|
|
501
|
+
const stalledSince = new Date(Date.now() - 200_000).toISOString();
|
|
502
|
+
const session = makeSession({
|
|
503
|
+
agentName: "doomed-agent",
|
|
504
|
+
tmuxSession: "agentplate-doomed-agent",
|
|
505
|
+
state: "stalled",
|
|
506
|
+
lastActivity: staleActivity,
|
|
507
|
+
escalationLevel: 2,
|
|
508
|
+
stalledSince,
|
|
509
|
+
});
|
|
510
|
+
|
|
511
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
512
|
+
|
|
513
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-doomed-agent": true });
|
|
514
|
+
|
|
515
|
+
await runDaemonTick({
|
|
516
|
+
root: tempRoot,
|
|
517
|
+
...THRESHOLDS,
|
|
518
|
+
nudgeIntervalMs: 60_000,
|
|
519
|
+
_tmux: tmuxMock,
|
|
520
|
+
_triage: triageAlways("extend"),
|
|
521
|
+
_nudge: nudgeTracker().nudge,
|
|
522
|
+
});
|
|
523
|
+
|
|
524
|
+
// Level 3 = terminate
|
|
525
|
+
expect(tmuxMock.killed).toContain("agentplate-doomed-agent");
|
|
526
|
+
|
|
527
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
528
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
529
|
+
// Escalation is reset after termination
|
|
530
|
+
expect(reloaded[0]?.escalationLevel).toBe(0);
|
|
531
|
+
expect(reloaded[0]?.stalledSince).toBeNull();
|
|
532
|
+
});
|
|
533
|
+
|
|
534
|
+
// Regression tests for agentplate-74ce: killAgent() must never call
|
|
535
|
+
// tmux.killSession("") for headless agents — an empty `-t` argument is
|
|
536
|
+
// prefix-matched and would wildcard-kill the entire agentplate tmux server.
|
|
537
|
+
|
|
538
|
+
test("spawn-per-turn agent at level 3 termination does NOT call tmux.killSession", async () => {
|
|
539
|
+
const nudgeIntervalMs = 60_000;
|
|
540
|
+
const stalledSince = new Date(Date.now() - 4 * nudgeIntervalMs).toISOString();
|
|
541
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
542
|
+
|
|
543
|
+
// Spawn-per-turn worker between turns: tmuxSession === "" AND pid === null.
|
|
544
|
+
// Before the fix, killAgent fell through to tmux.killSession("") which
|
|
545
|
+
// prefix-matches every session in the agentplate tmux server.
|
|
546
|
+
const session = makeSession({
|
|
547
|
+
agentName: "spawn-per-turn-doomed",
|
|
548
|
+
tmuxSession: "",
|
|
549
|
+
pid: null,
|
|
550
|
+
state: "stalled",
|
|
551
|
+
lastActivity: staleActivity,
|
|
552
|
+
escalationLevel: 2,
|
|
553
|
+
stalledSince,
|
|
554
|
+
});
|
|
555
|
+
|
|
556
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
557
|
+
|
|
558
|
+
// No tmux sessions registered — emulates production where the spawn-per-turn
|
|
559
|
+
// agent has no named session.
|
|
560
|
+
const tmuxMock = tmuxWithLiveness({});
|
|
561
|
+
|
|
562
|
+
await runDaemonTick({
|
|
563
|
+
root: tempRoot,
|
|
564
|
+
...THRESHOLDS,
|
|
565
|
+
nudgeIntervalMs,
|
|
566
|
+
tier1Enabled: false,
|
|
567
|
+
_tmux: tmuxMock,
|
|
568
|
+
_triage: triageAlways("extend"),
|
|
569
|
+
_nudge: nudgeTracker().nudge,
|
|
570
|
+
_eventStore: null,
|
|
571
|
+
_recordFailure: async () => {},
|
|
572
|
+
_getConnection: () => undefined,
|
|
573
|
+
_removeConnection: () => {},
|
|
574
|
+
_tailerRegistry: new Map(),
|
|
575
|
+
_findLatestStdoutLog: async () => null,
|
|
576
|
+
});
|
|
577
|
+
|
|
578
|
+
// Critical assertion: no wildcard kill attempt. tmuxMock.killed must be empty.
|
|
579
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
580
|
+
|
|
581
|
+
// The session is still transitioned to zombie — termination semantics are preserved,
|
|
582
|
+
// just without the wildcard tmux kill.
|
|
583
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
584
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
585
|
+
expect(reloaded[0]?.escalationLevel).toBe(0);
|
|
586
|
+
expect(reloaded[0]?.stalledSince).toBeNull();
|
|
587
|
+
});
|
|
588
|
+
|
|
589
|
+
test("long-lived headless agent at level 3 termination kills pid tree, not tmux", async () => {
|
|
590
|
+
const nudgeIntervalMs = 60_000;
|
|
591
|
+
const stalledSince = new Date(Date.now() - 4 * nudgeIntervalMs).toISOString();
|
|
592
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
593
|
+
|
|
594
|
+
// Long-lived headless capability (e.g. coordinator/orchestrator/monitor):
|
|
595
|
+
// tmuxSession === "" AND pid !== null. The PID tree should be killed; tmux
|
|
596
|
+
// must not be touched.
|
|
597
|
+
const session = makeSession({
|
|
598
|
+
agentName: "headless-long-lived-doomed",
|
|
599
|
+
tmuxSession: "",
|
|
600
|
+
pid: process.pid, // alive PID — health eval won't short-circuit to direct terminate
|
|
601
|
+
state: "stalled",
|
|
602
|
+
lastActivity: staleActivity,
|
|
603
|
+
escalationLevel: 2,
|
|
604
|
+
stalledSince,
|
|
605
|
+
});
|
|
606
|
+
|
|
607
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
608
|
+
|
|
609
|
+
const killedPids: number[] = [];
|
|
610
|
+
const procMock = {
|
|
611
|
+
isAlive: (pid: number) => {
|
|
612
|
+
try {
|
|
613
|
+
process.kill(pid, 0);
|
|
614
|
+
return true;
|
|
615
|
+
} catch {
|
|
616
|
+
return false;
|
|
617
|
+
}
|
|
618
|
+
},
|
|
619
|
+
killTree: async (pid: number) => {
|
|
620
|
+
killedPids.push(pid);
|
|
621
|
+
},
|
|
622
|
+
};
|
|
623
|
+
|
|
624
|
+
const tmuxMock = tmuxWithLiveness({});
|
|
625
|
+
|
|
626
|
+
await runDaemonTick({
|
|
627
|
+
root: tempRoot,
|
|
628
|
+
...THRESHOLDS,
|
|
629
|
+
nudgeIntervalMs,
|
|
630
|
+
tier1Enabled: false,
|
|
631
|
+
_tmux: tmuxMock,
|
|
632
|
+
_triage: triageAlways("extend"),
|
|
633
|
+
_nudge: nudgeTracker().nudge,
|
|
634
|
+
_process: procMock,
|
|
635
|
+
_eventStore: null,
|
|
636
|
+
_recordFailure: async () => {},
|
|
637
|
+
_getConnection: () => undefined,
|
|
638
|
+
_removeConnection: () => {},
|
|
639
|
+
_tailerRegistry: new Map(),
|
|
640
|
+
_findLatestStdoutLog: async () => null,
|
|
641
|
+
});
|
|
642
|
+
|
|
643
|
+
// PID tree was killed; tmux.killSession was never called.
|
|
644
|
+
expect(killedPids).toContain(process.pid);
|
|
645
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
646
|
+
|
|
647
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
648
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
649
|
+
});
|
|
650
|
+
|
|
651
|
+
test("triage retry sends nudge with recovery message", async () => {
|
|
652
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
653
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
654
|
+
const session = makeSession({
|
|
655
|
+
agentName: "retry-agent",
|
|
656
|
+
tmuxSession: "agentplate-retry-agent",
|
|
657
|
+
state: "stalled",
|
|
658
|
+
lastActivity: staleActivity,
|
|
659
|
+
escalationLevel: 1,
|
|
660
|
+
stalledSince,
|
|
661
|
+
});
|
|
662
|
+
|
|
663
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
664
|
+
|
|
665
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-retry-agent": true });
|
|
666
|
+
const nudgeMock = nudgeTracker();
|
|
667
|
+
|
|
668
|
+
await runDaemonTick({
|
|
669
|
+
root: tempRoot,
|
|
670
|
+
...THRESHOLDS,
|
|
671
|
+
nudgeIntervalMs: 60_000,
|
|
672
|
+
tier1Enabled: true,
|
|
673
|
+
_tmux: tmuxMock,
|
|
674
|
+
_triage: triageAlways("retry"),
|
|
675
|
+
_nudge: nudgeMock.nudge,
|
|
676
|
+
});
|
|
677
|
+
|
|
678
|
+
// Triage returned "retry" — nudge should be sent with recovery message
|
|
679
|
+
expect(nudgeMock.calls).toHaveLength(1);
|
|
680
|
+
expect(nudgeMock.calls[0]?.message).toContain("recovery");
|
|
681
|
+
|
|
682
|
+
// No kill
|
|
683
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
684
|
+
|
|
685
|
+
// Session stays stalled
|
|
686
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
687
|
+
expect(reloaded[0]?.state).toBe("stalled");
|
|
688
|
+
});
|
|
689
|
+
|
|
690
|
+
test("agent recovery resets escalation tracking", async () => {
|
|
691
|
+
// Agent was stalled but now has recent activity
|
|
692
|
+
const session = makeSession({
|
|
693
|
+
agentName: "recovered-agent",
|
|
694
|
+
tmuxSession: "agentplate-recovered-agent",
|
|
695
|
+
state: "working",
|
|
696
|
+
lastActivity: new Date().toISOString(), // Recent activity
|
|
697
|
+
escalationLevel: 2,
|
|
698
|
+
stalledSince: new Date(Date.now() - 130_000).toISOString(),
|
|
699
|
+
});
|
|
700
|
+
|
|
701
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
702
|
+
|
|
703
|
+
await runDaemonTick({
|
|
704
|
+
root: tempRoot,
|
|
705
|
+
...THRESHOLDS,
|
|
706
|
+
_tmux: tmuxAllAlive(),
|
|
707
|
+
_triage: triageAlways("extend"),
|
|
708
|
+
_nudge: nudgeTracker().nudge,
|
|
709
|
+
});
|
|
710
|
+
|
|
711
|
+
// Health check should return action: "none" for recovered agent
|
|
712
|
+
// Escalation tracking should be reset
|
|
713
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
714
|
+
expect(reloaded[0]?.state).toBe("working");
|
|
715
|
+
expect(reloaded[0]?.escalationLevel).toBe(0);
|
|
716
|
+
expect(reloaded[0]?.stalledSince).toBeNull();
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
// --- Test 5: session persistence round-trip ---
|
|
720
|
+
|
|
721
|
+
test("session persistence round-trip: load, modify, save, reload", async () => {
|
|
722
|
+
const sessions: AgentSession[] = [
|
|
723
|
+
makeSession({
|
|
724
|
+
id: "session-1",
|
|
725
|
+
agentName: "agent-alpha",
|
|
726
|
+
tmuxSession: "agentplate-agent-alpha",
|
|
727
|
+
state: "working",
|
|
728
|
+
lastActivity: new Date().toISOString(),
|
|
729
|
+
}),
|
|
730
|
+
makeSession({
|
|
731
|
+
id: "session-2",
|
|
732
|
+
agentName: "agent-beta",
|
|
733
|
+
tmuxSession: "agentplate-agent-beta",
|
|
734
|
+
state: "working",
|
|
735
|
+
// Make beta's tmux dead so it transitions to zombie
|
|
736
|
+
lastActivity: new Date().toISOString(),
|
|
737
|
+
}),
|
|
738
|
+
makeSession({
|
|
739
|
+
id: "session-3",
|
|
740
|
+
agentName: "agent-gamma",
|
|
741
|
+
tmuxSession: "agentplate-agent-gamma",
|
|
742
|
+
state: "completed",
|
|
743
|
+
lastActivity: new Date().toISOString(),
|
|
744
|
+
}),
|
|
745
|
+
];
|
|
746
|
+
|
|
747
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
748
|
+
|
|
749
|
+
const tmuxMock = tmuxWithLiveness({
|
|
750
|
+
"agentplate-agent-alpha": true,
|
|
751
|
+
"agentplate-agent-beta": false, // Dead — should become zombie
|
|
752
|
+
"agentplate-agent-gamma": true, // Doesn't matter — completed is skipped
|
|
753
|
+
});
|
|
754
|
+
|
|
755
|
+
const checks: HealthCheck[] = [];
|
|
756
|
+
|
|
757
|
+
await runDaemonTick({
|
|
758
|
+
root: tempRoot,
|
|
759
|
+
...THRESHOLDS,
|
|
760
|
+
onHealthCheck: (c) => checks.push(c),
|
|
761
|
+
_tmux: tmuxMock,
|
|
762
|
+
_triage: triageAlways("extend"),
|
|
763
|
+
});
|
|
764
|
+
|
|
765
|
+
// Completed sessions are skipped — only 2 health checks
|
|
766
|
+
expect(checks).toHaveLength(2);
|
|
767
|
+
|
|
768
|
+
// Reload and verify persistence
|
|
769
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
770
|
+
expect(reloaded).toHaveLength(3);
|
|
771
|
+
|
|
772
|
+
const alpha = reloaded.find((s) => s.agentName === "agent-alpha");
|
|
773
|
+
const beta = reloaded.find((s) => s.agentName === "agent-beta");
|
|
774
|
+
const gamma = reloaded.find((s) => s.agentName === "agent-gamma");
|
|
775
|
+
|
|
776
|
+
expect(alpha).toBeDefined();
|
|
777
|
+
expect(beta).toBeDefined();
|
|
778
|
+
expect(gamma).toBeDefined();
|
|
779
|
+
|
|
780
|
+
// Alpha: tmux alive + recent activity — stays working
|
|
781
|
+
expect(alpha?.state).toBe("working");
|
|
782
|
+
|
|
783
|
+
// Beta: tmux dead — zombie (ZFC rule 1)
|
|
784
|
+
expect(beta?.state).toBe("zombie");
|
|
785
|
+
|
|
786
|
+
// Gamma: completed — unchanged (skipped by daemon)
|
|
787
|
+
expect(gamma?.state).toBe("completed");
|
|
788
|
+
});
|
|
789
|
+
|
|
790
|
+
test("session persistence: state unchanged when nothing changes", async () => {
|
|
791
|
+
const session = makeSession({
|
|
792
|
+
state: "working",
|
|
793
|
+
lastActivity: new Date().toISOString(),
|
|
794
|
+
});
|
|
795
|
+
|
|
796
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
797
|
+
|
|
798
|
+
await runDaemonTick({
|
|
799
|
+
root: tempRoot,
|
|
800
|
+
...THRESHOLDS,
|
|
801
|
+
_tmux: tmuxAllAlive(),
|
|
802
|
+
_triage: triageAlways("extend"),
|
|
803
|
+
});
|
|
804
|
+
|
|
805
|
+
// Session state should remain unchanged since nothing triggered a transition
|
|
806
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
807
|
+
expect(reloaded).toHaveLength(1);
|
|
808
|
+
expect(reloaded[0]?.state).toBe("working");
|
|
809
|
+
});
|
|
810
|
+
|
|
811
|
+
// --- Edge cases ---
|
|
812
|
+
|
|
813
|
+
test("completed sessions are skipped entirely", async () => {
|
|
814
|
+
const session = makeSession({ state: "completed" });
|
|
815
|
+
|
|
816
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
817
|
+
|
|
818
|
+
const checks: HealthCheck[] = [];
|
|
819
|
+
|
|
820
|
+
await runDaemonTick({
|
|
821
|
+
root: tempRoot,
|
|
822
|
+
...THRESHOLDS,
|
|
823
|
+
onHealthCheck: (c) => checks.push(c),
|
|
824
|
+
_tmux: tmuxAllDead(), // Would be zombie if not skipped
|
|
825
|
+
_triage: triageAlways("extend"),
|
|
826
|
+
});
|
|
827
|
+
|
|
828
|
+
// No health checks emitted for completed sessions
|
|
829
|
+
expect(checks).toHaveLength(0);
|
|
830
|
+
|
|
831
|
+
// State unchanged
|
|
832
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
833
|
+
expect(reloaded[0]?.state).toBe("completed");
|
|
834
|
+
});
|
|
835
|
+
|
|
836
|
+
test("multiple sessions with mixed states are all processed", async () => {
|
|
837
|
+
const now = Date.now();
|
|
838
|
+
const sessions: AgentSession[] = [
|
|
839
|
+
makeSession({
|
|
840
|
+
id: "s1",
|
|
841
|
+
agentName: "healthy",
|
|
842
|
+
tmuxSession: "agentplate-healthy",
|
|
843
|
+
state: "working",
|
|
844
|
+
lastActivity: new Date(now).toISOString(),
|
|
845
|
+
}),
|
|
846
|
+
makeSession({
|
|
847
|
+
id: "s2",
|
|
848
|
+
agentName: "dying",
|
|
849
|
+
tmuxSession: "agentplate-dying",
|
|
850
|
+
state: "working",
|
|
851
|
+
lastActivity: new Date(now).toISOString(),
|
|
852
|
+
}),
|
|
853
|
+
makeSession({
|
|
854
|
+
id: "s3",
|
|
855
|
+
agentName: "stale",
|
|
856
|
+
tmuxSession: "agentplate-stale",
|
|
857
|
+
state: "working",
|
|
858
|
+
lastActivity: new Date(now - 60_000).toISOString(),
|
|
859
|
+
}),
|
|
860
|
+
makeSession({
|
|
861
|
+
id: "s4",
|
|
862
|
+
agentName: "done",
|
|
863
|
+
tmuxSession: "agentplate-done",
|
|
864
|
+
state: "completed",
|
|
865
|
+
}),
|
|
866
|
+
];
|
|
867
|
+
|
|
868
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
869
|
+
|
|
870
|
+
const tmuxMock = tmuxWithLiveness({
|
|
871
|
+
"agentplate-healthy": true,
|
|
872
|
+
"agentplate-dying": false,
|
|
873
|
+
"agentplate-stale": true,
|
|
874
|
+
"agentplate-done": false,
|
|
875
|
+
});
|
|
876
|
+
|
|
877
|
+
const checks: HealthCheck[] = [];
|
|
878
|
+
|
|
879
|
+
await runDaemonTick({
|
|
880
|
+
root: tempRoot,
|
|
881
|
+
...THRESHOLDS,
|
|
882
|
+
onHealthCheck: (c) => checks.push(c),
|
|
883
|
+
_tmux: tmuxMock,
|
|
884
|
+
_triage: triageAlways("extend"),
|
|
885
|
+
_nudge: nudgeTracker().nudge,
|
|
886
|
+
});
|
|
887
|
+
|
|
888
|
+
// 3 non-completed sessions processed
|
|
889
|
+
expect(checks).toHaveLength(3);
|
|
890
|
+
|
|
891
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
892
|
+
|
|
893
|
+
const healthy = reloaded.find((s) => s.agentName === "healthy");
|
|
894
|
+
const dying = reloaded.find((s) => s.agentName === "dying");
|
|
895
|
+
const stale = reloaded.find((s) => s.agentName === "stale");
|
|
896
|
+
const done = reloaded.find((s) => s.agentName === "done");
|
|
897
|
+
|
|
898
|
+
expect(healthy?.state).toBe("working");
|
|
899
|
+
expect(dying?.state).toBe("zombie");
|
|
900
|
+
expect(stale?.state).toBe("stalled");
|
|
901
|
+
expect(done?.state).toBe("completed");
|
|
902
|
+
});
|
|
903
|
+
|
|
904
|
+
test("empty sessions array is a no-op", async () => {
|
|
905
|
+
writeSessionsToStore(tempRoot, []);
|
|
906
|
+
|
|
907
|
+
const checks: HealthCheck[] = [];
|
|
908
|
+
|
|
909
|
+
await runDaemonTick({
|
|
910
|
+
root: tempRoot,
|
|
911
|
+
...THRESHOLDS,
|
|
912
|
+
onHealthCheck: (c) => checks.push(c),
|
|
913
|
+
_tmux: tmuxAllAlive(),
|
|
914
|
+
_triage: triageAlways("extend"),
|
|
915
|
+
});
|
|
916
|
+
|
|
917
|
+
expect(checks).toHaveLength(0);
|
|
918
|
+
});
|
|
919
|
+
|
|
920
|
+
test("booting session with recent activity transitions to working", async () => {
|
|
921
|
+
const session = makeSession({
|
|
922
|
+
state: "booting",
|
|
923
|
+
lastActivity: new Date().toISOString(),
|
|
924
|
+
});
|
|
925
|
+
|
|
926
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
927
|
+
|
|
928
|
+
const checks: HealthCheck[] = [];
|
|
929
|
+
|
|
930
|
+
await runDaemonTick({
|
|
931
|
+
root: tempRoot,
|
|
932
|
+
...THRESHOLDS,
|
|
933
|
+
onHealthCheck: (c) => checks.push(c),
|
|
934
|
+
_tmux: tmuxAllAlive(),
|
|
935
|
+
_triage: triageAlways("extend"),
|
|
936
|
+
});
|
|
937
|
+
|
|
938
|
+
expect(checks).toHaveLength(1);
|
|
939
|
+
expect(checks[0]?.state).toBe("working");
|
|
940
|
+
|
|
941
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
942
|
+
expect(reloaded[0]?.state).toBe("working");
|
|
943
|
+
});
|
|
944
|
+
|
|
945
|
+
// --- Backward compatibility ---
|
|
946
|
+
|
|
947
|
+
test("sessions with default escalation fields are processed correctly", async () => {
|
|
948
|
+
// Write a session with default (zero) escalation fields
|
|
949
|
+
const session = makeSession({
|
|
950
|
+
id: "session-old",
|
|
951
|
+
agentName: "old-agent",
|
|
952
|
+
worktreePath: "/tmp/test",
|
|
953
|
+
branchName: "agentplate/old-agent/task",
|
|
954
|
+
taskId: "task",
|
|
955
|
+
tmuxSession: "agentplate-old-agent",
|
|
956
|
+
state: "working",
|
|
957
|
+
pid: process.pid,
|
|
958
|
+
escalationLevel: 0,
|
|
959
|
+
stalledSince: null,
|
|
960
|
+
transcriptPath: null,
|
|
961
|
+
});
|
|
962
|
+
|
|
963
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
964
|
+
|
|
965
|
+
const checks: HealthCheck[] = [];
|
|
966
|
+
|
|
967
|
+
await runDaemonTick({
|
|
968
|
+
root: tempRoot,
|
|
969
|
+
...THRESHOLDS,
|
|
970
|
+
onHealthCheck: (c) => checks.push(c),
|
|
971
|
+
_tmux: tmuxAllAlive(),
|
|
972
|
+
_triage: triageAlways("extend"),
|
|
973
|
+
});
|
|
974
|
+
|
|
975
|
+
// Should process without errors
|
|
976
|
+
expect(checks).toHaveLength(1);
|
|
977
|
+
expect(checks[0]?.state).toBe("working");
|
|
978
|
+
});
|
|
979
|
+
});
|
|
980
|
+
|
|
981
|
+
// === Event recording tests ===
|
|
982
|
+
|
|
983
|
+
describe("daemon event recording", () => {
|
|
984
|
+
/** Open the events.db in the temp root and return all events. */
|
|
985
|
+
function readEvents(root: string): StoredEvent[] {
|
|
986
|
+
const dbPath = join(root, ".agentplate", "events.db");
|
|
987
|
+
const store = createEventStore(dbPath);
|
|
988
|
+
try {
|
|
989
|
+
// Get all events (no agent filter — use a broad timeline)
|
|
990
|
+
return store.getTimeline({ since: "2000-01-01T00:00:00Z" });
|
|
991
|
+
} finally {
|
|
992
|
+
store.close();
|
|
993
|
+
}
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
test("escalation level 0 (warn) records event with type=escalation", async () => {
|
|
997
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
998
|
+
const session = makeSession({
|
|
999
|
+
agentName: "stalled-agent",
|
|
1000
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
1001
|
+
state: "working",
|
|
1002
|
+
lastActivity: staleActivity,
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1006
|
+
|
|
1007
|
+
// Create EventStore and inject it
|
|
1008
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
1009
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
1010
|
+
|
|
1011
|
+
try {
|
|
1012
|
+
await runDaemonTick({
|
|
1013
|
+
root: tempRoot,
|
|
1014
|
+
...THRESHOLDS,
|
|
1015
|
+
nudgeIntervalMs: 60_000,
|
|
1016
|
+
_tmux: tmuxWithLiveness({ "agentplate-stalled-agent": true }),
|
|
1017
|
+
_triage: triageAlways("extend"),
|
|
1018
|
+
_nudge: nudgeTracker().nudge,
|
|
1019
|
+
_eventStore: eventStore,
|
|
1020
|
+
});
|
|
1021
|
+
} finally {
|
|
1022
|
+
eventStore.close();
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
const events = readEvents(tempRoot);
|
|
1026
|
+
expect(events.length).toBeGreaterThanOrEqual(1);
|
|
1027
|
+
|
|
1028
|
+
const warnEvent = events.find((e) => {
|
|
1029
|
+
if (!e.data) return false;
|
|
1030
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
1031
|
+
return data.type === "escalation" && data.escalationLevel === 0;
|
|
1032
|
+
});
|
|
1033
|
+
expect(warnEvent).toBeDefined();
|
|
1034
|
+
expect(warnEvent?.eventType).toBe("custom");
|
|
1035
|
+
expect(warnEvent?.level).toBe("warn");
|
|
1036
|
+
expect(warnEvent?.agentName).toBe("stalled-agent");
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
test("escalation level 1 (nudge) records event with delivered status", async () => {
|
|
1040
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1041
|
+
const stalledSince = new Date(Date.now() - 70_000).toISOString();
|
|
1042
|
+
const session = makeSession({
|
|
1043
|
+
agentName: "stalled-agent",
|
|
1044
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
1045
|
+
state: "stalled",
|
|
1046
|
+
lastActivity: staleActivity,
|
|
1047
|
+
escalationLevel: 0,
|
|
1048
|
+
stalledSince,
|
|
1049
|
+
});
|
|
1050
|
+
|
|
1051
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1052
|
+
|
|
1053
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
1054
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
1055
|
+
const nudgeMock = nudgeTracker();
|
|
1056
|
+
|
|
1057
|
+
try {
|
|
1058
|
+
await runDaemonTick({
|
|
1059
|
+
root: tempRoot,
|
|
1060
|
+
...THRESHOLDS,
|
|
1061
|
+
nudgeIntervalMs: 60_000,
|
|
1062
|
+
_tmux: tmuxWithLiveness({ "agentplate-stalled-agent": true }),
|
|
1063
|
+
_triage: triageAlways("extend"),
|
|
1064
|
+
_nudge: nudgeMock.nudge,
|
|
1065
|
+
_eventStore: eventStore,
|
|
1066
|
+
});
|
|
1067
|
+
} finally {
|
|
1068
|
+
eventStore.close();
|
|
1069
|
+
}
|
|
1070
|
+
|
|
1071
|
+
const events = readEvents(tempRoot);
|
|
1072
|
+
const nudgeEvent = events.find((e) => {
|
|
1073
|
+
if (!e.data) return false;
|
|
1074
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
1075
|
+
return data.type === "nudge" && data.escalationLevel === 1;
|
|
1076
|
+
});
|
|
1077
|
+
expect(nudgeEvent).toBeDefined();
|
|
1078
|
+
expect(nudgeEvent?.eventType).toBe("custom");
|
|
1079
|
+
expect(nudgeEvent?.level).toBe("warn");
|
|
1080
|
+
|
|
1081
|
+
const nudgeData = JSON.parse(nudgeEvent?.data ?? "{}") as Record<string, unknown>;
|
|
1082
|
+
expect(nudgeData.delivered).toBe(true);
|
|
1083
|
+
});
|
|
1084
|
+
|
|
1085
|
+
test("escalation level 2 (triage) records event with verdict", async () => {
|
|
1086
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1087
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
1088
|
+
const session = makeSession({
|
|
1089
|
+
agentName: "stalled-agent",
|
|
1090
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
1091
|
+
state: "stalled",
|
|
1092
|
+
lastActivity: staleActivity,
|
|
1093
|
+
escalationLevel: 1,
|
|
1094
|
+
stalledSince,
|
|
1095
|
+
});
|
|
1096
|
+
|
|
1097
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1098
|
+
|
|
1099
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
1100
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
1101
|
+
|
|
1102
|
+
try {
|
|
1103
|
+
await runDaemonTick({
|
|
1104
|
+
root: tempRoot,
|
|
1105
|
+
...THRESHOLDS,
|
|
1106
|
+
nudgeIntervalMs: 60_000,
|
|
1107
|
+
tier1Enabled: true,
|
|
1108
|
+
_tmux: tmuxWithLiveness({ "agentplate-stalled-agent": true }),
|
|
1109
|
+
_triage: triageAlways("extend"),
|
|
1110
|
+
_nudge: nudgeTracker().nudge,
|
|
1111
|
+
_eventStore: eventStore,
|
|
1112
|
+
});
|
|
1113
|
+
} finally {
|
|
1114
|
+
eventStore.close();
|
|
1115
|
+
}
|
|
1116
|
+
|
|
1117
|
+
const events = readEvents(tempRoot);
|
|
1118
|
+
const triageEvent = events.find((e) => {
|
|
1119
|
+
if (!e.data) return false;
|
|
1120
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
1121
|
+
return data.type === "triage" && data.escalationLevel === 2;
|
|
1122
|
+
});
|
|
1123
|
+
expect(triageEvent).toBeDefined();
|
|
1124
|
+
expect(triageEvent?.eventType).toBe("custom");
|
|
1125
|
+
expect(triageEvent?.level).toBe("warn");
|
|
1126
|
+
|
|
1127
|
+
const triageData = JSON.parse(triageEvent?.data ?? "{}") as Record<string, unknown>;
|
|
1128
|
+
expect(triageData.verdict).toBe("extend");
|
|
1129
|
+
});
|
|
1130
|
+
|
|
1131
|
+
test("triage fallback event includes triageFailed: true when _triage returns TriageResult with fallback", async () => {
|
|
1132
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1133
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
1134
|
+
const session = makeSession({
|
|
1135
|
+
agentName: "stalled-agent",
|
|
1136
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
1137
|
+
state: "stalled",
|
|
1138
|
+
lastActivity: staleActivity,
|
|
1139
|
+
escalationLevel: 1,
|
|
1140
|
+
stalledSince,
|
|
1141
|
+
});
|
|
1142
|
+
|
|
1143
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1144
|
+
|
|
1145
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
1146
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
1147
|
+
|
|
1148
|
+
try {
|
|
1149
|
+
await runDaemonTick({
|
|
1150
|
+
root: tempRoot,
|
|
1151
|
+
...THRESHOLDS,
|
|
1152
|
+
nudgeIntervalMs: 60_000,
|
|
1153
|
+
tier1Enabled: true,
|
|
1154
|
+
_tmux: tmuxWithLiveness({ "agentplate-stalled-agent": true }),
|
|
1155
|
+
_triage: async () => ({
|
|
1156
|
+
verdict: "extend" as const,
|
|
1157
|
+
fallback: true,
|
|
1158
|
+
reason: "Claude unavailable",
|
|
1159
|
+
}),
|
|
1160
|
+
_nudge: nudgeTracker().nudge,
|
|
1161
|
+
_eventStore: eventStore,
|
|
1162
|
+
});
|
|
1163
|
+
} finally {
|
|
1164
|
+
eventStore.close();
|
|
1165
|
+
}
|
|
1166
|
+
|
|
1167
|
+
const events = readEvents(tempRoot);
|
|
1168
|
+
const triageEvent = events.find((e) => {
|
|
1169
|
+
if (!e.data) return false;
|
|
1170
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
1171
|
+
return data.type === "triage" && data.escalationLevel === 2;
|
|
1172
|
+
});
|
|
1173
|
+
expect(triageEvent).toBeDefined();
|
|
1174
|
+
|
|
1175
|
+
const triageData = JSON.parse(triageEvent?.data ?? "{}") as Record<string, unknown>;
|
|
1176
|
+
expect(triageData.verdict).toBe("extend");
|
|
1177
|
+
expect(triageData.triageFailed).toBe(true);
|
|
1178
|
+
});
|
|
1179
|
+
|
|
1180
|
+
test("escalation level 3 (terminate) records event with level=error", async () => {
|
|
1181
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1182
|
+
const stalledSince = new Date(Date.now() - 200_000).toISOString();
|
|
1183
|
+
const session = makeSession({
|
|
1184
|
+
agentName: "doomed-agent",
|
|
1185
|
+
tmuxSession: "agentplate-doomed-agent",
|
|
1186
|
+
state: "stalled",
|
|
1187
|
+
lastActivity: staleActivity,
|
|
1188
|
+
escalationLevel: 2,
|
|
1189
|
+
stalledSince,
|
|
1190
|
+
});
|
|
1191
|
+
|
|
1192
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1193
|
+
|
|
1194
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
1195
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
1196
|
+
|
|
1197
|
+
try {
|
|
1198
|
+
await runDaemonTick({
|
|
1199
|
+
root: tempRoot,
|
|
1200
|
+
...THRESHOLDS,
|
|
1201
|
+
nudgeIntervalMs: 60_000,
|
|
1202
|
+
_tmux: tmuxWithLiveness({ "agentplate-doomed-agent": true }),
|
|
1203
|
+
_triage: triageAlways("extend"),
|
|
1204
|
+
_nudge: nudgeTracker().nudge,
|
|
1205
|
+
_eventStore: eventStore,
|
|
1206
|
+
});
|
|
1207
|
+
} finally {
|
|
1208
|
+
eventStore.close();
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
const events = readEvents(tempRoot);
|
|
1212
|
+
const terminateEvent = events.find((e) => {
|
|
1213
|
+
if (!e.data) return false;
|
|
1214
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
1215
|
+
return data.type === "escalation" && data.escalationLevel === 3;
|
|
1216
|
+
});
|
|
1217
|
+
expect(terminateEvent).toBeDefined();
|
|
1218
|
+
expect(terminateEvent?.eventType).toBe("custom");
|
|
1219
|
+
expect(terminateEvent?.level).toBe("error");
|
|
1220
|
+
|
|
1221
|
+
const terminateData = JSON.parse(terminateEvent?.data ?? "{}") as Record<string, unknown>;
|
|
1222
|
+
expect(terminateData.action).toBe("terminate");
|
|
1223
|
+
});
|
|
1224
|
+
|
|
1225
|
+
test("run_id is included in events when current-run.txt exists", async () => {
|
|
1226
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1227
|
+
const session = makeSession({
|
|
1228
|
+
agentName: "stalled-agent",
|
|
1229
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
1230
|
+
state: "working",
|
|
1231
|
+
lastActivity: staleActivity,
|
|
1232
|
+
});
|
|
1233
|
+
|
|
1234
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1235
|
+
|
|
1236
|
+
// Write a current-run.txt
|
|
1237
|
+
const runId = "run-2026-02-13T10-00-00-000Z";
|
|
1238
|
+
await setActiveRun(tempRoot, runId);
|
|
1239
|
+
|
|
1240
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
1241
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
1242
|
+
|
|
1243
|
+
try {
|
|
1244
|
+
await runDaemonTick({
|
|
1245
|
+
root: tempRoot,
|
|
1246
|
+
...THRESHOLDS,
|
|
1247
|
+
nudgeIntervalMs: 60_000,
|
|
1248
|
+
_tmux: tmuxWithLiveness({ "agentplate-stalled-agent": true }),
|
|
1249
|
+
_triage: triageAlways("extend"),
|
|
1250
|
+
_nudge: nudgeTracker().nudge,
|
|
1251
|
+
_eventStore: eventStore,
|
|
1252
|
+
});
|
|
1253
|
+
} finally {
|
|
1254
|
+
eventStore.close();
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
const events = readEvents(tempRoot);
|
|
1258
|
+
expect(events.length).toBeGreaterThanOrEqual(1);
|
|
1259
|
+
const event = events[0];
|
|
1260
|
+
expect(event?.runId).toBe(runId);
|
|
1261
|
+
});
|
|
1262
|
+
|
|
1263
|
+
test("daemon continues normally when _eventStore is null", async () => {
|
|
1264
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1265
|
+
const session = makeSession({
|
|
1266
|
+
agentName: "stalled-agent",
|
|
1267
|
+
tmuxSession: "agentplate-stalled-agent",
|
|
1268
|
+
state: "working",
|
|
1269
|
+
lastActivity: staleActivity,
|
|
1270
|
+
});
|
|
1271
|
+
|
|
1272
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1273
|
+
|
|
1274
|
+
const checks: HealthCheck[] = [];
|
|
1275
|
+
|
|
1276
|
+
// Inject null EventStore — daemon should still work fine
|
|
1277
|
+
await runDaemonTick({
|
|
1278
|
+
root: tempRoot,
|
|
1279
|
+
...THRESHOLDS,
|
|
1280
|
+
nudgeIntervalMs: 60_000,
|
|
1281
|
+
onHealthCheck: (c) => checks.push(c),
|
|
1282
|
+
_tmux: tmuxWithLiveness({ "agentplate-stalled-agent": true }),
|
|
1283
|
+
_triage: triageAlways("extend"),
|
|
1284
|
+
_nudge: nudgeTracker().nudge,
|
|
1285
|
+
_eventStore: null,
|
|
1286
|
+
});
|
|
1287
|
+
|
|
1288
|
+
// Daemon should still produce health checks even without EventStore
|
|
1289
|
+
expect(checks).toHaveLength(1);
|
|
1290
|
+
expect(checks[0]?.action).toBe("escalate");
|
|
1291
|
+
});
|
|
1292
|
+
});
|
|
1293
|
+
|
|
1294
|
+
// === Loam failure recording tests ===
|
|
1295
|
+
|
|
1296
|
+
describe("daemon loam failure recording", () => {
|
|
1297
|
+
let tempRoot: string;
|
|
1298
|
+
|
|
1299
|
+
beforeEach(async () => {
|
|
1300
|
+
tempRoot = await createTempRoot();
|
|
1301
|
+
});
|
|
1302
|
+
|
|
1303
|
+
afterEach(async () => {
|
|
1304
|
+
await cleanupTempDir(tempRoot);
|
|
1305
|
+
});
|
|
1306
|
+
|
|
1307
|
+
/** Track calls to the recordFailure mock. */
|
|
1308
|
+
interface FailureRecord {
|
|
1309
|
+
root: string;
|
|
1310
|
+
session: AgentSession;
|
|
1311
|
+
reason: string;
|
|
1312
|
+
tier: 0 | 1;
|
|
1313
|
+
triageSuggestion?: string;
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
function failureTracker(): {
|
|
1317
|
+
calls: FailureRecord[];
|
|
1318
|
+
recordFailure: (
|
|
1319
|
+
root: string,
|
|
1320
|
+
session: AgentSession,
|
|
1321
|
+
reason: string,
|
|
1322
|
+
tier: 0 | 1,
|
|
1323
|
+
triageSuggestion?: string,
|
|
1324
|
+
) => Promise<void>;
|
|
1325
|
+
} {
|
|
1326
|
+
const calls: FailureRecord[] = [];
|
|
1327
|
+
return {
|
|
1328
|
+
calls,
|
|
1329
|
+
async recordFailure(root, session, reason, tier, triageSuggestion) {
|
|
1330
|
+
calls.push({ root, session, reason, tier, triageSuggestion });
|
|
1331
|
+
},
|
|
1332
|
+
};
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
test("Tier 0: recordFailure called when action=terminate (process death)", async () => {
|
|
1336
|
+
const session = makeSession({
|
|
1337
|
+
agentName: "dying-agent",
|
|
1338
|
+
capability: "builder",
|
|
1339
|
+
taskId: "task-123",
|
|
1340
|
+
tmuxSession: "agentplate-dying-agent",
|
|
1341
|
+
state: "working",
|
|
1342
|
+
lastActivity: new Date().toISOString(),
|
|
1343
|
+
});
|
|
1344
|
+
|
|
1345
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1346
|
+
|
|
1347
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-dying-agent": false });
|
|
1348
|
+
const failureMock = failureTracker();
|
|
1349
|
+
|
|
1350
|
+
await runDaemonTick({
|
|
1351
|
+
root: tempRoot,
|
|
1352
|
+
...THRESHOLDS,
|
|
1353
|
+
_tmux: tmuxMock,
|
|
1354
|
+
_triage: triageAlways("extend"),
|
|
1355
|
+
_nudge: nudgeTracker().nudge,
|
|
1356
|
+
_recordFailure: failureMock.recordFailure,
|
|
1357
|
+
});
|
|
1358
|
+
|
|
1359
|
+
// recordFailure should be called with Tier 0
|
|
1360
|
+
expect(failureMock.calls).toHaveLength(1);
|
|
1361
|
+
expect(failureMock.calls[0]?.tier).toBe(0);
|
|
1362
|
+
expect(failureMock.calls[0]?.session.agentName).toBe("dying-agent");
|
|
1363
|
+
expect(failureMock.calls[0]?.session.capability).toBe("builder");
|
|
1364
|
+
expect(failureMock.calls[0]?.session.taskId).toBe("task-123");
|
|
1365
|
+
// Reason should be either the reconciliationNote or default "Process terminated"
|
|
1366
|
+
expect(failureMock.calls[0]?.reason).toBeDefined();
|
|
1367
|
+
});
|
|
1368
|
+
|
|
1369
|
+
test("Tier 1: recordFailure called when triage returns terminate", async () => {
|
|
1370
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1371
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
1372
|
+
const session = makeSession({
|
|
1373
|
+
agentName: "triaged-agent",
|
|
1374
|
+
capability: "scout",
|
|
1375
|
+
taskId: "task-456",
|
|
1376
|
+
tmuxSession: "agentplate-triaged-agent",
|
|
1377
|
+
state: "stalled",
|
|
1378
|
+
lastActivity: staleActivity,
|
|
1379
|
+
escalationLevel: 1,
|
|
1380
|
+
stalledSince,
|
|
1381
|
+
});
|
|
1382
|
+
|
|
1383
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1384
|
+
|
|
1385
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-triaged-agent": true });
|
|
1386
|
+
const failureMock = failureTracker();
|
|
1387
|
+
|
|
1388
|
+
await runDaemonTick({
|
|
1389
|
+
root: tempRoot,
|
|
1390
|
+
...THRESHOLDS,
|
|
1391
|
+
nudgeIntervalMs: 60_000,
|
|
1392
|
+
tier1Enabled: true,
|
|
1393
|
+
_tmux: tmuxMock,
|
|
1394
|
+
_triage: triageAlways("terminate"),
|
|
1395
|
+
_nudge: nudgeTracker().nudge,
|
|
1396
|
+
_recordFailure: failureMock.recordFailure,
|
|
1397
|
+
});
|
|
1398
|
+
|
|
1399
|
+
// recordFailure should be called with Tier 1 and triage verdict
|
|
1400
|
+
expect(failureMock.calls).toHaveLength(1);
|
|
1401
|
+
expect(failureMock.calls[0]?.tier).toBe(1);
|
|
1402
|
+
expect(failureMock.calls[0]?.session.agentName).toBe("triaged-agent");
|
|
1403
|
+
expect(failureMock.calls[0]?.session.capability).toBe("scout");
|
|
1404
|
+
expect(failureMock.calls[0]?.session.taskId).toBe("task-456");
|
|
1405
|
+
expect(failureMock.calls[0]?.triageSuggestion).toBe("terminate");
|
|
1406
|
+
expect(failureMock.calls[0]?.reason).toContain("AI triage");
|
|
1407
|
+
});
|
|
1408
|
+
|
|
1409
|
+
test("recordFailure not called when triage returns retry", async () => {
|
|
1410
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1411
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
1412
|
+
const session = makeSession({
|
|
1413
|
+
agentName: "retry-agent",
|
|
1414
|
+
tmuxSession: "agentplate-retry-agent",
|
|
1415
|
+
state: "stalled",
|
|
1416
|
+
lastActivity: staleActivity,
|
|
1417
|
+
escalationLevel: 1,
|
|
1418
|
+
stalledSince,
|
|
1419
|
+
});
|
|
1420
|
+
|
|
1421
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1422
|
+
|
|
1423
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-retry-agent": true });
|
|
1424
|
+
const failureMock = failureTracker();
|
|
1425
|
+
|
|
1426
|
+
await runDaemonTick({
|
|
1427
|
+
root: tempRoot,
|
|
1428
|
+
...THRESHOLDS,
|
|
1429
|
+
nudgeIntervalMs: 60_000,
|
|
1430
|
+
tier1Enabled: true,
|
|
1431
|
+
_tmux: tmuxMock,
|
|
1432
|
+
_triage: triageAlways("retry"),
|
|
1433
|
+
_nudge: nudgeTracker().nudge,
|
|
1434
|
+
_recordFailure: failureMock.recordFailure,
|
|
1435
|
+
});
|
|
1436
|
+
|
|
1437
|
+
// recordFailure should NOT be called for retry verdict
|
|
1438
|
+
expect(failureMock.calls).toHaveLength(0);
|
|
1439
|
+
});
|
|
1440
|
+
|
|
1441
|
+
test("recordFailure not called when triage returns extend", async () => {
|
|
1442
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1443
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
1444
|
+
const session = makeSession({
|
|
1445
|
+
agentName: "extend-agent",
|
|
1446
|
+
tmuxSession: "agentplate-extend-agent",
|
|
1447
|
+
state: "stalled",
|
|
1448
|
+
lastActivity: staleActivity,
|
|
1449
|
+
escalationLevel: 1,
|
|
1450
|
+
stalledSince,
|
|
1451
|
+
});
|
|
1452
|
+
|
|
1453
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1454
|
+
|
|
1455
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-extend-agent": true });
|
|
1456
|
+
const failureMock = failureTracker();
|
|
1457
|
+
|
|
1458
|
+
await runDaemonTick({
|
|
1459
|
+
root: tempRoot,
|
|
1460
|
+
...THRESHOLDS,
|
|
1461
|
+
nudgeIntervalMs: 60_000,
|
|
1462
|
+
tier1Enabled: true,
|
|
1463
|
+
_tmux: tmuxMock,
|
|
1464
|
+
_triage: triageAlways("extend"),
|
|
1465
|
+
_nudge: nudgeTracker().nudge,
|
|
1466
|
+
_recordFailure: failureMock.recordFailure,
|
|
1467
|
+
});
|
|
1468
|
+
|
|
1469
|
+
// recordFailure should NOT be called for extend verdict
|
|
1470
|
+
expect(failureMock.calls).toHaveLength(0);
|
|
1471
|
+
});
|
|
1472
|
+
|
|
1473
|
+
test("recordFailure includes evidenceBead when taskId is present", async () => {
|
|
1474
|
+
const session = makeSession({
|
|
1475
|
+
agentName: "beaded-agent",
|
|
1476
|
+
capability: "builder",
|
|
1477
|
+
taskId: "task-789",
|
|
1478
|
+
tmuxSession: "agentplate-beaded-agent",
|
|
1479
|
+
state: "working",
|
|
1480
|
+
lastActivity: new Date().toISOString(),
|
|
1481
|
+
});
|
|
1482
|
+
|
|
1483
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1484
|
+
|
|
1485
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-beaded-agent": false });
|
|
1486
|
+
const failureMock = failureTracker();
|
|
1487
|
+
|
|
1488
|
+
await runDaemonTick({
|
|
1489
|
+
root: tempRoot,
|
|
1490
|
+
...THRESHOLDS,
|
|
1491
|
+
_tmux: tmuxMock,
|
|
1492
|
+
_triage: triageAlways("extend"),
|
|
1493
|
+
_nudge: nudgeTracker().nudge,
|
|
1494
|
+
_recordFailure: failureMock.recordFailure,
|
|
1495
|
+
});
|
|
1496
|
+
|
|
1497
|
+
expect(failureMock.calls).toHaveLength(1);
|
|
1498
|
+
expect(failureMock.calls[0]?.session.taskId).toBe("task-789");
|
|
1499
|
+
});
|
|
1500
|
+
|
|
1501
|
+
test("Tier 0: recordFailure called at escalation level 3+ (progressive termination)", async () => {
|
|
1502
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
1503
|
+
const stalledSince = new Date(Date.now() - 200_000).toISOString();
|
|
1504
|
+
const session = makeSession({
|
|
1505
|
+
agentName: "doomed-agent",
|
|
1506
|
+
capability: "builder",
|
|
1507
|
+
taskId: "task-999",
|
|
1508
|
+
tmuxSession: "agentplate-doomed-agent",
|
|
1509
|
+
state: "stalled",
|
|
1510
|
+
lastActivity: staleActivity,
|
|
1511
|
+
escalationLevel: 2,
|
|
1512
|
+
stalledSince,
|
|
1513
|
+
});
|
|
1514
|
+
|
|
1515
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
1516
|
+
|
|
1517
|
+
const tmuxMock = tmuxWithLiveness({ "agentplate-doomed-agent": true });
|
|
1518
|
+
const failureMock = failureTracker();
|
|
1519
|
+
|
|
1520
|
+
await runDaemonTick({
|
|
1521
|
+
root: tempRoot,
|
|
1522
|
+
...THRESHOLDS,
|
|
1523
|
+
nudgeIntervalMs: 60_000,
|
|
1524
|
+
_tmux: tmuxMock,
|
|
1525
|
+
_triage: triageAlways("extend"),
|
|
1526
|
+
_nudge: nudgeTracker().nudge,
|
|
1527
|
+
_recordFailure: failureMock.recordFailure,
|
|
1528
|
+
});
|
|
1529
|
+
|
|
1530
|
+
// recordFailure should be called with Tier 0 for progressive escalation
|
|
1531
|
+
expect(failureMock.calls).toHaveLength(1);
|
|
1532
|
+
expect(failureMock.calls[0]?.tier).toBe(0);
|
|
1533
|
+
expect(failureMock.calls[0]?.session.agentName).toBe("doomed-agent");
|
|
1534
|
+
expect(failureMock.calls[0]?.reason).toContain("Progressive escalation");
|
|
1535
|
+
});
|
|
1536
|
+
});
|
|
1537
|
+
|
|
1538
|
+
// === Run completion detection tests ===
|
|
1539
|
+
|
|
1540
|
+
describe("run completion detection", () => {
|
|
1541
|
+
const runId = "run-2026-02-18T15-00-00-000Z";
|
|
1542
|
+
|
|
1543
|
+
test("nudges coordinator when all workers completed", async () => {
|
|
1544
|
+
const sessions = [
|
|
1545
|
+
makeSession({
|
|
1546
|
+
id: "s1",
|
|
1547
|
+
agentName: "builder-one",
|
|
1548
|
+
capability: "builder",
|
|
1549
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1550
|
+
state: "completed",
|
|
1551
|
+
runId,
|
|
1552
|
+
lastActivity: new Date().toISOString(),
|
|
1553
|
+
}),
|
|
1554
|
+
makeSession({
|
|
1555
|
+
id: "s2",
|
|
1556
|
+
agentName: "builder-two",
|
|
1557
|
+
capability: "builder",
|
|
1558
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
1559
|
+
state: "completed",
|
|
1560
|
+
runId,
|
|
1561
|
+
lastActivity: new Date().toISOString(),
|
|
1562
|
+
}),
|
|
1563
|
+
makeSession({
|
|
1564
|
+
id: "s3",
|
|
1565
|
+
agentName: "coordinator",
|
|
1566
|
+
capability: "coordinator",
|
|
1567
|
+
tmuxSession: "agentplate-agent-fake-coordinator",
|
|
1568
|
+
state: "working",
|
|
1569
|
+
runId,
|
|
1570
|
+
lastActivity: new Date().toISOString(),
|
|
1571
|
+
}),
|
|
1572
|
+
];
|
|
1573
|
+
|
|
1574
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1575
|
+
await setActiveRun(tempRoot, runId);
|
|
1576
|
+
|
|
1577
|
+
const nudgeMock = nudgeTracker();
|
|
1578
|
+
|
|
1579
|
+
await runDaemonTick({
|
|
1580
|
+
root: tempRoot,
|
|
1581
|
+
...THRESHOLDS,
|
|
1582
|
+
_tmux: tmuxAllAlive(),
|
|
1583
|
+
_triage: triageAlways("extend"),
|
|
1584
|
+
_nudge: nudgeMock.nudge,
|
|
1585
|
+
_eventStore: null,
|
|
1586
|
+
});
|
|
1587
|
+
|
|
1588
|
+
// Filter to only run-completion nudges targeting the coordinator
|
|
1589
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
1590
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
1591
|
+
);
|
|
1592
|
+
expect(coordinatorNudges).toHaveLength(1);
|
|
1593
|
+
// The test creates builders, so the message should be builder-specific
|
|
1594
|
+
expect(coordinatorNudges[0]?.message).toContain("builder");
|
|
1595
|
+
expect(coordinatorNudges[0]?.message).toContain("Awaiting lead verification");
|
|
1596
|
+
});
|
|
1597
|
+
|
|
1598
|
+
test("does not nudge when some workers still active", async () => {
|
|
1599
|
+
const sessions = [
|
|
1600
|
+
makeSession({
|
|
1601
|
+
id: "s1",
|
|
1602
|
+
agentName: "builder-one",
|
|
1603
|
+
capability: "builder",
|
|
1604
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1605
|
+
state: "completed",
|
|
1606
|
+
runId,
|
|
1607
|
+
lastActivity: new Date().toISOString(),
|
|
1608
|
+
}),
|
|
1609
|
+
makeSession({
|
|
1610
|
+
id: "s2",
|
|
1611
|
+
agentName: "builder-two",
|
|
1612
|
+
capability: "builder",
|
|
1613
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
1614
|
+
state: "working",
|
|
1615
|
+
runId,
|
|
1616
|
+
lastActivity: new Date().toISOString(),
|
|
1617
|
+
}),
|
|
1618
|
+
];
|
|
1619
|
+
|
|
1620
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1621
|
+
await setActiveRun(tempRoot, runId);
|
|
1622
|
+
|
|
1623
|
+
const nudgeMock = nudgeTracker();
|
|
1624
|
+
|
|
1625
|
+
await runDaemonTick({
|
|
1626
|
+
root: tempRoot,
|
|
1627
|
+
...THRESHOLDS,
|
|
1628
|
+
_tmux: tmuxAllAlive(),
|
|
1629
|
+
_triage: triageAlways("extend"),
|
|
1630
|
+
_nudge: nudgeMock.nudge,
|
|
1631
|
+
_eventStore: null,
|
|
1632
|
+
});
|
|
1633
|
+
|
|
1634
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
1635
|
+
(c) => c.agentName === "coordinator" && c.message.includes("worker"),
|
|
1636
|
+
);
|
|
1637
|
+
expect(coordinatorNudges).toHaveLength(0);
|
|
1638
|
+
});
|
|
1639
|
+
|
|
1640
|
+
test("does not nudge when already notified (dedup marker)", async () => {
|
|
1641
|
+
const sessions = [
|
|
1642
|
+
makeSession({
|
|
1643
|
+
id: "s1",
|
|
1644
|
+
agentName: "builder-one",
|
|
1645
|
+
capability: "builder",
|
|
1646
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1647
|
+
state: "completed",
|
|
1648
|
+
runId,
|
|
1649
|
+
lastActivity: new Date().toISOString(),
|
|
1650
|
+
}),
|
|
1651
|
+
makeSession({
|
|
1652
|
+
id: "s2",
|
|
1653
|
+
agentName: "builder-two",
|
|
1654
|
+
capability: "builder",
|
|
1655
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
1656
|
+
state: "completed",
|
|
1657
|
+
runId,
|
|
1658
|
+
lastActivity: new Date().toISOString(),
|
|
1659
|
+
}),
|
|
1660
|
+
];
|
|
1661
|
+
|
|
1662
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1663
|
+
await setActiveRun(tempRoot, runId);
|
|
1664
|
+
// Pre-write dedup marker
|
|
1665
|
+
await Bun.write(join(tempRoot, ".agentplate", "run-complete-notified.txt"), runId);
|
|
1666
|
+
|
|
1667
|
+
const nudgeMock = nudgeTracker();
|
|
1668
|
+
|
|
1669
|
+
await runDaemonTick({
|
|
1670
|
+
root: tempRoot,
|
|
1671
|
+
...THRESHOLDS,
|
|
1672
|
+
_tmux: tmuxAllAlive(),
|
|
1673
|
+
_triage: triageAlways("extend"),
|
|
1674
|
+
_nudge: nudgeMock.nudge,
|
|
1675
|
+
_eventStore: null,
|
|
1676
|
+
});
|
|
1677
|
+
|
|
1678
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
1679
|
+
(c) => c.agentName === "coordinator" && c.message.includes("worker"),
|
|
1680
|
+
);
|
|
1681
|
+
expect(coordinatorNudges).toHaveLength(0);
|
|
1682
|
+
});
|
|
1683
|
+
|
|
1684
|
+
test("skips completion check when no run ID", async () => {
|
|
1685
|
+
const sessions = [
|
|
1686
|
+
makeSession({
|
|
1687
|
+
id: "s1",
|
|
1688
|
+
agentName: "builder-one",
|
|
1689
|
+
capability: "builder",
|
|
1690
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1691
|
+
state: "completed",
|
|
1692
|
+
runId,
|
|
1693
|
+
lastActivity: new Date().toISOString(),
|
|
1694
|
+
}),
|
|
1695
|
+
makeSession({
|
|
1696
|
+
id: "s2",
|
|
1697
|
+
agentName: "builder-two",
|
|
1698
|
+
capability: "builder",
|
|
1699
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
1700
|
+
state: "completed",
|
|
1701
|
+
runId,
|
|
1702
|
+
lastActivity: new Date().toISOString(),
|
|
1703
|
+
}),
|
|
1704
|
+
];
|
|
1705
|
+
|
|
1706
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1707
|
+
// Do NOT write current-run.txt
|
|
1708
|
+
|
|
1709
|
+
const nudgeMock = nudgeTracker();
|
|
1710
|
+
|
|
1711
|
+
await runDaemonTick({
|
|
1712
|
+
root: tempRoot,
|
|
1713
|
+
...THRESHOLDS,
|
|
1714
|
+
_tmux: tmuxAllAlive(),
|
|
1715
|
+
_triage: triageAlways("extend"),
|
|
1716
|
+
_nudge: nudgeMock.nudge,
|
|
1717
|
+
_eventStore: null,
|
|
1718
|
+
});
|
|
1719
|
+
|
|
1720
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
1721
|
+
(c) => c.agentName === "coordinator" && c.message.includes("worker"),
|
|
1722
|
+
);
|
|
1723
|
+
expect(coordinatorNudges).toHaveLength(0);
|
|
1724
|
+
});
|
|
1725
|
+
|
|
1726
|
+
test("ignores coordinator and monitor sessions for completion check", async () => {
|
|
1727
|
+
const sessions = [
|
|
1728
|
+
makeSession({
|
|
1729
|
+
id: "s1",
|
|
1730
|
+
agentName: "coordinator",
|
|
1731
|
+
capability: "coordinator",
|
|
1732
|
+
tmuxSession: "agentplate-agent-fake-coordinator",
|
|
1733
|
+
state: "working",
|
|
1734
|
+
runId,
|
|
1735
|
+
lastActivity: new Date().toISOString(),
|
|
1736
|
+
}),
|
|
1737
|
+
makeSession({
|
|
1738
|
+
id: "s2",
|
|
1739
|
+
agentName: "monitor",
|
|
1740
|
+
capability: "monitor",
|
|
1741
|
+
tmuxSession: "agentplate-agent-fake-monitor",
|
|
1742
|
+
state: "working",
|
|
1743
|
+
runId,
|
|
1744
|
+
lastActivity: new Date().toISOString(),
|
|
1745
|
+
}),
|
|
1746
|
+
makeSession({
|
|
1747
|
+
id: "s3",
|
|
1748
|
+
agentName: "builder-one",
|
|
1749
|
+
capability: "builder",
|
|
1750
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1751
|
+
state: "completed",
|
|
1752
|
+
runId,
|
|
1753
|
+
lastActivity: new Date().toISOString(),
|
|
1754
|
+
}),
|
|
1755
|
+
makeSession({
|
|
1756
|
+
id: "s4",
|
|
1757
|
+
agentName: "builder-two",
|
|
1758
|
+
capability: "builder",
|
|
1759
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
1760
|
+
state: "completed",
|
|
1761
|
+
runId,
|
|
1762
|
+
lastActivity: new Date().toISOString(),
|
|
1763
|
+
}),
|
|
1764
|
+
];
|
|
1765
|
+
|
|
1766
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1767
|
+
await setActiveRun(tempRoot, runId);
|
|
1768
|
+
|
|
1769
|
+
const nudgeMock = nudgeTracker();
|
|
1770
|
+
|
|
1771
|
+
await runDaemonTick({
|
|
1772
|
+
root: tempRoot,
|
|
1773
|
+
...THRESHOLDS,
|
|
1774
|
+
_tmux: tmuxAllAlive(),
|
|
1775
|
+
_triage: triageAlways("extend"),
|
|
1776
|
+
_nudge: nudgeMock.nudge,
|
|
1777
|
+
_eventStore: null,
|
|
1778
|
+
});
|
|
1779
|
+
|
|
1780
|
+
// Nudge IS sent because coordinator/monitor are excluded from worker count
|
|
1781
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
1782
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
1783
|
+
);
|
|
1784
|
+
expect(coordinatorNudges).toHaveLength(1);
|
|
1785
|
+
// The test creates builders, so the message should be builder-specific
|
|
1786
|
+
expect(coordinatorNudges[0]?.message).toContain("builder");
|
|
1787
|
+
expect(coordinatorNudges[0]?.message).toContain("Awaiting lead verification");
|
|
1788
|
+
});
|
|
1789
|
+
|
|
1790
|
+
test("does not nudge when no worker sessions in run", async () => {
|
|
1791
|
+
const sessions = [
|
|
1792
|
+
makeSession({
|
|
1793
|
+
id: "s1",
|
|
1794
|
+
agentName: "coordinator",
|
|
1795
|
+
capability: "coordinator",
|
|
1796
|
+
tmuxSession: "agentplate-agent-fake-coordinator",
|
|
1797
|
+
state: "working",
|
|
1798
|
+
runId,
|
|
1799
|
+
lastActivity: new Date().toISOString(),
|
|
1800
|
+
}),
|
|
1801
|
+
makeSession({
|
|
1802
|
+
id: "s2",
|
|
1803
|
+
agentName: "monitor",
|
|
1804
|
+
capability: "monitor",
|
|
1805
|
+
tmuxSession: "agentplate-agent-fake-monitor",
|
|
1806
|
+
state: "working",
|
|
1807
|
+
runId,
|
|
1808
|
+
lastActivity: new Date().toISOString(),
|
|
1809
|
+
}),
|
|
1810
|
+
];
|
|
1811
|
+
|
|
1812
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1813
|
+
await setActiveRun(tempRoot, runId);
|
|
1814
|
+
|
|
1815
|
+
const nudgeMock = nudgeTracker();
|
|
1816
|
+
|
|
1817
|
+
await runDaemonTick({
|
|
1818
|
+
root: tempRoot,
|
|
1819
|
+
...THRESHOLDS,
|
|
1820
|
+
_tmux: tmuxAllAlive(),
|
|
1821
|
+
_triage: triageAlways("extend"),
|
|
1822
|
+
_nudge: nudgeMock.nudge,
|
|
1823
|
+
_eventStore: null,
|
|
1824
|
+
});
|
|
1825
|
+
|
|
1826
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
1827
|
+
(c) => c.agentName === "coordinator" && c.message.includes("worker"),
|
|
1828
|
+
);
|
|
1829
|
+
expect(coordinatorNudges).toHaveLength(0);
|
|
1830
|
+
});
|
|
1831
|
+
|
|
1832
|
+
test("records run_complete event when all workers done", async () => {
|
|
1833
|
+
const sessions = [
|
|
1834
|
+
makeSession({
|
|
1835
|
+
id: "s1",
|
|
1836
|
+
agentName: "builder-one",
|
|
1837
|
+
capability: "builder",
|
|
1838
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1839
|
+
state: "completed",
|
|
1840
|
+
runId,
|
|
1841
|
+
lastActivity: new Date().toISOString(),
|
|
1842
|
+
}),
|
|
1843
|
+
makeSession({
|
|
1844
|
+
id: "s2",
|
|
1845
|
+
agentName: "builder-two",
|
|
1846
|
+
capability: "builder",
|
|
1847
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
1848
|
+
state: "completed",
|
|
1849
|
+
runId,
|
|
1850
|
+
lastActivity: new Date().toISOString(),
|
|
1851
|
+
}),
|
|
1852
|
+
];
|
|
1853
|
+
|
|
1854
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1855
|
+
await setActiveRun(tempRoot, runId);
|
|
1856
|
+
|
|
1857
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
1858
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
1859
|
+
|
|
1860
|
+
try {
|
|
1861
|
+
await runDaemonTick({
|
|
1862
|
+
root: tempRoot,
|
|
1863
|
+
...THRESHOLDS,
|
|
1864
|
+
_tmux: tmuxAllAlive(),
|
|
1865
|
+
_triage: triageAlways("extend"),
|
|
1866
|
+
_nudge: nudgeTracker().nudge,
|
|
1867
|
+
_eventStore: eventStore,
|
|
1868
|
+
});
|
|
1869
|
+
} finally {
|
|
1870
|
+
eventStore.close();
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
// Read events back
|
|
1874
|
+
const store = createEventStore(eventsDbPath);
|
|
1875
|
+
try {
|
|
1876
|
+
const events = store.getTimeline({ since: "2000-01-01T00:00:00Z" });
|
|
1877
|
+
const runCompleteEvent = events.find((e) => {
|
|
1878
|
+
if (!e.data) return false;
|
|
1879
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
1880
|
+
return data.type === "run_complete";
|
|
1881
|
+
});
|
|
1882
|
+
expect(runCompleteEvent).toBeDefined();
|
|
1883
|
+
expect(runCompleteEvent?.level).toBe("info");
|
|
1884
|
+
expect(runCompleteEvent?.agentName).toBe("watchdog");
|
|
1885
|
+
} finally {
|
|
1886
|
+
store.close();
|
|
1887
|
+
}
|
|
1888
|
+
});
|
|
1889
|
+
|
|
1890
|
+
test("writes dedup marker after nudging", async () => {
|
|
1891
|
+
const sessions = [
|
|
1892
|
+
makeSession({
|
|
1893
|
+
id: "s1",
|
|
1894
|
+
agentName: "builder-one",
|
|
1895
|
+
capability: "builder",
|
|
1896
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1897
|
+
state: "completed",
|
|
1898
|
+
runId,
|
|
1899
|
+
lastActivity: new Date().toISOString(),
|
|
1900
|
+
}),
|
|
1901
|
+
makeSession({
|
|
1902
|
+
id: "s2",
|
|
1903
|
+
agentName: "builder-two",
|
|
1904
|
+
capability: "builder",
|
|
1905
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
1906
|
+
state: "completed",
|
|
1907
|
+
runId,
|
|
1908
|
+
lastActivity: new Date().toISOString(),
|
|
1909
|
+
}),
|
|
1910
|
+
];
|
|
1911
|
+
|
|
1912
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1913
|
+
await setActiveRun(tempRoot, runId);
|
|
1914
|
+
|
|
1915
|
+
await runDaemonTick({
|
|
1916
|
+
root: tempRoot,
|
|
1917
|
+
...THRESHOLDS,
|
|
1918
|
+
_tmux: tmuxAllAlive(),
|
|
1919
|
+
_triage: triageAlways("extend"),
|
|
1920
|
+
_nudge: nudgeTracker().nudge,
|
|
1921
|
+
_eventStore: null,
|
|
1922
|
+
});
|
|
1923
|
+
|
|
1924
|
+
// Verify dedup marker was written
|
|
1925
|
+
const markerFile = Bun.file(join(tempRoot, ".agentplate", "run-complete-notified.txt"));
|
|
1926
|
+
expect(await markerFile.exists()).toBe(true);
|
|
1927
|
+
const markerContent = await markerFile.text();
|
|
1928
|
+
expect(markerContent.trim()).toBe(runId);
|
|
1929
|
+
});
|
|
1930
|
+
|
|
1931
|
+
test("scout-only completion sends phase-appropriate message", async () => {
|
|
1932
|
+
const sessions = [
|
|
1933
|
+
makeSession({
|
|
1934
|
+
id: "s1",
|
|
1935
|
+
agentName: "scout-one",
|
|
1936
|
+
capability: "scout",
|
|
1937
|
+
tmuxSession: "agentplate-agent-fake-scout-one",
|
|
1938
|
+
state: "completed",
|
|
1939
|
+
runId,
|
|
1940
|
+
lastActivity: new Date().toISOString(),
|
|
1941
|
+
}),
|
|
1942
|
+
makeSession({
|
|
1943
|
+
id: "s2",
|
|
1944
|
+
agentName: "scout-two",
|
|
1945
|
+
capability: "scout",
|
|
1946
|
+
tmuxSession: "agentplate-agent-fake-scout-two",
|
|
1947
|
+
state: "completed",
|
|
1948
|
+
runId,
|
|
1949
|
+
lastActivity: new Date().toISOString(),
|
|
1950
|
+
}),
|
|
1951
|
+
];
|
|
1952
|
+
|
|
1953
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
1954
|
+
await setActiveRun(tempRoot, runId);
|
|
1955
|
+
|
|
1956
|
+
const nudgeMock = nudgeTracker();
|
|
1957
|
+
|
|
1958
|
+
await runDaemonTick({
|
|
1959
|
+
root: tempRoot,
|
|
1960
|
+
...THRESHOLDS,
|
|
1961
|
+
_tmux: tmuxAllAlive(),
|
|
1962
|
+
_triage: triageAlways("extend"),
|
|
1963
|
+
_nudge: nudgeMock.nudge,
|
|
1964
|
+
_eventStore: null,
|
|
1965
|
+
});
|
|
1966
|
+
|
|
1967
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
1968
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
1969
|
+
);
|
|
1970
|
+
expect(coordinatorNudges).toHaveLength(1);
|
|
1971
|
+
expect(coordinatorNudges[0]?.message).toContain("scout");
|
|
1972
|
+
expect(coordinatorNudges[0]?.message).toContain("next phase");
|
|
1973
|
+
// Must NOT say "merge/cleanup" for scouts
|
|
1974
|
+
expect(coordinatorNudges[0]?.message).not.toContain("merge/cleanup");
|
|
1975
|
+
});
|
|
1976
|
+
|
|
1977
|
+
test("mixed capabilities send generic message with breakdown", async () => {
|
|
1978
|
+
const sessions = [
|
|
1979
|
+
makeSession({
|
|
1980
|
+
id: "s1",
|
|
1981
|
+
agentName: "scout-one",
|
|
1982
|
+
capability: "scout",
|
|
1983
|
+
tmuxSession: "agentplate-agent-fake-scout-one",
|
|
1984
|
+
state: "completed",
|
|
1985
|
+
runId,
|
|
1986
|
+
lastActivity: new Date().toISOString(),
|
|
1987
|
+
}),
|
|
1988
|
+
makeSession({
|
|
1989
|
+
id: "s2",
|
|
1990
|
+
agentName: "builder-one",
|
|
1991
|
+
capability: "builder",
|
|
1992
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
1993
|
+
state: "completed",
|
|
1994
|
+
runId,
|
|
1995
|
+
lastActivity: new Date().toISOString(),
|
|
1996
|
+
}),
|
|
1997
|
+
];
|
|
1998
|
+
|
|
1999
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2000
|
+
await setActiveRun(tempRoot, runId);
|
|
2001
|
+
|
|
2002
|
+
const nudgeMock = nudgeTracker();
|
|
2003
|
+
|
|
2004
|
+
await runDaemonTick({
|
|
2005
|
+
root: tempRoot,
|
|
2006
|
+
...THRESHOLDS,
|
|
2007
|
+
_tmux: tmuxAllAlive(),
|
|
2008
|
+
_triage: triageAlways("extend"),
|
|
2009
|
+
_nudge: nudgeMock.nudge,
|
|
2010
|
+
_eventStore: null,
|
|
2011
|
+
});
|
|
2012
|
+
|
|
2013
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
2014
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
2015
|
+
);
|
|
2016
|
+
expect(coordinatorNudges).toHaveLength(1);
|
|
2017
|
+
expect(coordinatorNudges[0]?.message).toContain("(builder, scout)");
|
|
2018
|
+
expect(coordinatorNudges[0]?.message).toContain("next steps");
|
|
2019
|
+
});
|
|
2020
|
+
|
|
2021
|
+
test("reviewer-only completion sends review-specific message", async () => {
|
|
2022
|
+
const sessions = [
|
|
2023
|
+
makeSession({
|
|
2024
|
+
id: "s1",
|
|
2025
|
+
agentName: "reviewer-one",
|
|
2026
|
+
capability: "reviewer",
|
|
2027
|
+
tmuxSession: "agentplate-agent-fake-reviewer-one",
|
|
2028
|
+
state: "completed",
|
|
2029
|
+
runId,
|
|
2030
|
+
lastActivity: new Date().toISOString(),
|
|
2031
|
+
}),
|
|
2032
|
+
];
|
|
2033
|
+
|
|
2034
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2035
|
+
await setActiveRun(tempRoot, runId);
|
|
2036
|
+
|
|
2037
|
+
const nudgeMock = nudgeTracker();
|
|
2038
|
+
|
|
2039
|
+
await runDaemonTick({
|
|
2040
|
+
root: tempRoot,
|
|
2041
|
+
...THRESHOLDS,
|
|
2042
|
+
_tmux: tmuxAllAlive(),
|
|
2043
|
+
_triage: triageAlways("extend"),
|
|
2044
|
+
_nudge: nudgeMock.nudge,
|
|
2045
|
+
_eventStore: null,
|
|
2046
|
+
});
|
|
2047
|
+
|
|
2048
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
2049
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
2050
|
+
);
|
|
2051
|
+
expect(coordinatorNudges).toHaveLength(1);
|
|
2052
|
+
expect(coordinatorNudges[0]?.message).toContain("reviewer");
|
|
2053
|
+
expect(coordinatorNudges[0]?.message).toContain("Reviews done");
|
|
2054
|
+
});
|
|
2055
|
+
|
|
2056
|
+
test("run_complete event includes capabilities and phase fields", async () => {
|
|
2057
|
+
const sessions = [
|
|
2058
|
+
makeSession({
|
|
2059
|
+
id: "s1",
|
|
2060
|
+
agentName: "builder-one",
|
|
2061
|
+
capability: "builder",
|
|
2062
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
2063
|
+
state: "completed",
|
|
2064
|
+
runId,
|
|
2065
|
+
lastActivity: new Date().toISOString(),
|
|
2066
|
+
}),
|
|
2067
|
+
];
|
|
2068
|
+
|
|
2069
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2070
|
+
await setActiveRun(tempRoot, runId);
|
|
2071
|
+
|
|
2072
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
2073
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
2074
|
+
|
|
2075
|
+
try {
|
|
2076
|
+
await runDaemonTick({
|
|
2077
|
+
root: tempRoot,
|
|
2078
|
+
...THRESHOLDS,
|
|
2079
|
+
_tmux: tmuxAllAlive(),
|
|
2080
|
+
_triage: triageAlways("extend"),
|
|
2081
|
+
_nudge: nudgeTracker().nudge,
|
|
2082
|
+
_eventStore: eventStore,
|
|
2083
|
+
});
|
|
2084
|
+
} finally {
|
|
2085
|
+
eventStore.close();
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
const store = createEventStore(eventsDbPath);
|
|
2089
|
+
try {
|
|
2090
|
+
const events = store.getTimeline({ since: "2000-01-01T00:00:00Z" });
|
|
2091
|
+
const runCompleteEvent = events.find((e) => {
|
|
2092
|
+
if (!e.data) return false;
|
|
2093
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
2094
|
+
return data.type === "run_complete";
|
|
2095
|
+
});
|
|
2096
|
+
expect(runCompleteEvent).toBeDefined();
|
|
2097
|
+
const data = JSON.parse(runCompleteEvent?.data ?? "{}") as Record<string, unknown>;
|
|
2098
|
+
expect(data.capabilities).toEqual(["builder"]);
|
|
2099
|
+
expect(data.phase).toBe("builder");
|
|
2100
|
+
} finally {
|
|
2101
|
+
store.close();
|
|
2102
|
+
}
|
|
2103
|
+
});
|
|
2104
|
+
|
|
2105
|
+
// agentplate-e130: a run that mixes `completed` and `zombie` workers must
|
|
2106
|
+
// still notify the coordinator. Before the fix, the every-completed predicate
|
|
2107
|
+
// stranded the coordinator forever whenever the watchdog killed any worker.
|
|
2108
|
+
test("nudges coordinator when workers are a mix of completed and zombie", async () => {
|
|
2109
|
+
const sessions = [
|
|
2110
|
+
makeSession({
|
|
2111
|
+
id: "s1",
|
|
2112
|
+
agentName: "builder-one",
|
|
2113
|
+
capability: "builder",
|
|
2114
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
2115
|
+
state: "completed",
|
|
2116
|
+
runId,
|
|
2117
|
+
lastActivity: new Date().toISOString(),
|
|
2118
|
+
}),
|
|
2119
|
+
makeSession({
|
|
2120
|
+
id: "s2",
|
|
2121
|
+
agentName: "builder-two",
|
|
2122
|
+
capability: "builder",
|
|
2123
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
2124
|
+
state: "zombie",
|
|
2125
|
+
runId,
|
|
2126
|
+
lastActivity: new Date().toISOString(),
|
|
2127
|
+
}),
|
|
2128
|
+
];
|
|
2129
|
+
|
|
2130
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2131
|
+
await setActiveRun(tempRoot, runId);
|
|
2132
|
+
|
|
2133
|
+
const nudgeMock = nudgeTracker();
|
|
2134
|
+
|
|
2135
|
+
await runDaemonTick({
|
|
2136
|
+
root: tempRoot,
|
|
2137
|
+
...THRESHOLDS,
|
|
2138
|
+
_tmux: tmuxAllAlive(),
|
|
2139
|
+
_triage: triageAlways("extend"),
|
|
2140
|
+
_nudge: nudgeMock.nudge,
|
|
2141
|
+
_eventStore: null,
|
|
2142
|
+
});
|
|
2143
|
+
|
|
2144
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
2145
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
2146
|
+
);
|
|
2147
|
+
expect(coordinatorNudges).toHaveLength(1);
|
|
2148
|
+
expect(coordinatorNudges[0]?.message).toContain("have terminated");
|
|
2149
|
+
expect(coordinatorNudges[0]?.message).toContain("(1 completed, 1 zombie)");
|
|
2150
|
+
});
|
|
2151
|
+
|
|
2152
|
+
test("nudges coordinator when every worker is zombie", async () => {
|
|
2153
|
+
const sessions = [
|
|
2154
|
+
makeSession({
|
|
2155
|
+
id: "s1",
|
|
2156
|
+
agentName: "builder-one",
|
|
2157
|
+
capability: "builder",
|
|
2158
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
2159
|
+
state: "zombie",
|
|
2160
|
+
runId,
|
|
2161
|
+
lastActivity: new Date().toISOString(),
|
|
2162
|
+
}),
|
|
2163
|
+
makeSession({
|
|
2164
|
+
id: "s2",
|
|
2165
|
+
agentName: "builder-two",
|
|
2166
|
+
capability: "builder",
|
|
2167
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
2168
|
+
state: "zombie",
|
|
2169
|
+
runId,
|
|
2170
|
+
lastActivity: new Date().toISOString(),
|
|
2171
|
+
}),
|
|
2172
|
+
];
|
|
2173
|
+
|
|
2174
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2175
|
+
await setActiveRun(tempRoot, runId);
|
|
2176
|
+
|
|
2177
|
+
const nudgeMock = nudgeTracker();
|
|
2178
|
+
|
|
2179
|
+
await runDaemonTick({
|
|
2180
|
+
root: tempRoot,
|
|
2181
|
+
...THRESHOLDS,
|
|
2182
|
+
_tmux: tmuxAllAlive(),
|
|
2183
|
+
_triage: triageAlways("extend"),
|
|
2184
|
+
_nudge: nudgeMock.nudge,
|
|
2185
|
+
_eventStore: null,
|
|
2186
|
+
});
|
|
2187
|
+
|
|
2188
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
2189
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
2190
|
+
);
|
|
2191
|
+
expect(coordinatorNudges).toHaveLength(1);
|
|
2192
|
+
expect(coordinatorNudges[0]?.message).toContain("(0 completed, 2 zombie)");
|
|
2193
|
+
});
|
|
2194
|
+
|
|
2195
|
+
test("does not nudge when a working worker remains alongside a zombie", async () => {
|
|
2196
|
+
const sessions = [
|
|
2197
|
+
makeSession({
|
|
2198
|
+
id: "s1",
|
|
2199
|
+
agentName: "builder-one",
|
|
2200
|
+
capability: "builder",
|
|
2201
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
2202
|
+
state: "zombie",
|
|
2203
|
+
runId,
|
|
2204
|
+
lastActivity: new Date().toISOString(),
|
|
2205
|
+
}),
|
|
2206
|
+
makeSession({
|
|
2207
|
+
id: "s2",
|
|
2208
|
+
agentName: "builder-two",
|
|
2209
|
+
capability: "builder",
|
|
2210
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
2211
|
+
state: "working",
|
|
2212
|
+
runId,
|
|
2213
|
+
lastActivity: new Date().toISOString(),
|
|
2214
|
+
}),
|
|
2215
|
+
];
|
|
2216
|
+
|
|
2217
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2218
|
+
await setActiveRun(tempRoot, runId);
|
|
2219
|
+
|
|
2220
|
+
const nudgeMock = nudgeTracker();
|
|
2221
|
+
|
|
2222
|
+
await runDaemonTick({
|
|
2223
|
+
root: tempRoot,
|
|
2224
|
+
...THRESHOLDS,
|
|
2225
|
+
_tmux: tmuxAllAlive(),
|
|
2226
|
+
_triage: triageAlways("extend"),
|
|
2227
|
+
_nudge: nudgeMock.nudge,
|
|
2228
|
+
_eventStore: null,
|
|
2229
|
+
});
|
|
2230
|
+
|
|
2231
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
2232
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
2233
|
+
);
|
|
2234
|
+
expect(coordinatorNudges).toHaveLength(0);
|
|
2235
|
+
});
|
|
2236
|
+
|
|
2237
|
+
test("run_complete event with zombies records zombieAgents and warn level", async () => {
|
|
2238
|
+
const sessions = [
|
|
2239
|
+
makeSession({
|
|
2240
|
+
id: "s1",
|
|
2241
|
+
agentName: "builder-one",
|
|
2242
|
+
capability: "builder",
|
|
2243
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
2244
|
+
state: "completed",
|
|
2245
|
+
runId,
|
|
2246
|
+
lastActivity: new Date().toISOString(),
|
|
2247
|
+
}),
|
|
2248
|
+
makeSession({
|
|
2249
|
+
id: "s2",
|
|
2250
|
+
agentName: "builder-two",
|
|
2251
|
+
capability: "builder",
|
|
2252
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
2253
|
+
state: "zombie",
|
|
2254
|
+
runId,
|
|
2255
|
+
lastActivity: new Date().toISOString(),
|
|
2256
|
+
}),
|
|
2257
|
+
];
|
|
2258
|
+
|
|
2259
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2260
|
+
await setActiveRun(tempRoot, runId);
|
|
2261
|
+
|
|
2262
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
2263
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
2264
|
+
|
|
2265
|
+
try {
|
|
2266
|
+
await runDaemonTick({
|
|
2267
|
+
root: tempRoot,
|
|
2268
|
+
...THRESHOLDS,
|
|
2269
|
+
_tmux: tmuxAllAlive(),
|
|
2270
|
+
_triage: triageAlways("extend"),
|
|
2271
|
+
_nudge: nudgeTracker().nudge,
|
|
2272
|
+
_eventStore: eventStore,
|
|
2273
|
+
});
|
|
2274
|
+
} finally {
|
|
2275
|
+
eventStore.close();
|
|
2276
|
+
}
|
|
2277
|
+
|
|
2278
|
+
const store = createEventStore(eventsDbPath);
|
|
2279
|
+
try {
|
|
2280
|
+
const events = store.getTimeline({ since: "2000-01-01T00:00:00Z" });
|
|
2281
|
+
const runCompleteEvent = events.find((e) => {
|
|
2282
|
+
if (!e.data) return false;
|
|
2283
|
+
const data = JSON.parse(e.data) as Record<string, unknown>;
|
|
2284
|
+
return data.type === "run_complete";
|
|
2285
|
+
});
|
|
2286
|
+
expect(runCompleteEvent).toBeDefined();
|
|
2287
|
+
expect(runCompleteEvent?.level).toBe("warn");
|
|
2288
|
+
const data = JSON.parse(runCompleteEvent?.data ?? "{}") as Record<string, unknown>;
|
|
2289
|
+
expect(data.completedAgents).toEqual(["builder-one"]);
|
|
2290
|
+
expect(data.zombieAgents).toEqual(["builder-two"]);
|
|
2291
|
+
expect(data.workerCount).toBe(2);
|
|
2292
|
+
} finally {
|
|
2293
|
+
store.close();
|
|
2294
|
+
}
|
|
2295
|
+
});
|
|
2296
|
+
|
|
2297
|
+
test("missing current-run.txt: warns once, skips run-completion check (agentplate-87bf)", async () => {
|
|
2298
|
+
const sessions = [
|
|
2299
|
+
makeSession({
|
|
2300
|
+
id: "s1",
|
|
2301
|
+
agentName: "builder-one",
|
|
2302
|
+
capability: "builder",
|
|
2303
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
2304
|
+
state: "completed",
|
|
2305
|
+
runId,
|
|
2306
|
+
lastActivity: new Date().toISOString(),
|
|
2307
|
+
}),
|
|
2308
|
+
makeSession({
|
|
2309
|
+
id: "s2",
|
|
2310
|
+
agentName: "builder-two",
|
|
2311
|
+
capability: "builder",
|
|
2312
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
2313
|
+
state: "completed",
|
|
2314
|
+
runId,
|
|
2315
|
+
lastActivity: new Date().toISOString(),
|
|
2316
|
+
}),
|
|
2317
|
+
];
|
|
2318
|
+
|
|
2319
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2320
|
+
// Deliberately do NOT call setActiveRun — current-run.txt absent.
|
|
2321
|
+
|
|
2322
|
+
const nudgeMock = nudgeTracker();
|
|
2323
|
+
const warnState = freshRunIdWarnState();
|
|
2324
|
+
|
|
2325
|
+
const stderrWrites: string[] = [];
|
|
2326
|
+
const originalStderrWrite = process.stderr.write.bind(process.stderr);
|
|
2327
|
+
process.stderr.write = ((chunk: unknown, ...rest: unknown[]) => {
|
|
2328
|
+
stderrWrites.push(typeof chunk === "string" ? chunk : String(chunk));
|
|
2329
|
+
return originalStderrWrite(chunk as string, ...(rest as []));
|
|
2330
|
+
}) as typeof process.stderr.write;
|
|
2331
|
+
|
|
2332
|
+
try {
|
|
2333
|
+
await runDaemonTick({
|
|
2334
|
+
root: tempRoot,
|
|
2335
|
+
...THRESHOLDS,
|
|
2336
|
+
_tmux: tmuxAllAlive(),
|
|
2337
|
+
_triage: triageAlways("extend"),
|
|
2338
|
+
_nudge: nudgeMock.nudge,
|
|
2339
|
+
_eventStore: null,
|
|
2340
|
+
_runIdWarnState: warnState,
|
|
2341
|
+
});
|
|
2342
|
+
|
|
2343
|
+
// Tick again to confirm the warning dedupes for the same cause.
|
|
2344
|
+
await runDaemonTick({
|
|
2345
|
+
root: tempRoot,
|
|
2346
|
+
...THRESHOLDS,
|
|
2347
|
+
_tmux: tmuxAllAlive(),
|
|
2348
|
+
_triage: triageAlways("extend"),
|
|
2349
|
+
_nudge: nudgeMock.nudge,
|
|
2350
|
+
_eventStore: null,
|
|
2351
|
+
_runIdWarnState: warnState,
|
|
2352
|
+
});
|
|
2353
|
+
} finally {
|
|
2354
|
+
process.stderr.write = originalStderrWrite;
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
// Run-completion skip is observable: no coordinator nudge was sent.
|
|
2358
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
2359
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
2360
|
+
);
|
|
2361
|
+
expect(coordinatorNudges).toHaveLength(0);
|
|
2362
|
+
|
|
2363
|
+
// Warning logged exactly once across the two ticks.
|
|
2364
|
+
expect(warnState.missingFileWarned).toBe(true);
|
|
2365
|
+
const missingWarnings = stderrWrites.filter((w) =>
|
|
2366
|
+
w.includes("[WATCHDOG] current-run.txt missing"),
|
|
2367
|
+
);
|
|
2368
|
+
expect(missingWarnings).toHaveLength(1);
|
|
2369
|
+
});
|
|
2370
|
+
|
|
2371
|
+
test("stale current-run.txt id (no row in runs table): warns once per id, skips check (agentplate-87bf)", async () => {
|
|
2372
|
+
const staleId = "run-stale-2026-01-01T00-00-00-000Z";
|
|
2373
|
+
const sessions = [
|
|
2374
|
+
makeSession({
|
|
2375
|
+
id: "s1",
|
|
2376
|
+
agentName: "builder-one",
|
|
2377
|
+
capability: "builder",
|
|
2378
|
+
tmuxSession: "agentplate-agent-fake-builder-one",
|
|
2379
|
+
state: "completed",
|
|
2380
|
+
runId: staleId,
|
|
2381
|
+
lastActivity: new Date().toISOString(),
|
|
2382
|
+
}),
|
|
2383
|
+
makeSession({
|
|
2384
|
+
id: "s2",
|
|
2385
|
+
agentName: "builder-two",
|
|
2386
|
+
capability: "builder",
|
|
2387
|
+
tmuxSession: "agentplate-agent-fake-builder-two",
|
|
2388
|
+
state: "completed",
|
|
2389
|
+
runId: staleId,
|
|
2390
|
+
lastActivity: new Date().toISOString(),
|
|
2391
|
+
}),
|
|
2392
|
+
];
|
|
2393
|
+
|
|
2394
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
2395
|
+
// Write current-run.txt but DO NOT seed the runs table — the lookup
|
|
2396
|
+
// will return null, exercising the stale-id branch.
|
|
2397
|
+
await Bun.write(join(tempRoot, ".agentplate", "current-run.txt"), staleId);
|
|
2398
|
+
|
|
2399
|
+
const nudgeMock = nudgeTracker();
|
|
2400
|
+
const warnState = freshRunIdWarnState();
|
|
2401
|
+
|
|
2402
|
+
const stderrWrites: string[] = [];
|
|
2403
|
+
const originalStderrWrite = process.stderr.write.bind(process.stderr);
|
|
2404
|
+
process.stderr.write = ((chunk: unknown, ...rest: unknown[]) => {
|
|
2405
|
+
stderrWrites.push(typeof chunk === "string" ? chunk : String(chunk));
|
|
2406
|
+
return originalStderrWrite(chunk as string, ...(rest as []));
|
|
2407
|
+
}) as typeof process.stderr.write;
|
|
2408
|
+
|
|
2409
|
+
try {
|
|
2410
|
+
await runDaemonTick({
|
|
2411
|
+
root: tempRoot,
|
|
2412
|
+
...THRESHOLDS,
|
|
2413
|
+
_tmux: tmuxAllAlive(),
|
|
2414
|
+
_triage: triageAlways("extend"),
|
|
2415
|
+
_nudge: nudgeMock.nudge,
|
|
2416
|
+
_eventStore: null,
|
|
2417
|
+
_runIdWarnState: warnState,
|
|
2418
|
+
});
|
|
2419
|
+
|
|
2420
|
+
await runDaemonTick({
|
|
2421
|
+
root: tempRoot,
|
|
2422
|
+
...THRESHOLDS,
|
|
2423
|
+
_tmux: tmuxAllAlive(),
|
|
2424
|
+
_triage: triageAlways("extend"),
|
|
2425
|
+
_nudge: nudgeMock.nudge,
|
|
2426
|
+
_eventStore: null,
|
|
2427
|
+
_runIdWarnState: warnState,
|
|
2428
|
+
});
|
|
2429
|
+
} finally {
|
|
2430
|
+
process.stderr.write = originalStderrWrite;
|
|
2431
|
+
}
|
|
2432
|
+
|
|
2433
|
+
// Run-completion skip is observable: no coordinator nudge.
|
|
2434
|
+
const coordinatorNudges = nudgeMock.calls.filter(
|
|
2435
|
+
(c) => c.agentName === "coordinator" && c.message.includes("WATCHDOG"),
|
|
2436
|
+
);
|
|
2437
|
+
expect(coordinatorNudges).toHaveLength(0);
|
|
2438
|
+
|
|
2439
|
+
// Stale-id was recorded once, missing-file path was NOT triggered.
|
|
2440
|
+
expect(warnState.unknownIds.has(staleId)).toBe(true);
|
|
2441
|
+
expect(warnState.missingFileWarned).toBe(false);
|
|
2442
|
+
const staleWarnings = stderrWrites.filter((w) =>
|
|
2443
|
+
w.includes(`points to unknown run "${staleId}"`),
|
|
2444
|
+
);
|
|
2445
|
+
expect(staleWarnings).toHaveLength(1);
|
|
2446
|
+
});
|
|
2447
|
+
});
|
|
2448
|
+
|
|
2449
|
+
// === buildCompletionMessage unit tests ===
|
|
2450
|
+
|
|
2451
|
+
describe("buildCompletionMessage", () => {
|
|
2452
|
+
const testRunId = "run-test-123";
|
|
2453
|
+
|
|
2454
|
+
test("all scouts → contains 'scout' and 'Ready for next phase'", () => {
|
|
2455
|
+
const sessions = [
|
|
2456
|
+
makeSession({ capability: "scout", agentName: "scout-1" }),
|
|
2457
|
+
makeSession({ capability: "scout", agentName: "scout-2" }),
|
|
2458
|
+
];
|
|
2459
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2460
|
+
expect(msg).toContain("scout");
|
|
2461
|
+
expect(msg).toContain("Ready for next phase");
|
|
2462
|
+
expect(msg).not.toContain("merge/cleanup");
|
|
2463
|
+
});
|
|
2464
|
+
|
|
2465
|
+
test("all builders → contains 'builder' and 'Awaiting lead verification' (not merge authorization)", () => {
|
|
2466
|
+
const sessions = [
|
|
2467
|
+
makeSession({ capability: "builder", agentName: "builder-1" }),
|
|
2468
|
+
makeSession({ capability: "builder", agentName: "builder-2" }),
|
|
2469
|
+
];
|
|
2470
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2471
|
+
expect(msg).toContain("builder");
|
|
2472
|
+
expect(msg).toContain("Awaiting lead verification");
|
|
2473
|
+
expect(msg).not.toContain("merge/cleanup");
|
|
2474
|
+
});
|
|
2475
|
+
|
|
2476
|
+
test("all reviewers → contains 'reviewer' and 'Reviews done'", () => {
|
|
2477
|
+
const sessions = [makeSession({ capability: "reviewer", agentName: "reviewer-1" })];
|
|
2478
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2479
|
+
expect(msg).toContain("reviewer");
|
|
2480
|
+
expect(msg).toContain("Reviews done");
|
|
2481
|
+
});
|
|
2482
|
+
|
|
2483
|
+
test("all leads → contains 'lead' and 'Ready for merge/cleanup'", () => {
|
|
2484
|
+
const sessions = [makeSession({ capability: "lead", agentName: "lead-1" })];
|
|
2485
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2486
|
+
expect(msg).toContain("lead");
|
|
2487
|
+
expect(msg).toContain("Ready for merge/cleanup");
|
|
2488
|
+
});
|
|
2489
|
+
|
|
2490
|
+
test("all mergers → contains 'merger' and 'Merges done'", () => {
|
|
2491
|
+
const sessions = [makeSession({ capability: "merger", agentName: "merger-1" })];
|
|
2492
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2493
|
+
expect(msg).toContain("merger");
|
|
2494
|
+
expect(msg).toContain("Merges done");
|
|
2495
|
+
});
|
|
2496
|
+
|
|
2497
|
+
test("mixed capabilities → contains breakdown and 'Ready for next steps'", () => {
|
|
2498
|
+
const sessions = [
|
|
2499
|
+
makeSession({ capability: "scout", agentName: "scout-1" }),
|
|
2500
|
+
makeSession({ capability: "builder", agentName: "builder-1" }),
|
|
2501
|
+
];
|
|
2502
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2503
|
+
expect(msg).toContain("(builder, scout)");
|
|
2504
|
+
expect(msg).toContain("Ready for next steps");
|
|
2505
|
+
});
|
|
2506
|
+
|
|
2507
|
+
test("message includes the run ID", () => {
|
|
2508
|
+
const sessions = [makeSession({ capability: "builder", agentName: "builder-1" })];
|
|
2509
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2510
|
+
expect(msg).toContain(testRunId);
|
|
2511
|
+
});
|
|
2512
|
+
|
|
2513
|
+
test("message includes the worker count", () => {
|
|
2514
|
+
const sessions = [
|
|
2515
|
+
makeSession({ capability: "scout", agentName: "scout-1" }),
|
|
2516
|
+
makeSession({ capability: "scout", agentName: "scout-2" }),
|
|
2517
|
+
makeSession({ capability: "scout", agentName: "scout-3" }),
|
|
2518
|
+
];
|
|
2519
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2520
|
+
expect(msg).toContain("3");
|
|
2521
|
+
});
|
|
2522
|
+
|
|
2523
|
+
// agentplate-e130: zombie workers must surface in the message so the coordinator
|
|
2524
|
+
// reads "have terminated (...)" instead of being misled into "have completed".
|
|
2525
|
+
test("mix of completed and zombie workers → 'have terminated' with completed/zombie qualifier", () => {
|
|
2526
|
+
const sessions = [
|
|
2527
|
+
makeSession({ capability: "builder", agentName: "builder-1", state: "completed" }),
|
|
2528
|
+
makeSession({ capability: "builder", agentName: "builder-2", state: "zombie" }),
|
|
2529
|
+
makeSession({ capability: "builder", agentName: "builder-3", state: "completed" }),
|
|
2530
|
+
];
|
|
2531
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2532
|
+
expect(msg).toContain("have terminated");
|
|
2533
|
+
expect(msg).toContain("(2 completed, 1 zombie)");
|
|
2534
|
+
expect(msg).not.toContain("have completed");
|
|
2535
|
+
// Capability-specific suffix is preserved
|
|
2536
|
+
expect(msg).toContain("Awaiting lead verification");
|
|
2537
|
+
});
|
|
2538
|
+
|
|
2539
|
+
test("all-zombie batch → '(0 completed, N zombie)' qualifier", () => {
|
|
2540
|
+
const sessions = [
|
|
2541
|
+
makeSession({ capability: "scout", agentName: "scout-1", state: "zombie" }),
|
|
2542
|
+
makeSession({ capability: "scout", agentName: "scout-2", state: "zombie" }),
|
|
2543
|
+
];
|
|
2544
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2545
|
+
expect(msg).toContain("have terminated");
|
|
2546
|
+
expect(msg).toContain("(0 completed, 2 zombie)");
|
|
2547
|
+
expect(msg).toContain("Ready for next phase");
|
|
2548
|
+
});
|
|
2549
|
+
|
|
2550
|
+
test("mixed-capability batch with zombies includes both qualifier and capability breakdown", () => {
|
|
2551
|
+
const sessions = [
|
|
2552
|
+
makeSession({ capability: "scout", agentName: "scout-1", state: "completed" }),
|
|
2553
|
+
makeSession({ capability: "builder", agentName: "builder-1", state: "zombie" }),
|
|
2554
|
+
];
|
|
2555
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2556
|
+
expect(msg).toContain("have terminated");
|
|
2557
|
+
expect(msg).toContain("(1 completed, 1 zombie)");
|
|
2558
|
+
expect(msg).toContain("(builder, scout)");
|
|
2559
|
+
expect(msg).toContain("Ready for next steps");
|
|
2560
|
+
});
|
|
2561
|
+
|
|
2562
|
+
test("all-completed batch keeps existing 'have completed' phrasing (no zombie qualifier)", () => {
|
|
2563
|
+
const sessions = [
|
|
2564
|
+
makeSession({ capability: "builder", agentName: "builder-1", state: "completed" }),
|
|
2565
|
+
makeSession({ capability: "builder", agentName: "builder-2", state: "completed" }),
|
|
2566
|
+
];
|
|
2567
|
+
const msg = buildCompletionMessage(sessions, testRunId);
|
|
2568
|
+
expect(msg).toContain("have completed");
|
|
2569
|
+
expect(msg).not.toContain("have terminated");
|
|
2570
|
+
expect(msg).not.toContain("zombie");
|
|
2571
|
+
});
|
|
2572
|
+
});
|
|
2573
|
+
|
|
2574
|
+
// === Bug fix tests: headless agent kill blast radius + stale detection ===
|
|
2575
|
+
|
|
2576
|
+
describe("headless agent kill blast radius fix (Bug 1)", () => {
|
|
2577
|
+
/**
|
|
2578
|
+
* Track PID kill calls without spawning real processes.
|
|
2579
|
+
* Also surfaces killTree calls so tests can assert on them.
|
|
2580
|
+
*/
|
|
2581
|
+
function processTracker(): {
|
|
2582
|
+
isAlive: (pid: number) => boolean;
|
|
2583
|
+
killTree: (pid: number) => Promise<void>;
|
|
2584
|
+
killed: number[];
|
|
2585
|
+
} {
|
|
2586
|
+
const killed: number[] = [];
|
|
2587
|
+
return {
|
|
2588
|
+
isAlive: (pid: number) => {
|
|
2589
|
+
try {
|
|
2590
|
+
process.kill(pid, 0);
|
|
2591
|
+
return true;
|
|
2592
|
+
} catch {
|
|
2593
|
+
return false;
|
|
2594
|
+
}
|
|
2595
|
+
},
|
|
2596
|
+
killTree: async (pid: number) => {
|
|
2597
|
+
killed.push(pid);
|
|
2598
|
+
},
|
|
2599
|
+
killed,
|
|
2600
|
+
};
|
|
2601
|
+
}
|
|
2602
|
+
|
|
2603
|
+
test("headless agent at escalation level 3 kills PID, not tmux session", async () => {
|
|
2604
|
+
const nudgeIntervalMs = 60_000;
|
|
2605
|
+
// stalledSince is 4 intervals ago — expectedLevel = floor(4) = 4, clamped to MAX (3)
|
|
2606
|
+
const stalledSince = new Date(Date.now() - 4 * nudgeIntervalMs).toISOString();
|
|
2607
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
2608
|
+
|
|
2609
|
+
const session = makeSession({
|
|
2610
|
+
agentName: "headless-stalled",
|
|
2611
|
+
tmuxSession: "", // headless
|
|
2612
|
+
pid: process.pid, // alive PID — ZFC won't trigger direct terminate
|
|
2613
|
+
state: "stalled",
|
|
2614
|
+
lastActivity: staleActivity,
|
|
2615
|
+
escalationLevel: 2,
|
|
2616
|
+
stalledSince,
|
|
2617
|
+
});
|
|
2618
|
+
|
|
2619
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
2620
|
+
|
|
2621
|
+
const proc = processTracker();
|
|
2622
|
+
// tmux mock: isSessionAlive("") returns true — simulates prefix-match bug scenario
|
|
2623
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
2624
|
+
|
|
2625
|
+
await runDaemonTick({
|
|
2626
|
+
root: tempRoot,
|
|
2627
|
+
...THRESHOLDS,
|
|
2628
|
+
nudgeIntervalMs,
|
|
2629
|
+
tier1Enabled: false,
|
|
2630
|
+
_tmux: tmuxMock,
|
|
2631
|
+
_triage: triageAlways("extend"),
|
|
2632
|
+
_process: proc,
|
|
2633
|
+
_eventStore: null,
|
|
2634
|
+
_recordFailure: async () => {},
|
|
2635
|
+
_getConnection: () => undefined,
|
|
2636
|
+
_removeConnection: () => {},
|
|
2637
|
+
_tailerRegistry: new Map(),
|
|
2638
|
+
_findLatestStdoutLog: async () => null,
|
|
2639
|
+
});
|
|
2640
|
+
|
|
2641
|
+
// PID was killed via killTree, NOT via tmux killSession("")
|
|
2642
|
+
expect(proc.killed).toContain(process.pid);
|
|
2643
|
+
expect(tmuxMock.killed).not.toContain("");
|
|
2644
|
+
});
|
|
2645
|
+
|
|
2646
|
+
test("headless agent direct terminate kills PID, not tmux", async () => {
|
|
2647
|
+
// PID 999999 is virtually guaranteed not to exist — health check sees it as dead
|
|
2648
|
+
const deadPid = 999999;
|
|
2649
|
+
const session = makeSession({
|
|
2650
|
+
agentName: "headless-dead-pid",
|
|
2651
|
+
tmuxSession: "", // headless
|
|
2652
|
+
pid: deadPid,
|
|
2653
|
+
state: "working",
|
|
2654
|
+
lastActivity: new Date().toISOString(),
|
|
2655
|
+
});
|
|
2656
|
+
|
|
2657
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
2658
|
+
|
|
2659
|
+
const proc = processTracker();
|
|
2660
|
+
// tmux mock: isSessionAlive("") returns true — would kill everything without the fix
|
|
2661
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
2662
|
+
|
|
2663
|
+
await runDaemonTick({
|
|
2664
|
+
root: tempRoot,
|
|
2665
|
+
...THRESHOLDS,
|
|
2666
|
+
_tmux: tmuxMock,
|
|
2667
|
+
_triage: triageAlways("extend"),
|
|
2668
|
+
_process: proc,
|
|
2669
|
+
_eventStore: null,
|
|
2670
|
+
_recordFailure: async () => {},
|
|
2671
|
+
_getConnection: () => undefined,
|
|
2672
|
+
_removeConnection: () => {},
|
|
2673
|
+
_tailerRegistry: new Map(),
|
|
2674
|
+
_findLatestStdoutLog: async () => null,
|
|
2675
|
+
});
|
|
2676
|
+
|
|
2677
|
+
// Should have attempted PID kill, NOT tmux killSession("")
|
|
2678
|
+
expect(proc.killed).toContain(deadPid);
|
|
2679
|
+
expect(tmuxMock.killed).not.toContain("");
|
|
2680
|
+
});
|
|
2681
|
+
|
|
2682
|
+
test("triage terminate on headless agent kills PID, not tmux", async () => {
|
|
2683
|
+
const nudgeIntervalMs = 60_000;
|
|
2684
|
+
// stalledSince is 2.5 intervals ago — expectedLevel = floor(2.5) = 2 → triage fires
|
|
2685
|
+
const stalledSince = new Date(Date.now() - 2.5 * nudgeIntervalMs).toISOString();
|
|
2686
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
2687
|
+
|
|
2688
|
+
const session = makeSession({
|
|
2689
|
+
agentName: "headless-triage-terminate",
|
|
2690
|
+
tmuxSession: "", // headless
|
|
2691
|
+
pid: process.pid, // alive
|
|
2692
|
+
state: "stalled",
|
|
2693
|
+
lastActivity: staleActivity,
|
|
2694
|
+
escalationLevel: 1,
|
|
2695
|
+
stalledSince,
|
|
2696
|
+
});
|
|
2697
|
+
|
|
2698
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
2699
|
+
|
|
2700
|
+
const proc = processTracker();
|
|
2701
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
2702
|
+
|
|
2703
|
+
await runDaemonTick({
|
|
2704
|
+
root: tempRoot,
|
|
2705
|
+
...THRESHOLDS,
|
|
2706
|
+
nudgeIntervalMs,
|
|
2707
|
+
tier1Enabled: true,
|
|
2708
|
+
_tmux: tmuxMock,
|
|
2709
|
+
_triage: triageAlways("terminate"), // AI triage says terminate
|
|
2710
|
+
_nudge: nudgeTracker().nudge,
|
|
2711
|
+
_process: proc,
|
|
2712
|
+
_eventStore: null,
|
|
2713
|
+
_recordFailure: async () => {},
|
|
2714
|
+
_getConnection: () => undefined,
|
|
2715
|
+
_removeConnection: () => {},
|
|
2716
|
+
_tailerRegistry: new Map(),
|
|
2717
|
+
_findLatestStdoutLog: async () => null,
|
|
2718
|
+
});
|
|
2719
|
+
|
|
2720
|
+
// Should have killed the PID, not the tmux session
|
|
2721
|
+
expect(proc.killed).toContain(process.pid);
|
|
2722
|
+
expect(tmuxMock.killed).not.toContain("");
|
|
2723
|
+
});
|
|
2724
|
+
});
|
|
2725
|
+
|
|
2726
|
+
describe("headless agent stale detection via events.db (Bug 2)", () => {
|
|
2727
|
+
test("headless agent with recent events in events.db is not flagged stale", async () => {
|
|
2728
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
2729
|
+
|
|
2730
|
+
const session = makeSession({
|
|
2731
|
+
agentName: "headless-active",
|
|
2732
|
+
tmuxSession: "", // headless
|
|
2733
|
+
pid: process.pid, // alive
|
|
2734
|
+
state: "working",
|
|
2735
|
+
lastActivity: staleActivity, // stale — would trigger escalate without event fallback
|
|
2736
|
+
});
|
|
2737
|
+
|
|
2738
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
2739
|
+
|
|
2740
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
2741
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
2742
|
+
|
|
2743
|
+
try {
|
|
2744
|
+
// Insert a recent event for this agent (within the stale threshold window)
|
|
2745
|
+
eventStore.insert({
|
|
2746
|
+
runId: null,
|
|
2747
|
+
agentName: "headless-active",
|
|
2748
|
+
sessionId: null,
|
|
2749
|
+
eventType: "tool_end",
|
|
2750
|
+
toolName: "Read",
|
|
2751
|
+
toolArgs: null,
|
|
2752
|
+
toolDurationMs: 100,
|
|
2753
|
+
level: "info",
|
|
2754
|
+
data: null,
|
|
2755
|
+
});
|
|
2756
|
+
|
|
2757
|
+
const checks: HealthCheck[] = [];
|
|
2758
|
+
|
|
2759
|
+
await runDaemonTick({
|
|
2760
|
+
root: tempRoot,
|
|
2761
|
+
...THRESHOLDS,
|
|
2762
|
+
onHealthCheck: (c) => checks.push(c),
|
|
2763
|
+
_tmux: tmuxAllAlive(),
|
|
2764
|
+
_triage: triageAlways("extend"),
|
|
2765
|
+
_process: { isAlive: () => true, killTree: async () => {} },
|
|
2766
|
+
_eventStore: eventStore,
|
|
2767
|
+
_recordFailure: async () => {},
|
|
2768
|
+
_getConnection: () => undefined,
|
|
2769
|
+
_removeConnection: () => {},
|
|
2770
|
+
_tailerRegistry: new Map(),
|
|
2771
|
+
_findLatestStdoutLog: async () => null,
|
|
2772
|
+
});
|
|
2773
|
+
|
|
2774
|
+
// Recent events found — lastActivity was refreshed, agent is NOT stalled
|
|
2775
|
+
expect(checks).toHaveLength(1);
|
|
2776
|
+
expect(checks[0]?.action).toBe("none");
|
|
2777
|
+
expect(checks[0]?.state).toBe("working");
|
|
2778
|
+
|
|
2779
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
2780
|
+
expect(reloaded[0]?.state).toBe("working");
|
|
2781
|
+
} finally {
|
|
2782
|
+
eventStore.close();
|
|
2783
|
+
}
|
|
2784
|
+
});
|
|
2785
|
+
|
|
2786
|
+
test("headless agent with no recent events IS flagged stale", async () => {
|
|
2787
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
2788
|
+
|
|
2789
|
+
const session = makeSession({
|
|
2790
|
+
agentName: "headless-silent",
|
|
2791
|
+
tmuxSession: "", // headless
|
|
2792
|
+
pid: process.pid, // alive
|
|
2793
|
+
state: "working",
|
|
2794
|
+
lastActivity: staleActivity, // stale
|
|
2795
|
+
});
|
|
2796
|
+
|
|
2797
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
2798
|
+
|
|
2799
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
2800
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
2801
|
+
|
|
2802
|
+
try {
|
|
2803
|
+
// No events inserted for this agent — event fallback finds nothing
|
|
2804
|
+
|
|
2805
|
+
const checks: HealthCheck[] = [];
|
|
2806
|
+
|
|
2807
|
+
await runDaemonTick({
|
|
2808
|
+
root: tempRoot,
|
|
2809
|
+
...THRESHOLDS,
|
|
2810
|
+
onHealthCheck: (c) => checks.push(c),
|
|
2811
|
+
_tmux: tmuxAllAlive(),
|
|
2812
|
+
_triage: triageAlways("extend"),
|
|
2813
|
+
_process: { isAlive: () => true, killTree: async () => {} },
|
|
2814
|
+
_eventStore: eventStore,
|
|
2815
|
+
_recordFailure: async () => {},
|
|
2816
|
+
_getConnection: () => undefined,
|
|
2817
|
+
_removeConnection: () => {},
|
|
2818
|
+
_tailerRegistry: new Map(),
|
|
2819
|
+
_findLatestStdoutLog: async () => null,
|
|
2820
|
+
});
|
|
2821
|
+
|
|
2822
|
+
// No recent events — lastActivity stays stale, agent IS flagged stalled
|
|
2823
|
+
expect(checks).toHaveLength(1);
|
|
2824
|
+
expect(checks[0]?.action).toBe("escalate");
|
|
2825
|
+
} finally {
|
|
2826
|
+
eventStore.close();
|
|
2827
|
+
}
|
|
2828
|
+
});
|
|
2829
|
+
|
|
2830
|
+
test("spawn-per-turn worker (pid=null) is NOT flagged zombie when actively emitting events (agentplate-7a34)", async () => {
|
|
2831
|
+
// Repro: ap sling --capability lead → freshly slung headless lead has
|
|
2832
|
+
// tmuxSession='' AND pid=null (no persistent process between turns).
|
|
2833
|
+
// Previously the daemon's event-based liveness fallback was gated by
|
|
2834
|
+
// `pid !== null`, so spawn-per-turn workers' lastActivity was never
|
|
2835
|
+
// refreshed from events.db and they would flip to stalled / zombie
|
|
2836
|
+
// despite ap feed showing live tool activity.
|
|
2837
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
2838
|
+
|
|
2839
|
+
const session = makeSession({
|
|
2840
|
+
agentName: "spawn-per-turn-lead",
|
|
2841
|
+
capability: "lead",
|
|
2842
|
+
tmuxSession: "", // headless
|
|
2843
|
+
pid: null, // spawn-per-turn: no persistent process between turns
|
|
2844
|
+
state: "working",
|
|
2845
|
+
lastActivity: staleActivity, // stale — would flip without event fallback
|
|
2846
|
+
});
|
|
2847
|
+
|
|
2848
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
2849
|
+
|
|
2850
|
+
const eventsDbPath = join(tempRoot, ".agentplate", "events.db");
|
|
2851
|
+
const eventStore = createEventStore(eventsDbPath);
|
|
2852
|
+
|
|
2853
|
+
try {
|
|
2854
|
+
// Insert a recent tool event for this agent (matches ap feed activity)
|
|
2855
|
+
eventStore.insert({
|
|
2856
|
+
runId: null,
|
|
2857
|
+
agentName: "spawn-per-turn-lead",
|
|
2858
|
+
sessionId: null,
|
|
2859
|
+
eventType: "tool_end",
|
|
2860
|
+
toolName: "Edit",
|
|
2861
|
+
toolArgs: null,
|
|
2862
|
+
toolDurationMs: 50,
|
|
2863
|
+
level: "info",
|
|
2864
|
+
data: null,
|
|
2865
|
+
});
|
|
2866
|
+
|
|
2867
|
+
const checks: HealthCheck[] = [];
|
|
2868
|
+
|
|
2869
|
+
await runDaemonTick({
|
|
2870
|
+
root: tempRoot,
|
|
2871
|
+
...THRESHOLDS,
|
|
2872
|
+
onHealthCheck: (c) => checks.push(c),
|
|
2873
|
+
_tmux: tmuxAllAlive(),
|
|
2874
|
+
_triage: triageAlways("extend"),
|
|
2875
|
+
_process: { isAlive: () => true, killTree: async () => {} },
|
|
2876
|
+
_eventStore: eventStore,
|
|
2877
|
+
_recordFailure: async () => {},
|
|
2878
|
+
_getConnection: () => undefined,
|
|
2879
|
+
_removeConnection: () => {},
|
|
2880
|
+
_tailerRegistry: new Map(),
|
|
2881
|
+
_findLatestStdoutLog: async () => null,
|
|
2882
|
+
});
|
|
2883
|
+
|
|
2884
|
+
// lastActivity refreshed from events.db → spawn-per-turn evaluation
|
|
2885
|
+
// path keeps the agent active (action=none), NOT zombie. The
|
|
2886
|
+
// healthy classification reports `between_turns` (agentplate-3087)
|
|
2887
|
+
// for spawn-per-turn workers; the legacy `working` row stays at
|
|
2888
|
+
// `working` on disk because the matrix does not list `working` as
|
|
2889
|
+
// a predecessor of `between_turns` and the CAS rejects the write
|
|
2890
|
+
// (the substate cycle is reserved for the turn-runner).
|
|
2891
|
+
expect(checks).toHaveLength(1);
|
|
2892
|
+
expect(checks[0]?.action).toBe("none");
|
|
2893
|
+
expect(checks[0]?.state).toBe("between_turns");
|
|
2894
|
+
|
|
2895
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
2896
|
+
expect(reloaded[0]?.state).toBe("working");
|
|
2897
|
+
} finally {
|
|
2898
|
+
eventStore.close();
|
|
2899
|
+
}
|
|
2900
|
+
});
|
|
2901
|
+
});
|
|
2902
|
+
|
|
2903
|
+
// ============================================================
|
|
2904
|
+
// startDaemon() shutdown cleanup
|
|
2905
|
+
// ============================================================
|
|
2906
|
+
|
|
2907
|
+
describe("startDaemon() stop() cleans up tailer registry", () => {
|
|
2908
|
+
let tempRoot: string;
|
|
2909
|
+
|
|
2910
|
+
beforeEach(async () => {
|
|
2911
|
+
tempRoot = await createTempRoot();
|
|
2912
|
+
});
|
|
2913
|
+
|
|
2914
|
+
afterEach(async () => {
|
|
2915
|
+
await cleanupTempDir(tempRoot);
|
|
2916
|
+
});
|
|
2917
|
+
|
|
2918
|
+
test("stop() calls handle.stop() on all registry entries and empties the map", async () => {
|
|
2919
|
+
// Build a fake tailer registry with two entries.
|
|
2920
|
+
const stopped: Record<string, boolean> = { tailer1: false, tailer2: false };
|
|
2921
|
+
|
|
2922
|
+
const registry = new Map<string, { agentName: string; logPath: string; stop(): void }>([
|
|
2923
|
+
[
|
|
2924
|
+
"agent-one",
|
|
2925
|
+
{
|
|
2926
|
+
agentName: "agent-one",
|
|
2927
|
+
logPath: "/fake/one/stdout.log",
|
|
2928
|
+
stop: () => {
|
|
2929
|
+
stopped.tailer1 = true;
|
|
2930
|
+
},
|
|
2931
|
+
},
|
|
2932
|
+
],
|
|
2933
|
+
[
|
|
2934
|
+
"agent-two",
|
|
2935
|
+
{
|
|
2936
|
+
agentName: "agent-two",
|
|
2937
|
+
logPath: "/fake/two/stdout.log",
|
|
2938
|
+
stop: () => {
|
|
2939
|
+
stopped.tailer2 = true;
|
|
2940
|
+
},
|
|
2941
|
+
},
|
|
2942
|
+
],
|
|
2943
|
+
]);
|
|
2944
|
+
|
|
2945
|
+
// Use a long interval so the periodic tick never fires during this test.
|
|
2946
|
+
const daemon = startDaemon({
|
|
2947
|
+
root: tempRoot,
|
|
2948
|
+
intervalMs: 60_000,
|
|
2949
|
+
...THRESHOLDS,
|
|
2950
|
+
_tmux: { isSessionAlive: async () => false, killSession: async () => {} },
|
|
2951
|
+
_nudge: async () => ({ delivered: false }),
|
|
2952
|
+
_process: { isAlive: () => false, killTree: async () => {} },
|
|
2953
|
+
_triage: async () => "extend",
|
|
2954
|
+
_recordFailure: async () => {},
|
|
2955
|
+
_getConnection: () => undefined,
|
|
2956
|
+
_removeConnection: () => {},
|
|
2957
|
+
_eventStore: null,
|
|
2958
|
+
_mailStore: null,
|
|
2959
|
+
_tailerRegistry: registry,
|
|
2960
|
+
_tailerFactory: () => ({ agentName: "", logPath: "", stop: () => {} }),
|
|
2961
|
+
_findLatestStdoutLog: async () => null,
|
|
2962
|
+
});
|
|
2963
|
+
|
|
2964
|
+
// Allow the first (immediate) tick to settle.
|
|
2965
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 20));
|
|
2966
|
+
|
|
2967
|
+
daemon.stop();
|
|
2968
|
+
|
|
2969
|
+
expect(stopped.tailer1).toBe(true);
|
|
2970
|
+
expect(stopped.tailer2).toBe(true);
|
|
2971
|
+
expect(registry.size).toBe(0);
|
|
2972
|
+
});
|
|
2973
|
+
});
|
|
2974
|
+
|
|
2975
|
+
// ============================================================
|
|
2976
|
+
// RPC getState() timeout removes stale connection
|
|
2977
|
+
// ============================================================
|
|
2978
|
+
|
|
2979
|
+
describe("RPC getState() timeout removes stale connection", () => {
|
|
2980
|
+
test("_removeConnection is called when getState() rejects", async () => {
|
|
2981
|
+
const session = makeSession({
|
|
2982
|
+
agentName: "rpc-agent",
|
|
2983
|
+
tmuxSession: "", // headless
|
|
2984
|
+
pid: process.pid, // alive
|
|
2985
|
+
state: "working",
|
|
2986
|
+
lastActivity: new Date().toISOString(),
|
|
2987
|
+
});
|
|
2988
|
+
|
|
2989
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
2990
|
+
|
|
2991
|
+
const removedNames: string[] = [];
|
|
2992
|
+
|
|
2993
|
+
await runDaemonTick({
|
|
2994
|
+
root: tempRoot,
|
|
2995
|
+
...THRESHOLDS,
|
|
2996
|
+
_tmux: { isSessionAlive: async () => false, killSession: async () => {} },
|
|
2997
|
+
_triage: triageAlways("extend"),
|
|
2998
|
+
_process: { isAlive: () => true, killTree: async () => {} },
|
|
2999
|
+
_eventStore: null,
|
|
3000
|
+
_recordFailure: async () => {},
|
|
3001
|
+
_getConnection: (name: string) => {
|
|
3002
|
+
if (name !== "rpc-agent") return undefined;
|
|
3003
|
+
return {
|
|
3004
|
+
getState: () => Promise.reject(new Error("connection error")),
|
|
3005
|
+
sendPrompt: async () => {},
|
|
3006
|
+
followUp: async () => {},
|
|
3007
|
+
abort: async () => {},
|
|
3008
|
+
close: () => {},
|
|
3009
|
+
};
|
|
3010
|
+
},
|
|
3011
|
+
_removeConnection: (name: string) => {
|
|
3012
|
+
removedNames.push(name);
|
|
3013
|
+
},
|
|
3014
|
+
_tailerRegistry: new Map(),
|
|
3015
|
+
_findLatestStdoutLog: async () => null,
|
|
3016
|
+
_mailStore: null,
|
|
3017
|
+
});
|
|
3018
|
+
|
|
3019
|
+
expect(removedNames).toContain("rpc-agent");
|
|
3020
|
+
});
|
|
3021
|
+
});
|
|
3022
|
+
|
|
3023
|
+
// ============================================================
|
|
3024
|
+
// Triage concurrency limit (_maxTriagePerTick)
|
|
3025
|
+
// ============================================================
|
|
3026
|
+
|
|
3027
|
+
describe("triage concurrency limit (_maxTriagePerTick)", () => {
|
|
3028
|
+
test("only _maxTriagePerTick triage calls happen when multiple sessions need level-2 escalation", async () => {
|
|
3029
|
+
const staleActivity = new Date(Date.now() - 60_000).toISOString();
|
|
3030
|
+
const stalledSince = new Date(Date.now() - 130_000).toISOString();
|
|
3031
|
+
|
|
3032
|
+
// 4 sessions all at escalation level 2
|
|
3033
|
+
const sessions: AgentSession[] = [
|
|
3034
|
+
makeSession({
|
|
3035
|
+
id: "s-1",
|
|
3036
|
+
agentName: "agent-1",
|
|
3037
|
+
tmuxSession: "ap-agent-1",
|
|
3038
|
+
state: "stalled",
|
|
3039
|
+
lastActivity: staleActivity,
|
|
3040
|
+
escalationLevel: 2,
|
|
3041
|
+
stalledSince,
|
|
3042
|
+
}),
|
|
3043
|
+
makeSession({
|
|
3044
|
+
id: "s-2",
|
|
3045
|
+
agentName: "agent-2",
|
|
3046
|
+
tmuxSession: "ap-agent-2",
|
|
3047
|
+
state: "stalled",
|
|
3048
|
+
lastActivity: staleActivity,
|
|
3049
|
+
escalationLevel: 2,
|
|
3050
|
+
stalledSince,
|
|
3051
|
+
}),
|
|
3052
|
+
makeSession({
|
|
3053
|
+
id: "s-3",
|
|
3054
|
+
agentName: "agent-3",
|
|
3055
|
+
tmuxSession: "ap-agent-3",
|
|
3056
|
+
state: "stalled",
|
|
3057
|
+
lastActivity: staleActivity,
|
|
3058
|
+
escalationLevel: 2,
|
|
3059
|
+
stalledSince,
|
|
3060
|
+
}),
|
|
3061
|
+
makeSession({
|
|
3062
|
+
id: "s-4",
|
|
3063
|
+
agentName: "agent-4",
|
|
3064
|
+
tmuxSession: "ap-agent-4",
|
|
3065
|
+
state: "stalled",
|
|
3066
|
+
lastActivity: staleActivity,
|
|
3067
|
+
escalationLevel: 2,
|
|
3068
|
+
stalledSince,
|
|
3069
|
+
}),
|
|
3070
|
+
];
|
|
3071
|
+
|
|
3072
|
+
writeSessionsToStore(tempRoot, sessions);
|
|
3073
|
+
|
|
3074
|
+
let triageCallCount = 0;
|
|
3075
|
+
const triageMock = async (_opts: { agentName: string; root: string; lastActivity: string }) => {
|
|
3076
|
+
triageCallCount++;
|
|
3077
|
+
return "extend" as const;
|
|
3078
|
+
};
|
|
3079
|
+
|
|
3080
|
+
await runDaemonTick({
|
|
3081
|
+
root: tempRoot,
|
|
3082
|
+
...THRESHOLDS,
|
|
3083
|
+
nudgeIntervalMs: 60_000,
|
|
3084
|
+
tier1Enabled: true,
|
|
3085
|
+
_maxTriagePerTick: 2,
|
|
3086
|
+
_tmux: tmuxWithLiveness({
|
|
3087
|
+
"ap-agent-1": true,
|
|
3088
|
+
"ap-agent-2": true,
|
|
3089
|
+
"ap-agent-3": true,
|
|
3090
|
+
"ap-agent-4": true,
|
|
3091
|
+
}),
|
|
3092
|
+
_triage: triageMock,
|
|
3093
|
+
_nudge: nudgeTracker().nudge,
|
|
3094
|
+
_eventStore: null,
|
|
3095
|
+
_recordFailure: async () => {},
|
|
3096
|
+
_getConnection: () => undefined,
|
|
3097
|
+
_removeConnection: () => {},
|
|
3098
|
+
_tailerRegistry: new Map(),
|
|
3099
|
+
_findLatestStdoutLog: async () => null,
|
|
3100
|
+
_mailStore: null,
|
|
3101
|
+
});
|
|
3102
|
+
|
|
3103
|
+
// Only 2 of the 4 sessions should have triggered triage
|
|
3104
|
+
expect(triageCallCount).toBe(2);
|
|
3105
|
+
});
|
|
3106
|
+
});
|
|
3107
|
+
|
|
3108
|
+
// ============================================================
|
|
3109
|
+
// RuntimeConnection-aware kill and liveness (agentplate-32cd)
|
|
3110
|
+
// ============================================================
|
|
3111
|
+
|
|
3112
|
+
describe("killAgent uses RuntimeConnection.abort() when available", () => {
|
|
3113
|
+
const deadPid = 999999;
|
|
3114
|
+
|
|
3115
|
+
function connProcessTracker(): {
|
|
3116
|
+
isAlive: (pid: number) => boolean;
|
|
3117
|
+
killTree: (pid: number) => Promise<void>;
|
|
3118
|
+
killed: number[];
|
|
3119
|
+
} {
|
|
3120
|
+
const killed: number[] = [];
|
|
3121
|
+
return {
|
|
3122
|
+
isAlive: (pid: number) => {
|
|
3123
|
+
try {
|
|
3124
|
+
process.kill(pid, 0);
|
|
3125
|
+
return true;
|
|
3126
|
+
} catch {
|
|
3127
|
+
return false;
|
|
3128
|
+
}
|
|
3129
|
+
},
|
|
3130
|
+
killTree: async (pid: number) => {
|
|
3131
|
+
killed.push(pid);
|
|
3132
|
+
},
|
|
3133
|
+
killed,
|
|
3134
|
+
};
|
|
3135
|
+
}
|
|
3136
|
+
|
|
3137
|
+
// Test A: killAgent uses connection.abort() when a connection is registered
|
|
3138
|
+
test("Test A: abort() called for ZFC-terminated headless agent with registered connection", async () => {
|
|
3139
|
+
const session = makeSession({
|
|
3140
|
+
agentName: "headless-conn-agent",
|
|
3141
|
+
tmuxSession: "", // headless
|
|
3142
|
+
pid: deadPid, // dead PID → ZFC fires (pidAlive=false)
|
|
3143
|
+
state: "working",
|
|
3144
|
+
lastActivity: new Date().toISOString(),
|
|
3145
|
+
});
|
|
3146
|
+
|
|
3147
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3148
|
+
|
|
3149
|
+
let abortCount = 0;
|
|
3150
|
+
const removedNames: string[] = [];
|
|
3151
|
+
const proc = connProcessTracker();
|
|
3152
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
3153
|
+
|
|
3154
|
+
await runDaemonTick({
|
|
3155
|
+
root: tempRoot,
|
|
3156
|
+
...THRESHOLDS,
|
|
3157
|
+
_tmux: tmuxMock,
|
|
3158
|
+
_triage: triageAlways("extend"),
|
|
3159
|
+
_process: proc,
|
|
3160
|
+
_eventStore: null,
|
|
3161
|
+
_recordFailure: async () => {},
|
|
3162
|
+
_getConnection: (name: string) => {
|
|
3163
|
+
if (name !== "headless-conn-agent") return undefined;
|
|
3164
|
+
return {
|
|
3165
|
+
getState: async () => ({ status: "working" as const }),
|
|
3166
|
+
sendPrompt: async () => {},
|
|
3167
|
+
followUp: async () => {},
|
|
3168
|
+
abort: async () => {
|
|
3169
|
+
abortCount++;
|
|
3170
|
+
},
|
|
3171
|
+
close: () => {},
|
|
3172
|
+
};
|
|
3173
|
+
},
|
|
3174
|
+
_removeConnection: (name: string) => {
|
|
3175
|
+
removedNames.push(name);
|
|
3176
|
+
},
|
|
3177
|
+
_tailerRegistry: new Map(),
|
|
3178
|
+
_findLatestStdoutLog: async () => null,
|
|
3179
|
+
_mailStore: null,
|
|
3180
|
+
});
|
|
3181
|
+
|
|
3182
|
+
// abort() called exactly once
|
|
3183
|
+
expect(abortCount).toBe(1);
|
|
3184
|
+
// killTree NOT called (abort succeeded)
|
|
3185
|
+
expect(proc.killed).toHaveLength(0);
|
|
3186
|
+
// removeConnection called for the agent
|
|
3187
|
+
expect(removedNames).toContain("headless-conn-agent");
|
|
3188
|
+
});
|
|
3189
|
+
|
|
3190
|
+
// Test B: killAgent falls back to killTree when conn.abort() throws
|
|
3191
|
+
test("Test B: killTree called as fallback when abort() throws", async () => {
|
|
3192
|
+
const session = makeSession({
|
|
3193
|
+
agentName: "headless-abort-fail",
|
|
3194
|
+
tmuxSession: "",
|
|
3195
|
+
pid: deadPid,
|
|
3196
|
+
state: "working",
|
|
3197
|
+
lastActivity: new Date().toISOString(),
|
|
3198
|
+
});
|
|
3199
|
+
|
|
3200
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3201
|
+
|
|
3202
|
+
let abortCalled = false;
|
|
3203
|
+
const removedNames: string[] = [];
|
|
3204
|
+
const proc = connProcessTracker();
|
|
3205
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
3206
|
+
|
|
3207
|
+
await runDaemonTick({
|
|
3208
|
+
root: tempRoot,
|
|
3209
|
+
...THRESHOLDS,
|
|
3210
|
+
_tmux: tmuxMock,
|
|
3211
|
+
_triage: triageAlways("extend"),
|
|
3212
|
+
_process: proc,
|
|
3213
|
+
_eventStore: null,
|
|
3214
|
+
_recordFailure: async () => {},
|
|
3215
|
+
_getConnection: (name: string) => {
|
|
3216
|
+
if (name !== "headless-abort-fail") return undefined;
|
|
3217
|
+
return {
|
|
3218
|
+
getState: async () => ({ status: "working" as const }),
|
|
3219
|
+
sendPrompt: async () => {},
|
|
3220
|
+
followUp: async () => {},
|
|
3221
|
+
abort: async () => {
|
|
3222
|
+
abortCalled = true;
|
|
3223
|
+
throw new Error("process already dead");
|
|
3224
|
+
},
|
|
3225
|
+
close: () => {},
|
|
3226
|
+
};
|
|
3227
|
+
},
|
|
3228
|
+
_removeConnection: (name: string) => {
|
|
3229
|
+
removedNames.push(name);
|
|
3230
|
+
},
|
|
3231
|
+
_tailerRegistry: new Map(),
|
|
3232
|
+
_findLatestStdoutLog: async () => null,
|
|
3233
|
+
_mailStore: null,
|
|
3234
|
+
});
|
|
3235
|
+
|
|
3236
|
+
// abort() was attempted
|
|
3237
|
+
expect(abortCalled).toBe(true);
|
|
3238
|
+
// killTree called as defense-in-depth fallback
|
|
3239
|
+
expect(proc.killed).toContain(deadPid);
|
|
3240
|
+
// removeConnection still called (before fallback)
|
|
3241
|
+
expect(removedNames).toContain("headless-abort-fail");
|
|
3242
|
+
});
|
|
3243
|
+
|
|
3244
|
+
// Test C: killAgent uses conn.abort() for triage-terminate path (level 2 → terminate)
|
|
3245
|
+
test("Test C: abort() called in triage-terminate path (level 2 → terminate verdict)", async () => {
|
|
3246
|
+
const nudgeIntervalMs = 60_000;
|
|
3247
|
+
// stalledSince 2.5 intervals ago → expectedLevel = floor(2.5) = 2 → triage fires
|
|
3248
|
+
const stalledSince = new Date(Date.now() - 2.5 * nudgeIntervalMs).toISOString();
|
|
3249
|
+
// staleActivity: 2x staleThreshold (60s) — stale but not zombie, so escalate fires
|
|
3250
|
+
const staleActivity = new Date(Date.now() - THRESHOLDS.staleThresholdMs * 2).toISOString();
|
|
3251
|
+
|
|
3252
|
+
const session = makeSession({
|
|
3253
|
+
agentName: "headless-triage-conn",
|
|
3254
|
+
tmuxSession: "",
|
|
3255
|
+
pid: process.pid, // alive — ZFC won't fire; escalation path triggers triage
|
|
3256
|
+
state: "stalled",
|
|
3257
|
+
lastActivity: staleActivity,
|
|
3258
|
+
escalationLevel: 1,
|
|
3259
|
+
stalledSince,
|
|
3260
|
+
});
|
|
3261
|
+
|
|
3262
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3263
|
+
|
|
3264
|
+
let abortCount = 0;
|
|
3265
|
+
const removedNames: string[] = [];
|
|
3266
|
+
const proc = connProcessTracker();
|
|
3267
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
3268
|
+
|
|
3269
|
+
await runDaemonTick({
|
|
3270
|
+
root: tempRoot,
|
|
3271
|
+
...THRESHOLDS,
|
|
3272
|
+
nudgeIntervalMs,
|
|
3273
|
+
tier1Enabled: true,
|
|
3274
|
+
_tmux: tmuxMock,
|
|
3275
|
+
_triage: triageAlways("terminate"),
|
|
3276
|
+
_nudge: nudgeTracker().nudge,
|
|
3277
|
+
_process: proc,
|
|
3278
|
+
_eventStore: null,
|
|
3279
|
+
_recordFailure: async () => {},
|
|
3280
|
+
// getState returns "error" so lastActivity is NOT refreshed — stale condition preserved
|
|
3281
|
+
_getConnection: (name: string) => {
|
|
3282
|
+
if (name !== "headless-triage-conn") return undefined;
|
|
3283
|
+
return {
|
|
3284
|
+
getState: async () => ({ status: "error" as const }),
|
|
3285
|
+
sendPrompt: async () => {},
|
|
3286
|
+
followUp: async () => {},
|
|
3287
|
+
abort: async () => {
|
|
3288
|
+
abortCount++;
|
|
3289
|
+
},
|
|
3290
|
+
close: () => {},
|
|
3291
|
+
};
|
|
3292
|
+
},
|
|
3293
|
+
_removeConnection: (name: string) => {
|
|
3294
|
+
removedNames.push(name);
|
|
3295
|
+
},
|
|
3296
|
+
_tailerRegistry: new Map(),
|
|
3297
|
+
_findLatestStdoutLog: async () => null,
|
|
3298
|
+
_mailStore: null,
|
|
3299
|
+
});
|
|
3300
|
+
|
|
3301
|
+
// abort() called via triage-terminate → killAgent path
|
|
3302
|
+
expect(abortCount).toBe(1);
|
|
3303
|
+
// killTree NOT called (abort succeeded)
|
|
3304
|
+
expect(proc.killed).toHaveLength(0);
|
|
3305
|
+
// tmux killSession NOT called (headless path only)
|
|
3306
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
3307
|
+
});
|
|
3308
|
+
|
|
3309
|
+
// Test D: integration — watchdog terminates a hung headless agent without touching tmux
|
|
3310
|
+
test("Test D: conn.abort() called, tmux.killSession and killTree NEVER called, state → zombie", async () => {
|
|
3311
|
+
const session = makeSession({
|
|
3312
|
+
agentName: "headless-zombie-conn",
|
|
3313
|
+
tmuxSession: "",
|
|
3314
|
+
pid: deadPid, // dead PID → ZFC fires
|
|
3315
|
+
state: "working",
|
|
3316
|
+
lastActivity: new Date(Date.now() - THRESHOLDS.zombieThresholdMs * 2).toISOString(),
|
|
3317
|
+
});
|
|
3318
|
+
|
|
3319
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3320
|
+
|
|
3321
|
+
let abortCount = 0;
|
|
3322
|
+
const proc = connProcessTracker();
|
|
3323
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
3324
|
+
|
|
3325
|
+
await runDaemonTick({
|
|
3326
|
+
root: tempRoot,
|
|
3327
|
+
...THRESHOLDS,
|
|
3328
|
+
_tmux: tmuxMock,
|
|
3329
|
+
_triage: triageAlways("extend"),
|
|
3330
|
+
_process: proc,
|
|
3331
|
+
_eventStore: null,
|
|
3332
|
+
_recordFailure: async () => {},
|
|
3333
|
+
_getConnection: (name: string) => {
|
|
3334
|
+
if (name !== "headless-zombie-conn") return undefined;
|
|
3335
|
+
return {
|
|
3336
|
+
getState: async () => ({ status: "working" as const }),
|
|
3337
|
+
sendPrompt: async () => {},
|
|
3338
|
+
followUp: async () => {},
|
|
3339
|
+
abort: async () => {
|
|
3340
|
+
abortCount++;
|
|
3341
|
+
},
|
|
3342
|
+
close: () => {},
|
|
3343
|
+
};
|
|
3344
|
+
},
|
|
3345
|
+
_removeConnection: () => {},
|
|
3346
|
+
_tailerRegistry: new Map(),
|
|
3347
|
+
_findLatestStdoutLog: async () => null,
|
|
3348
|
+
_mailStore: null,
|
|
3349
|
+
});
|
|
3350
|
+
|
|
3351
|
+
// abort() called
|
|
3352
|
+
expect(abortCount).toBe(1);
|
|
3353
|
+
// tmux.killSession NEVER called
|
|
3354
|
+
expect(tmuxMock.killed).toHaveLength(0);
|
|
3355
|
+
// killTree NEVER called (abort succeeded)
|
|
3356
|
+
expect(proc.killed).toHaveLength(0);
|
|
3357
|
+
// Agent state transitioned to zombie
|
|
3358
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
3359
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
3360
|
+
});
|
|
3361
|
+
|
|
3362
|
+
// Test E: liveness — getState() returning error status drives the agent toward zombie
|
|
3363
|
+
test("Test E: getState()=error + dead PID → tmuxAlive=false, state=zombie, terminate, abort called", async () => {
|
|
3364
|
+
const session = makeSession({
|
|
3365
|
+
agentName: "headless-error-conn",
|
|
3366
|
+
tmuxSession: "",
|
|
3367
|
+
pid: deadPid, // dead → ZFC fires: pidAlive=false
|
|
3368
|
+
state: "working",
|
|
3369
|
+
lastActivity: new Date().toISOString(), // fresh — time-based won't fire; ZFC does
|
|
3370
|
+
});
|
|
3371
|
+
|
|
3372
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3373
|
+
|
|
3374
|
+
let abortCount = 0;
|
|
3375
|
+
const proc = connProcessTracker();
|
|
3376
|
+
const checks: HealthCheck[] = [];
|
|
3377
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
3378
|
+
|
|
3379
|
+
await runDaemonTick({
|
|
3380
|
+
root: tempRoot,
|
|
3381
|
+
...THRESHOLDS,
|
|
3382
|
+
onHealthCheck: (c) => checks.push(c),
|
|
3383
|
+
_tmux: tmuxMock,
|
|
3384
|
+
_triage: triageAlways("extend"),
|
|
3385
|
+
_process: proc,
|
|
3386
|
+
_eventStore: null,
|
|
3387
|
+
_recordFailure: async () => {},
|
|
3388
|
+
_getConnection: (name: string) => {
|
|
3389
|
+
if (name !== "headless-error-conn") return undefined;
|
|
3390
|
+
return {
|
|
3391
|
+
getState: async () => ({ status: "error" as const }),
|
|
3392
|
+
sendPrompt: async () => {},
|
|
3393
|
+
followUp: async () => {},
|
|
3394
|
+
abort: async () => {
|
|
3395
|
+
abortCount++;
|
|
3396
|
+
},
|
|
3397
|
+
close: () => {},
|
|
3398
|
+
};
|
|
3399
|
+
},
|
|
3400
|
+
_removeConnection: () => {},
|
|
3401
|
+
_tailerRegistry: new Map(),
|
|
3402
|
+
_findLatestStdoutLog: async () => null,
|
|
3403
|
+
_mailStore: null,
|
|
3404
|
+
});
|
|
3405
|
+
|
|
3406
|
+
// Health check produced
|
|
3407
|
+
expect(checks).toHaveLength(1);
|
|
3408
|
+
// tmuxAlive=false because getState returned "error"
|
|
3409
|
+
expect(checks[0]?.tmuxAlive).toBe(false);
|
|
3410
|
+
// ZFC fires (pidAlive=false for dead PID) → zombie/terminate
|
|
3411
|
+
expect(checks[0]?.state).toBe("zombie");
|
|
3412
|
+
expect(checks[0]?.action).toBe("terminate");
|
|
3413
|
+
// abort() called via killAgent
|
|
3414
|
+
expect(abortCount).toBe(1);
|
|
3415
|
+
// killTree NOT called (abort succeeded)
|
|
3416
|
+
expect(proc.killed).toHaveLength(0);
|
|
3417
|
+
});
|
|
3418
|
+
|
|
3419
|
+
// Test F: connection.getState() rejection drops the connection and falls back to tmux
|
|
3420
|
+
test("Test F: getState() rejection → removeConnection called, tmux liveness used as fallback", async () => {
|
|
3421
|
+
const session = makeSession({
|
|
3422
|
+
agentName: "headless-reject-conn",
|
|
3423
|
+
tmuxSession: "",
|
|
3424
|
+
pid: process.pid, // alive
|
|
3425
|
+
state: "working",
|
|
3426
|
+
lastActivity: new Date().toISOString(), // fresh — no stale
|
|
3427
|
+
});
|
|
3428
|
+
|
|
3429
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3430
|
+
|
|
3431
|
+
const removedNames: string[] = [];
|
|
3432
|
+
const checks: HealthCheck[] = [];
|
|
3433
|
+
// tmux returns alive — used as fallback when getState rejects
|
|
3434
|
+
const tmuxMock = tmuxWithLiveness({ "": true });
|
|
3435
|
+
|
|
3436
|
+
await runDaemonTick({
|
|
3437
|
+
root: tempRoot,
|
|
3438
|
+
...THRESHOLDS,
|
|
3439
|
+
onHealthCheck: (c) => checks.push(c),
|
|
3440
|
+
_tmux: tmuxMock,
|
|
3441
|
+
_triage: triageAlways("extend"),
|
|
3442
|
+
_process: { isAlive: () => true, killTree: async () => {} },
|
|
3443
|
+
_eventStore: null,
|
|
3444
|
+
_recordFailure: async () => {},
|
|
3445
|
+
_getConnection: (name: string) => {
|
|
3446
|
+
if (name !== "headless-reject-conn") return undefined;
|
|
3447
|
+
return {
|
|
3448
|
+
getState: () => Promise.reject(new Error("connection error")),
|
|
3449
|
+
sendPrompt: async () => {},
|
|
3450
|
+
followUp: async () => {},
|
|
3451
|
+
abort: async () => {},
|
|
3452
|
+
close: () => {},
|
|
3453
|
+
};
|
|
3454
|
+
},
|
|
3455
|
+
_removeConnection: (name: string) => {
|
|
3456
|
+
removedNames.push(name);
|
|
3457
|
+
},
|
|
3458
|
+
_tailerRegistry: new Map(),
|
|
3459
|
+
_findLatestStdoutLog: async () => null,
|
|
3460
|
+
_mailStore: null,
|
|
3461
|
+
});
|
|
3462
|
+
|
|
3463
|
+
// removeConnection called (connection dropped after rejection)
|
|
3464
|
+
expect(removedNames).toContain("headless-reject-conn");
|
|
3465
|
+
// Agent is healthy (alive PID, fresh lastActivity, tmux fallback returns alive)
|
|
3466
|
+
expect(checks).toHaveLength(1);
|
|
3467
|
+
expect(checks[0]?.action).toBe("none");
|
|
3468
|
+
});
|
|
3469
|
+
});
|
|
3470
|
+
|
|
3471
|
+
// ============================================================
|
|
3472
|
+
// worker_died notification (agentplate-c111)
|
|
3473
|
+
// ============================================================
|
|
3474
|
+
|
|
3475
|
+
describe("worker_died parent notification", () => {
|
|
3476
|
+
let tempRoot: string;
|
|
3477
|
+
|
|
3478
|
+
beforeEach(async () => {
|
|
3479
|
+
tempRoot = await createTempRoot();
|
|
3480
|
+
});
|
|
3481
|
+
|
|
3482
|
+
afterEach(async () => {
|
|
3483
|
+
await cleanupTempDir(tempRoot);
|
|
3484
|
+
});
|
|
3485
|
+
|
|
3486
|
+
test("terminate path sends worker_died mail to parentAgent on first zombify", async () => {
|
|
3487
|
+
const session = makeSession({
|
|
3488
|
+
agentName: "dead-builder",
|
|
3489
|
+
capability: "builder",
|
|
3490
|
+
parentAgent: "lead-1",
|
|
3491
|
+
tmuxSession: "agentplate-dead-builder",
|
|
3492
|
+
state: "working",
|
|
3493
|
+
lastActivity: new Date().toISOString(),
|
|
3494
|
+
});
|
|
3495
|
+
|
|
3496
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3497
|
+
|
|
3498
|
+
const mailDb = join(tempRoot, ".agentplate", "mail.db");
|
|
3499
|
+
const mailStore = createMailStore(mailDb);
|
|
3500
|
+
|
|
3501
|
+
try {
|
|
3502
|
+
await runDaemonTick({
|
|
3503
|
+
root: tempRoot,
|
|
3504
|
+
...THRESHOLDS,
|
|
3505
|
+
_tmux: tmuxWithLiveness({ "agentplate-dead-builder": false }),
|
|
3506
|
+
_triage: triageAlways("extend"),
|
|
3507
|
+
_recordFailure: async () => {},
|
|
3508
|
+
_mailStore: mailStore,
|
|
3509
|
+
});
|
|
3510
|
+
|
|
3511
|
+
const inbox = mailStore.getUnread("lead-1");
|
|
3512
|
+
expect(inbox).toHaveLength(1);
|
|
3513
|
+
const msg = inbox[0];
|
|
3514
|
+
expect(msg).toBeDefined();
|
|
3515
|
+
if (!msg) return;
|
|
3516
|
+
expect(msg.type).toBe("worker_died");
|
|
3517
|
+
expect(msg.from).toBe("dead-builder");
|
|
3518
|
+
expect(msg.to).toBe("lead-1");
|
|
3519
|
+
expect(msg.priority).toBe("high");
|
|
3520
|
+
expect(msg.payload).not.toBeNull();
|
|
3521
|
+
const payload = JSON.parse(msg.payload ?? "{}") as WorkerDiedPayload;
|
|
3522
|
+
expect(payload.agentName).toBe("dead-builder");
|
|
3523
|
+
expect(payload.capability).toBe("builder");
|
|
3524
|
+
expect(payload.terminatedBy).toBe("tier0");
|
|
3525
|
+
expect(payload.reason).toBeTruthy();
|
|
3526
|
+
} finally {
|
|
3527
|
+
mailStore.close();
|
|
3528
|
+
}
|
|
3529
|
+
});
|
|
3530
|
+
|
|
3531
|
+
test("orphan agent (parentAgent=null) receives no notification", async () => {
|
|
3532
|
+
const session = makeSession({
|
|
3533
|
+
agentName: "orphan-agent",
|
|
3534
|
+
parentAgent: null,
|
|
3535
|
+
tmuxSession: "agentplate-orphan-agent",
|
|
3536
|
+
state: "working",
|
|
3537
|
+
lastActivity: new Date().toISOString(),
|
|
3538
|
+
});
|
|
3539
|
+
|
|
3540
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3541
|
+
|
|
3542
|
+
const mailDb = join(tempRoot, ".agentplate", "mail.db");
|
|
3543
|
+
const mailStore = createMailStore(mailDb);
|
|
3544
|
+
|
|
3545
|
+
try {
|
|
3546
|
+
await runDaemonTick({
|
|
3547
|
+
root: tempRoot,
|
|
3548
|
+
...THRESHOLDS,
|
|
3549
|
+
_tmux: tmuxWithLiveness({ "agentplate-orphan-agent": false }),
|
|
3550
|
+
_triage: triageAlways("extend"),
|
|
3551
|
+
_recordFailure: async () => {},
|
|
3552
|
+
_mailStore: mailStore,
|
|
3553
|
+
});
|
|
3554
|
+
|
|
3555
|
+
expect(mailStore.getAll({ type: "worker_died" })).toHaveLength(0);
|
|
3556
|
+
} finally {
|
|
3557
|
+
mailStore.close();
|
|
3558
|
+
}
|
|
3559
|
+
});
|
|
3560
|
+
|
|
3561
|
+
test("re-tick on already-zombie session does not send a second worker_died", async () => {
|
|
3562
|
+
// Subsequent ticks see the session already in `zombie`. The state matrix
|
|
3563
|
+
// rejects zombie → zombie transitions, so notify is gated on `outcome.ok`.
|
|
3564
|
+
const session = makeSession({
|
|
3565
|
+
agentName: "re-zombie-agent",
|
|
3566
|
+
parentAgent: "lead-2",
|
|
3567
|
+
tmuxSession: "agentplate-re-zombie-agent",
|
|
3568
|
+
state: "working",
|
|
3569
|
+
lastActivity: new Date().toISOString(),
|
|
3570
|
+
});
|
|
3571
|
+
|
|
3572
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3573
|
+
|
|
3574
|
+
const mailDb = join(tempRoot, ".agentplate", "mail.db");
|
|
3575
|
+
const mailStore = createMailStore(mailDb);
|
|
3576
|
+
|
|
3577
|
+
try {
|
|
3578
|
+
const tickOpts = {
|
|
3579
|
+
root: tempRoot,
|
|
3580
|
+
...THRESHOLDS,
|
|
3581
|
+
_tmux: tmuxWithLiveness({ "agentplate-re-zombie-agent": false }),
|
|
3582
|
+
_triage: triageAlways("extend"),
|
|
3583
|
+
_recordFailure: async () => {},
|
|
3584
|
+
_mailStore: mailStore,
|
|
3585
|
+
};
|
|
3586
|
+
await runDaemonTick(tickOpts);
|
|
3587
|
+
await runDaemonTick(tickOpts);
|
|
3588
|
+
await runDaemonTick(tickOpts);
|
|
3589
|
+
|
|
3590
|
+
expect(mailStore.getAll({ to: "lead-2", type: "worker_died" })).toHaveLength(1);
|
|
3591
|
+
} finally {
|
|
3592
|
+
mailStore.close();
|
|
3593
|
+
}
|
|
3594
|
+
});
|
|
3595
|
+
|
|
3596
|
+
test("notifyParentOnDeath=false suppresses the synthetic mail", async () => {
|
|
3597
|
+
const session = makeSession({
|
|
3598
|
+
agentName: "opt-out-agent",
|
|
3599
|
+
parentAgent: "lead-3",
|
|
3600
|
+
tmuxSession: "agentplate-opt-out-agent",
|
|
3601
|
+
state: "working",
|
|
3602
|
+
lastActivity: new Date().toISOString(),
|
|
3603
|
+
});
|
|
3604
|
+
|
|
3605
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3606
|
+
|
|
3607
|
+
const mailDb = join(tempRoot, ".agentplate", "mail.db");
|
|
3608
|
+
const mailStore = createMailStore(mailDb);
|
|
3609
|
+
|
|
3610
|
+
try {
|
|
3611
|
+
await runDaemonTick({
|
|
3612
|
+
root: tempRoot,
|
|
3613
|
+
...THRESHOLDS,
|
|
3614
|
+
notifyParentOnDeath: false,
|
|
3615
|
+
_tmux: tmuxWithLiveness({ "agentplate-opt-out-agent": false }),
|
|
3616
|
+
_triage: triageAlways("extend"),
|
|
3617
|
+
_recordFailure: async () => {},
|
|
3618
|
+
_mailStore: mailStore,
|
|
3619
|
+
});
|
|
3620
|
+
|
|
3621
|
+
expect(mailStore.getAll({ type: "worker_died" })).toHaveLength(0);
|
|
3622
|
+
// State should still transition normally
|
|
3623
|
+
const reloaded = readSessionsFromStore(tempRoot);
|
|
3624
|
+
expect(reloaded[0]?.state).toBe("zombie");
|
|
3625
|
+
} finally {
|
|
3626
|
+
mailStore.close();
|
|
3627
|
+
}
|
|
3628
|
+
});
|
|
3629
|
+
|
|
3630
|
+
test("escalation-level-3 terminate also notifies parent with tier0 reason", async () => {
|
|
3631
|
+
// Stalled agent with alive tmux: progressive escalation drives it to level 3
|
|
3632
|
+
// terminate. The notify path runs through the escalation branch, not the
|
|
3633
|
+
// `check.action === "terminate"` branch.
|
|
3634
|
+
const stalledSince = new Date(Date.now() - 4 * 60_000).toISOString();
|
|
3635
|
+
const lastActivity = new Date(Date.now() - 60_000).toISOString();
|
|
3636
|
+
const session = makeSession({
|
|
3637
|
+
agentName: "escalated-agent",
|
|
3638
|
+
parentAgent: "coordinator",
|
|
3639
|
+
tmuxSession: "agentplate-escalated-agent",
|
|
3640
|
+
state: "working",
|
|
3641
|
+
lastActivity,
|
|
3642
|
+
stalledSince,
|
|
3643
|
+
escalationLevel: 3,
|
|
3644
|
+
});
|
|
3645
|
+
|
|
3646
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3647
|
+
|
|
3648
|
+
const mailDb = join(tempRoot, ".agentplate", "mail.db");
|
|
3649
|
+
const mailStore = createMailStore(mailDb);
|
|
3650
|
+
|
|
3651
|
+
try {
|
|
3652
|
+
await runDaemonTick({
|
|
3653
|
+
root: tempRoot,
|
|
3654
|
+
...THRESHOLDS,
|
|
3655
|
+
nudgeIntervalMs: 60_000,
|
|
3656
|
+
_tmux: tmuxWithLiveness({ "agentplate-escalated-agent": true }),
|
|
3657
|
+
_triage: triageAlways("extend"),
|
|
3658
|
+
_nudge: async () => ({ delivered: true }),
|
|
3659
|
+
_recordFailure: async () => {},
|
|
3660
|
+
_mailStore: mailStore,
|
|
3661
|
+
});
|
|
3662
|
+
|
|
3663
|
+
const inbox = mailStore.getUnread("coordinator");
|
|
3664
|
+
expect(inbox).toHaveLength(1);
|
|
3665
|
+
const msg = inbox[0];
|
|
3666
|
+
if (!msg) return;
|
|
3667
|
+
expect(msg.type).toBe("worker_died");
|
|
3668
|
+
const payload = JSON.parse(msg.payload ?? "{}") as WorkerDiedPayload;
|
|
3669
|
+
expect(payload.terminatedBy).toBe("tier0");
|
|
3670
|
+
expect(payload.reason).toContain("Progressive escalation");
|
|
3671
|
+
} finally {
|
|
3672
|
+
mailStore.close();
|
|
3673
|
+
}
|
|
3674
|
+
});
|
|
3675
|
+
|
|
3676
|
+
test("tier1 triage terminate sets terminatedBy=tier1 in payload", async () => {
|
|
3677
|
+
// stalledSince must produce expectedLevel==2 from nudgeIntervalMs=60_000:
|
|
3678
|
+
// floor(stalledMs / 60_000) === 2 requires 2*60_000 <= stalledMs < 3*60_000.
|
|
3679
|
+
const stalledSince = new Date(Date.now() - 150_000).toISOString();
|
|
3680
|
+
const lastActivity = new Date(Date.now() - 60_000).toISOString();
|
|
3681
|
+
const session = makeSession({
|
|
3682
|
+
agentName: "triaged-agent",
|
|
3683
|
+
parentAgent: "lead-triage",
|
|
3684
|
+
tmuxSession: "agentplate-triaged-agent",
|
|
3685
|
+
state: "working",
|
|
3686
|
+
lastActivity,
|
|
3687
|
+
stalledSince,
|
|
3688
|
+
escalationLevel: 2,
|
|
3689
|
+
});
|
|
3690
|
+
|
|
3691
|
+
writeSessionsToStore(tempRoot, [session]);
|
|
3692
|
+
|
|
3693
|
+
const mailDb = join(tempRoot, ".agentplate", "mail.db");
|
|
3694
|
+
const mailStore = createMailStore(mailDb);
|
|
3695
|
+
|
|
3696
|
+
try {
|
|
3697
|
+
await runDaemonTick({
|
|
3698
|
+
root: tempRoot,
|
|
3699
|
+
...THRESHOLDS,
|
|
3700
|
+
nudgeIntervalMs: 60_000,
|
|
3701
|
+
tier1Enabled: true,
|
|
3702
|
+
_tmux: tmuxWithLiveness({ "agentplate-triaged-agent": true }),
|
|
3703
|
+
_triage: triageAlways("terminate"),
|
|
3704
|
+
_nudge: async () => ({ delivered: true }),
|
|
3705
|
+
_recordFailure: async () => {},
|
|
3706
|
+
_mailStore: mailStore,
|
|
3707
|
+
});
|
|
3708
|
+
|
|
3709
|
+
const inbox = mailStore.getUnread("lead-triage");
|
|
3710
|
+
expect(inbox).toHaveLength(1);
|
|
3711
|
+
const msg = inbox[0];
|
|
3712
|
+
if (!msg) return;
|
|
3713
|
+
expect(msg.type).toBe("worker_died");
|
|
3714
|
+
const payload = JSON.parse(msg.payload ?? "{}") as WorkerDiedPayload;
|
|
3715
|
+
expect(payload.terminatedBy).toBe("tier1");
|
|
3716
|
+
expect(payload.reason).toContain("AI triage");
|
|
3717
|
+
} finally {
|
|
3718
|
+
mailStore.close();
|
|
3719
|
+
}
|
|
3720
|
+
});
|
|
3721
|
+
});
|