@zerwiz/ymir 0.1.4 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/README.md +291 -0
- package/.agents/agents/bragi-marketer.md +111 -0
- package/.agents/agents/brokk.md +352 -0
- package/.agents/agents/forseti-reviewer.md +93 -0
- package/.agents/agents/frigg-consent.md +77 -0
- package/.agents/agents/groa-update.md +77 -0
- package/.agents/agents/hnoss-designer.md +69 -0
- package/.agents/agents/huginn-researcher.md +99 -0
- package/.agents/agents/jord-projects.md +78 -0
- package/.agents/agents/kvasir-scout.md +91 -0
- package/.agents/agents/mimir-planner.md +92 -0
- package/.agents/agents/muninn-stow.md +77 -0
- package/.agents/agents/saga-bearings.md +78 -0
- package/.agents/agents/sindri-developer.md +114 -0
- package/.agents/agents/snotra-documenter.md +105 -0
- package/.agents/agents/syn-recovery.md +78 -0
- package/.agents/agents/tyr-check.md +78 -0
- package/.agents/agents/urdh-hold.md +77 -0
- package/.agents/agents/volundr-smith.md +80 -0
- package/.agents/agents/vor-diagnostics.md +77 -0
- package/.agents/assets/agents/naming.md +50 -0
- package/.agents/assets/agents/registry.md +93 -0
- package/.agents/assets/agents/runtime.md +36 -0
- package/.agents/assets/agents/toon-tasks-cli.md +42 -0
- package/.agents/assets/templates/company_entity.template.md +37 -0
- package/.agents/backend/README.md +74 -0
- package/.agents/backend/backends/cmux.sh +649 -0
- package/.agents/backend/backends/herdr-eventwait.py +157 -0
- package/.agents/backend/backends/herdr-workspace-move.py +114 -0
- package/.agents/backend/backends/herdr.sh +3386 -0
- package/.agents/backend/backends/orca.sh +290 -0
- package/.agents/backend/backends/tmux.sh +357 -0
- package/.agents/backend/backends/zellij.sh +700 -0
- package/.agents/backend/fm-afk-launch.sh +654 -0
- package/.agents/backend/fm-afk-return.sh +241 -0
- package/.agents/backend/fm-afk-start.sh +172 -0
- package/.agents/backend/fm-arm-command-policy.mjs +960 -0
- package/.agents/backend/fm-arm-pretool-check.sh +200 -0
- package/.agents/backend/fm-backend-hometag-lib.sh +52 -0
- package/.agents/backend/fm-backend.sh +981 -0
- package/.agents/backend/fm-backlog-handoff.sh +909 -0
- package/.agents/backend/fm-backlog-receive.sh +187 -0
- package/.agents/backend/fm-backlog-transition-lib.sh +779 -0
- package/.agents/backend/fm-bearings-board.sh +204 -0
- package/.agents/backend/fm-bearings-snapshot.sh +546 -0
- package/.agents/backend/fm-bootstrap.sh +1568 -0
- package/.agents/backend/fm-branch-outcome.sh +203 -0
- package/.agents/backend/fm-branch-prompt.sh +101 -0
- package/.agents/backend/fm-brief.sh +452 -0
- package/.agents/backend/fm-busy-event.sh +234 -0
- package/.agents/backend/fm-busy-lib.sh +986 -0
- package/.agents/backend/fm-captain-hold.sh +1001 -0
- package/.agents/backend/fm-cd-command-policy.mjs +153 -0
- package/.agents/backend/fm-cd-pretool-check.sh +190 -0
- package/.agents/backend/fm-check-lib.sh +74 -0
- package/.agents/backend/fm-check-register.sh +42 -0
- package/.agents/backend/fm-check-unregister.sh +52 -0
- package/.agents/backend/fm-classify-lib.sh +1770 -0
- package/.agents/backend/fm-claude-stop-autoarm.sh +334 -0
- package/.agents/backend/fm-composer-lib.sh +1416 -0
- package/.agents/backend/fm-config-inherit-lib.sh +1202 -0
- package/.agents/backend/fm-config-push.sh +254 -0
- package/.agents/backend/fm-control-lib.sh +251 -0
- package/.agents/backend/fm-control.sh +878 -0
- package/.agents/backend/fm-crew-state.sh +625 -0
- package/.agents/backend/fm-cursor-lib.sh +243 -0
- package/.agents/backend/fm-decision-hold.sh +232 -0
- package/.agents/backend/fm-doc-audience-check.sh +269 -0
- package/.agents/backend/fm-dod-lib.sh +67 -0
- package/.agents/backend/fm-ensure-agents-md.sh +253 -0
- package/.agents/backend/fm-extension-launch-barrier.mjs +129 -0
- package/.agents/backend/fm-extension.mjs +2577 -0
- package/.agents/backend/fm-extension.sh +16 -0
- package/.agents/backend/fm-ff-lib.sh +420 -0
- package/.agents/backend/fm-fleet-snapshot.sh +1463 -0
- package/.agents/backend/fm-fleet-sync.sh +458 -0
- package/.agents/backend/fm-fleet-view.sh +96 -0
- package/.agents/backend/fm-gate-refuse-lib.sh +102 -0
- package/.agents/backend/fm-guard.sh +242 -0
- package/.agents/backend/fm-harness.sh +195 -0
- package/.agents/backend/fm-herdr-ci-cleanup.sh +115 -0
- package/.agents/backend/fm-herdr-lab.sh +346 -0
- package/.agents/backend/fm-herdr-session-cleanup.sh +337 -0
- package/.agents/backend/fm-home-seed.sh +984 -0
- package/.agents/backend/fm-home-summary-refresh.sh +255 -0
- package/.agents/backend/fm-hook-host-lib.sh +36 -0
- package/.agents/backend/fm-inactive-reconcile.sh +531 -0
- package/.agents/backend/fm-inbox.sh +399 -0
- package/.agents/backend/fm-install-actionlint.sh +84 -0
- package/.agents/backend/fm-install-herdr.sh +96 -0
- package/.agents/backend/fm-install-shellcheck.sh +84 -0
- package/.agents/backend/fm-install-treehouse.sh +95 -0
- package/.agents/backend/fm-kimi-turnend-hook.sh +276 -0
- package/.agents/backend/fm-lease-lib.sh +218 -0
- package/.agents/backend/fm-lease.sh +189 -0
- package/.agents/backend/fm-line-cap-lib.sh +51 -0
- package/.agents/backend/fm-lint-workflows.sh +137 -0
- package/.agents/backend/fm-lint.sh +591 -0
- package/.agents/backend/fm-lock-lib.sh +104 -0
- package/.agents/backend/fm-lock.sh +107 -0
- package/.agents/backend/fm-marker-lib.sh +12 -0
- package/.agents/backend/fm-merge-local.sh +74 -0
- package/.agents/backend/fm-merge-outcome-lib.sh +138 -0
- package/.agents/backend/fm-nm-run-lib.sh +124 -0
- package/.agents/backend/fm-on.sh +125 -0
- package/.agents/backend/fm-operational-input.sh +253 -0
- package/.agents/backend/fm-peek.sh +46 -0
- package/.agents/backend/fm-pending-reply-lib.sh +1409 -0
- package/.agents/backend/fm-pr-check.sh +135 -0
- package/.agents/backend/fm-pr-lib.sh +1016 -0
- package/.agents/backend/fm-pr-merge.sh +707 -0
- package/.agents/backend/fm-pr-poll.sh +110 -0
- package/.agents/backend/fm-primary-scope-lib.sh +33 -0
- package/.agents/backend/fm-procevent-extension-capture.pl +259 -0
- package/.agents/backend/fm-procevent-lavish.sh +628 -0
- package/.agents/backend/fm-procevent-lib.sh +892 -0
- package/.agents/backend/fm-procevent-quota.sh +290 -0
- package/.agents/backend/fm-procevent-remote-reply.sh +577 -0
- package/.agents/backend/fm-procevent-when.sh +504 -0
- package/.agents/backend/fm-procevent.sh +1466 -0
- package/.agents/backend/fm-project-mode.sh +92 -0
- package/.agents/backend/fm-project-origin-lib.sh +180 -0
- package/.agents/backend/fm-promote.sh +282 -0
- package/.agents/backend/fm-public-followup-emit.sh +260 -0
- package/.agents/backend/fm-public-followup-lib.sh +398 -0
- package/.agents/backend/fm-public-followup.sh +1330 -0
- package/.agents/backend/fm-push-transition-lib.sh +168 -0
- package/.agents/backend/fm-quota-axi-lib.sh +93 -0
- package/.agents/backend/fm-quota-choose.sh +384 -0
- package/.agents/backend/fm-remote-delta-read.sh +189 -0
- package/.agents/backend/fm-remote-doctor.sh +800 -0
- package/.agents/backend/fm-remote-entrypoint.sh +193 -0
- package/.agents/backend/fm-remote-file.sh +234 -0
- package/.agents/backend/fm-remote-home-provision.sh +259 -0
- package/.agents/backend/fm-remote-home-seed.sh +259 -0
- package/.agents/backend/fm-remote-inherit-push.sh +90 -0
- package/.agents/backend/fm-remote-inherit.sh +179 -0
- package/.agents/backend/fm-remote-job-lib.sh +1241 -0
- package/.agents/backend/fm-remote-job-reap-orphans.sh +137 -0
- package/.agents/backend/fm-remote-job-worker.sh +1124 -0
- package/.agents/backend/fm-remote-readiness-lib.sh +44 -0
- package/.agents/backend/fm-remote-secondmate-control.sh +340 -0
- package/.agents/backend/fm-review-diff.sh +158 -0
- package/.agents/backend/fm-secondmate-charter-lib.sh +44 -0
- package/.agents/backend/fm-secondmate-nudge-lib.sh +68 -0
- package/.agents/backend/fm-secondmate-parent-lib.sh +70 -0
- package/.agents/backend/fm-secondmate-reconcile.sh +371 -0
- package/.agents/backend/fm-secondmate-registry-lib.sh +311 -0
- package/.agents/backend/fm-secondmate-report.sh +88 -0
- package/.agents/backend/fm-send.sh +1059 -0
- package/.agents/backend/fm-session-lock-lib.sh +176 -0
- package/.agents/backend/fm-session-start.sh +970 -0
- package/.agents/backend/fm-sessionstart-cursor.sh +40 -0
- package/.agents/backend/fm-sessionstart-nudge.sh +45 -0
- package/.agents/backend/fm-sessionstart-run.sh +145 -0
- package/.agents/backend/fm-spawn.sh +3153 -0
- package/.agents/backend/fm-startup-memory-budget-lib.sh +224 -0
- package/.agents/backend/fm-startup-memory-budget.sh +94 -0
- package/.agents/backend/fm-startup-network.sh +642 -0
- package/.agents/backend/fm-stow-cascade.sh +251 -0
- package/.agents/backend/fm-subagent-pretool-check.sh +207 -0
- package/.agents/backend/fm-supervise-daemon.sh +1732 -0
- package/.agents/backend/fm-supervision-instructions.sh +215 -0
- package/.agents/backend/fm-supervision-lib.sh +90 -0
- package/.agents/backend/fm-supervisor-target-lib.sh +78 -0
- package/.agents/backend/fm-tangle-lib.sh +53 -0
- package/.agents/backend/fm-task-inbox-lib.sh +402 -0
- package/.agents/backend/fm-tasks-axi-lib.sh +123 -0
- package/.agents/backend/fm-teardown.sh +2922 -0
- package/.agents/backend/fm-test-isolation-proof.sh +588 -0
- package/.agents/backend/fm-test-run.sh +2191 -0
- package/.agents/backend/fm-timeout-lib.sh +141 -0
- package/.agents/backend/fm-timing-lib.sh +179 -0
- package/.agents/backend/fm-tmux-lib.sh +291 -0
- package/.agents/backend/fm-tool-update-check.sh +898 -0
- package/.agents/backend/fm-trace-context-lib.sh +227 -0
- package/.agents/backend/fm-transition-lib.sh +103 -0
- package/.agents/backend/fm-turnend-guard-cursor.sh +391 -0
- package/.agents/backend/fm-turnend-guard-grok.sh +90 -0
- package/.agents/backend/fm-turnend-guard.sh +432 -0
- package/.agents/backend/fm-update.sh +110 -0
- package/.agents/backend/fm-vendor-auth-probe.sh +190 -0
- package/.agents/backend/fm-voice-client.py +1373 -0
- package/.agents/backend/fm-voice-relay.py +1256 -0
- package/.agents/backend/fm-wake-drain.sh +592 -0
- package/.agents/backend/fm-wake-grant.sh +127 -0
- package/.agents/backend/fm-wake-lib.sh +1849 -0
- package/.agents/backend/fm-watch-arm.sh +602 -0
- package/.agents/backend/fm-watch-checkpoint.sh +109 -0
- package/.agents/backend/fm-watch.sh +1962 -0
- package/.agents/backend/fm-x-dismiss.sh +116 -0
- package/.agents/backend/fm-x-followup.sh +286 -0
- package/.agents/backend/fm-x-lib.sh +1000 -0
- package/.agents/backend/fm-x-link.sh +234 -0
- package/.agents/backend/fm-x-poll.sh +185 -0
- package/.agents/backend/fm-x-reply.sh +385 -0
- package/.agents/backend/fm_voice_frame.py +166 -0
- package/.agents/backend/fm_voice_records.py +574 -0
- package/.agents/backend/model-bridge.py +360 -0
- package/.agents/backend/opencode-go-bridge.py +30 -0
- package/.agents/bus/README.md +12 -0
- package/.agents/bus/messages.json +1 -0
- package/.agents/config/agents.machine.example.yaml +16 -0
- package/.agents/config/agents.yaml.example +72 -0
- package/.agents/config/crew-dispatch.json +22 -0
- package/.agents/config/cron.yaml +11 -0
- package/.agents/config/eindri-dispatch.json +67 -0
- package/.agents/config/eindri-harness +1 -0
- package/.agents/config/startup-memory-budget +1 -0
- package/.agents/filebrowser/.filebrowser.json +6 -0
- package/.agents/filebrowser/README.md +11 -0
- package/.agents/gateway/README.md +12 -0
- package/.agents/github/README.md +15 -0
- package/.agents/github/webhooks/.gitkeep +0 -0
- package/.agents/github/workflows/.gitkeep +0 -0
- package/.agents/harness/opencode/plugins/lib/rodd-operational-input.js +37 -0
- package/.agents/harness/opencode/plugins/package.json +4 -0
- package/.agents/harness/opencode/plugins/saga-sessionstart.js +77 -0
- package/.agents/harness/opencode/plugins/syn-cd-check.js +61 -0
- package/.agents/harness/opencode/plugins/syn-pretool-check.js +68 -0
- package/.agents/harness/opencode/plugins/syn-turnend-guard.js +105 -0
- package/.agents/harness/opencode/plugins/syn-watch-arm.js +521 -0
- package/.agents/memory/README.md +46 -0
- package/.agents/memory/runes_audit.md +5 -0
- package/.agents/migrations/0001-hodd-layout.sh +38 -0
- package/.agents/migrations/0002-a2a-mcp.sh +27 -0
- package/.agents/migrations/0003-private-data-separation.sh +121 -0
- package/.agents/migrations/0004-hoard-and-realms.sh +104 -0
- package/.agents/migrations/0005-roots-out-of-tree.sh +158 -0
- package/.agents/sandbox/Dockerfile.utgard +19 -0
- package/.agents/sandbox/README.md +14 -0
- package/.agents/sandbox/utgard.config.json +14 -0
- package/.agents/skills/README.md +59 -0
- package/.agents/skills/bragi-marketing/README.md +3 -0
- package/.agents/skills/bragi-marketing/SKILL.md +82 -0
- package/.agents/skills/eindri-homes/README.md +3 -0
- package/.agents/skills/eindri-homes/SKILL.md +291 -0
- package/.agents/skills/eindri-homes/assets/control-plane.md +122 -0
- package/.agents/skills/eindri-homes/assets/dispatch.md +80 -0
- package/.agents/skills/frigg-consent/README.md +3 -0
- package/.agents/skills/frigg-consent/SKILL.md +56 -0
- package/.agents/skills/galdr-ymirsystem/README.md +3 -0
- package/.agents/skills/galdr-ymirsystem/SKILL.md +159 -0
- package/.agents/skills/galdr-ymirsystem/assets/README.md +74 -0
- package/.agents/skills/galdr-ymirsystem/assets/brokk-distro-runtime.md +390 -0
- package/.agents/skills/galdr-ymirsystem/assets/build-method.md +113 -0
- package/.agents/skills/galdr-ymirsystem/assets/build-tool-categories.md +178 -0
- package/.agents/skills/galdr-ymirsystem/assets/eindri-orchestration.md +518 -0
- package/.agents/skills/galdr-ymirsystem/assets/eindri-profiles.md +136 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/README.md +736 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/claude-code.md +197 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/codex.md +191 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/cursor.md +193 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/opencode.md +262 -0
- package/.agents/skills/galdr-ymirsystem/assets/harness-integration/pi.md +295 -0
- package/.agents/skills/galdr-ymirsystem/assets/hlidskjalf-ui.md +539 -0
- package/.agents/skills/galdr-ymirsystem/assets/installation.md +763 -0
- package/.agents/skills/galdr-ymirsystem/assets/local-models/HOST-DATA.example.md +72 -0
- package/.agents/skills/galdr-ymirsystem/assets/local-models.md +274 -0
- package/.agents/skills/galdr-ymirsystem/assets/memory-well.md +163 -0
- package/.agents/skills/galdr-ymirsystem/assets/nornir-jobs.md +415 -0
- package/.agents/skills/galdr-ymirsystem/assets/norse-naming.md +325 -0
- package/.agents/skills/galdr-ymirsystem/assets/odrerir-hall.md +133 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/einherjar-spawn.schema.json +81 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/herdr-profile.toml +75 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/pi-profile.yml +101 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot/supervision-tree.yml +163 -0
- package/.agents/skills/galdr-ymirsystem/assets/pi-boot-guide.md +58 -0
- package/.agents/skills/galdr-ymirsystem/assets/porting-upstream-to-norse.md +301 -0
- package/.agents/skills/galdr-ymirsystem/assets/principles.md +385 -0
- package/.agents/skills/galdr-ymirsystem/assets/reference-adoption.md +99 -0
- package/.agents/skills/galdr-ymirsystem/assets/registry.md +379 -0
- package/.agents/skills/galdr-ymirsystem/assets/runtime-compliance.md +463 -0
- package/.agents/skills/galdr-ymirsystem/assets/runtime-components.md +553 -0
- package/.agents/skills/galdr-ymirsystem/assets/smidja.md +180 -0
- package/.agents/skills/galdr-ymirsystem/schemas/toon-schemas.md +190 -0
- package/.agents/skills/galdr-ymirsystem/scripts/README.md +45 -0
- package/.agents/skills/galdr-ymirsystem/scripts/bench-one.sh +248 -0
- package/.agents/skills/galdr-ymirsystem/scripts/compliance-check.sh +503 -0
- package/.agents/skills/galdr-ymirsystem/scripts/gpu-sample.sh +43 -0
- package/.agents/skills/galdr-ymirsystem/scripts/stop-all.sh +34 -0
- package/.agents/skills/galdr-ymirsystem/scripts/toon-check.py +188 -0
- package/.agents/skills/gjallarhorn-relay/README.md +3 -0
- package/.agents/skills/gjallarhorn-relay/SKILL.md +283 -0
- package/.agents/skills/groa-update/SKILL.md +90 -0
- package/.agents/skills/hamr-adapters/README.md +3 -0
- package/.agents/skills/hamr-adapters/SKILL.md +123 -0
- package/.agents/skills/hamr-adapters/references/common/control-and-recovery.md +37 -0
- package/.agents/skills/hamr-adapters/references/common/dispatch.md +32 -0
- package/.agents/skills/hamr-adapters/references/common/model-and-effort.md +42 -0
- package/.agents/skills/hamr-adapters/references/common/primary-hooks.md +40 -0
- package/.agents/skills/hamr-adapters/references/harness/claude.md +55 -0
- package/.agents/skills/hamr-adapters/references/harness/codex.md +43 -0
- package/.agents/skills/hamr-adapters/references/harness/cursor.md +75 -0
- package/.agents/skills/hamr-adapters/references/harness/grok.md +69 -0
- package/.agents/skills/hamr-adapters/references/harness/kimi.md +51 -0
- package/.agents/skills/hamr-adapters/references/harness/muse.md +70 -0
- package/.agents/skills/hamr-adapters/references/harness/opencode.md +42 -0
- package/.agents/skills/hamr-adapters/references/harness/pi.md +56 -0
- package/.agents/skills/herdr-panes/README.md +3 -0
- package/.agents/skills/herdr-panes/SKILL.md +92 -0
- package/.agents/skills/herdr-panes/assets/herdr-backend.md +346 -0
- package/.agents/skills/herdr-panes/assets/tmux-backend.md +115 -0
- package/.agents/skills/hnoss-design/DESIGN.md +19 -0
- package/.agents/skills/hnoss-design/README.md +3 -0
- package/.agents/skills/hnoss-design/SKILL.md +83 -0
- package/.agents/skills/hvild-afk/README.md +3 -0
- package/.agents/skills/hvild-afk/SKILL.md +241 -0
- package/.agents/skills/jord-projects/README.md +3 -0
- package/.agents/skills/jord-projects/SKILL.md +92 -0
- package/.agents/skills/lifecycle/SKILL.md +56 -0
- package/.agents/skills/lifecycle/smoke_test.sh +57 -0
- package/.agents/skills/lifecycle/start.sh +21 -0
- package/.agents/skills/lifecycle/status.sh +35 -0
- package/.agents/skills/lifecycle/stop.sh +16 -0
- package/.agents/skills/muninn-stow/README.md +3 -0
- package/.agents/skills/muninn-stow/SKILL.md +309 -0
- package/.agents/skills/no-mistakes/SKILL.md +379 -0
- package/.agents/skills/nornir-schedule/README.md +3 -0
- package/.agents/skills/nornir-schedule/SKILL.md +25 -0
- package/.agents/skills/nornir-schedule/assets/events.md +137 -0
- package/.agents/skills/nornir-schedule/assets/quota.md +131 -0
- package/.agents/skills/nsr-compliance/README.md +3 -0
- package/.agents/skills/nsr-compliance/SKILL.md +29 -0
- package/.agents/skills/nsr-compliance/assets/nsr/README.md +40 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/README.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/git-workflow.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/security.md +22 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/RULES/tech-stack.md +8 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/SKILL.md +71 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/config/core_four.yaml +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/validate_code.sh +29 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/gates/verify_docs.py +66 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/generate.sh +109 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/NSRcompliance/templates/compliance/telemetry/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/README.md +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/features/README.md +36 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/features/SKILL-template.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/README.md +25 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/SKILL.md +13 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/create_branch.sh +6 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/safe_commit.sh +9 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/git_ops/sync_upstream.sh +6 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/README.md +27 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/SKILL.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/smoke_test.sh +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/start.sh +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/status.sh +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/agents_skills/lifecycle/stop.sh +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/config/core_four.yaml +13 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/telemetry/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/compliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/README.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/api.md +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/compliance-wiring.md +155 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/BEST_PRACTICES/frontend.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/README.md +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/clients/README.md +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/clients/template-README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/deploy.sh +49 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/development.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/production.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/envs/staging.env.example +17 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/tenants/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/CI_CD/deployment/tenants/template.env.example +8 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/README.md +26 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/agent.md +7 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/developers/dev-1.md +20 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/linux.md +9 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/macos.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/shared.md +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/DEVELOPER_SETUP/windows.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/README.md +25 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/_template.md +41 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/HOSTING/hosting-1.md +41 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/README.md +21 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/README.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/deployment.md +19 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/disaster-recovery.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/RUNBOOK/ops-tasks.md +14 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/features/README.md +16 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/project-skills.md +146 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/research/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/research/software-compliance-research.md +269 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/docs/working-with-agents.md +260 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/AGENTS.md +38 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/ARCHITECTURE.md +36 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/BEST_PRACTICES.md +27 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/CI_CD.md +23 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/FEATURES.md +15 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/README.md +22 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/RULES.md +37 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/STRUCTURE.md +45 -0
- package/.agents/skills/nsr-compliance/assets/nsr/assets/root/TECH_STACK.md +28 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scaffold-spec.md +63 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scripts/scaffold.sh +103 -0
- package/.agents/skills/nsr-compliance/assets/nsr/scripts/validate.sh +52 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/SKILL.md +71 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/config/core_four.yaml +16 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/check_danger.sh +102 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/check_wiring.sh +69 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/validate_code.sh +29 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/gates/verify_docs.py +66 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/generate.sh +109 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/envelopes/result_envelope.json +14 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/envelopes/task_envelope.json +20 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/harness/runner.py +303 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/telemetry/logger.py +112 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/README.md +18 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/config/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/README.md +12 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_env.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_paths.sh +20 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/gates/check_platform.sh +23 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/harness/README.md +11 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/harness/envelopes/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/installer/README.md +10 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/installer/install.sh +125 -0
- package/.agents/skills/nsr-compliance/assets/nsrcompliance/templates/compliance/telemetry/README.md +11 -0
- package/.agents/skills/pr-ops/README.md +3 -0
- package/.agents/skills/pr-ops/SKILL.md +109 -0
- package/.agents/skills/ratatoskr-a2a/README.md +3 -0
- package/.agents/skills/ratatoskr-a2a/SKILL.md +148 -0
- package/.agents/skills/rules-check-drift/SKILL.md +81 -0
- package/.agents/skills/saga-bearings/README.md +3 -0
- package/.agents/skills/saga-bearings/SKILL.md +26 -0
- package/.agents/skills/saga-bearings/assets/bearings.md +161 -0
- package/.agents/skills/saga-bearings/assets/board-template.html +731 -0
- package/.agents/skills/saga-bearings/assets/recap.md +56 -0
- package/.agents/skills/syn-recovery/README.md +3 -0
- package/.agents/skills/syn-recovery/SKILL.md +54 -0
- package/.agents/skills/tyr-check/README.md +3 -0
- package/.agents/skills/tyr-check/SKILL.md +218 -0
- package/.agents/skills/urdh-hold/README.md +3 -0
- package/.agents/skills/urdh-hold/SKILL.md +25 -0
- package/.agents/skills/urdh-hold/assets/decisions.md +15 -0
- package/.agents/skills/urdh-hold/assets/hold.md +54 -0
- package/.agents/skills/vor-diagnostics/README.md +3 -0
- package/.agents/skills/vor-diagnostics/SKILL.md +68 -0
- package/.agents/skills/ymir-host/README.md +3 -0
- package/.agents/skills/ymir-host/SKILL.md +35 -0
- package/.agents/skills/ymir-host/assets/desktop.md +36 -0
- package/.agents/skills/ymir-host/assets/install.md +69 -0
- package/.agents/skills/ymir-host/assets/omarchy.md +297 -0
- package/.agents/skills/ymir-host/assets/runbooks.md +15 -0
- package/.agents/skills/ymir-host/assets/thjazi.md +220 -0
- package/.agents/skills/ymir-host/assets/update.md +64 -0
- package/.agents/tests/assets/board-render-harness.mjs +117 -0
- package/.agents/tests/cmux-test-safety.sh +51 -0
- package/.agents/tests/fixtures.sh +291 -0
- package/.agents/tests/fm-afk-inject-e2e.test.sh +428 -0
- package/.agents/tests/fm-afk-inject-herdr-e2e.test.sh +536 -0
- package/.agents/tests/fm-afk-launch.test.sh +961 -0
- package/.agents/tests/fm-afk-pi-herdr-return-e2e.test.sh +297 -0
- package/.agents/tests/fm-afk-return.test.sh +283 -0
- package/.agents/tests/fm-arm-pretool-check.test.sh +482 -0
- package/.agents/tests/fm-ask-user-authority.test.sh +42 -0
- package/.agents/tests/fm-backend-autodetect-smoke.test.sh +176 -0
- package/.agents/tests/fm-backend-cmux-smoke.test.sh +188 -0
- package/.agents/tests/fm-backend-cmux.test.sh +1166 -0
- package/.agents/tests/fm-backend-herdr-eventwait-smoke.test.sh +136 -0
- package/.agents/tests/fm-backend-herdr-eventwait.test.py +106 -0
- package/.agents/tests/fm-backend-herdr-focus-flash-e2e.test.sh +409 -0
- package/.agents/tests/fm-backend-herdr-launcher-workspace-e2e.test.sh +437 -0
- package/.agents/tests/fm-backend-herdr-presentation-e2e.test.sh +1424 -0
- package/.agents/tests/fm-backend-herdr-prune-safety-e2e.test.sh +180 -0
- package/.agents/tests/fm-backend-herdr-respawn-idem-e2e.test.sh +182 -0
- package/.agents/tests/fm-backend-herdr-smoke.test.sh +355 -0
- package/.agents/tests/fm-backend-herdr-workspace-per-home-e2e.test.sh +251 -0
- package/.agents/tests/fm-backend-herdr.test.sh +4606 -0
- package/.agents/tests/fm-backend-orca.test.sh +1357 -0
- package/.agents/tests/fm-backend-tmux-smoke.test.sh +173 -0
- package/.agents/tests/fm-backend-zellij-smoke.test.sh +205 -0
- package/.agents/tests/fm-backend-zellij.test.sh +1360 -0
- package/.agents/tests/fm-backend.test.sh +1142 -0
- package/.agents/tests/fm-backlog-atomicity.test.sh +2302 -0
- package/.agents/tests/fm-backlog-handoff.test.sh +1360 -0
- package/.agents/tests/fm-bearings-board-render.test.sh +135 -0
- package/.agents/tests/fm-bearings-board.test.sh +414 -0
- package/.agents/tests/fm-bearings-snapshot.test.sh +1990 -0
- package/.agents/tests/fm-bootstrap-network-parallel.test.sh +316 -0
- package/.agents/tests/fm-bootstrap.test.sh +1178 -0
- package/.agents/tests/fm-branch-supervision.test.sh +552 -0
- package/.agents/tests/fm-brief.test.sh +785 -0
- package/.agents/tests/fm-busy-adapter-wiring.test.sh +324 -0
- package/.agents/tests/fm-busy-state.test.sh +465 -0
- package/.agents/tests/fm-calm-pi-extension.test.sh +3979 -0
- package/.agents/tests/fm-captain-hold-lifecycle.test.sh +1187 -0
- package/.agents/tests/fm-cd-pretool-check.test.sh +400 -0
- package/.agents/tests/fm-check-unregister.test.sh +198 -0
- package/.agents/tests/fm-classify-corr-token.test.sh +543 -0
- package/.agents/tests/fm-classify-decision-key.test.sh +340 -0
- package/.agents/tests/fm-claude-stop-autoarm-live-e2e.test.sh +164 -0
- package/.agents/tests/fm-claude-stop-autoarm.test.sh +1190 -0
- package/.agents/tests/fm-cmux-claude-composer-live-e2e.test.sh +110 -0
- package/.agents/tests/fm-codex-continuity-live-e2e.test.sh +55 -0
- package/.agents/tests/fm-composer-ghost.test.sh +716 -0
- package/.agents/tests/fm-composer-lib.test.sh +666 -0
- package/.agents/tests/fm-composer-matrix-live-e2e.test.sh +220 -0
- package/.agents/tests/fm-control-herdr-smoke.test.sh +149 -0
- package/.agents/tests/fm-control-relaunch.test.sh +1530 -0
- package/.agents/tests/fm-control.test.sh +910 -0
- package/.agents/tests/fm-crew-state.test.sh +1610 -0
- package/.agents/tests/fm-cursor-harness.test.sh +404 -0
- package/.agents/tests/fm-cursor-primary-live-e2e.test.sh +217 -0
- package/.agents/tests/fm-cursor-primary.test.sh +708 -0
- package/.agents/tests/fm-daemon.test.sh +2740 -0
- package/.agents/tests/fm-documentation-audiences.test.sh +141 -0
- package/.agents/tests/fm-ensure-agents-md.test.sh +371 -0
- package/.agents/tests/fm-extension-binding.test.sh +2187 -0
- package/.agents/tests/fm-fleet-snapshot-view.test.sh +816 -0
- package/.agents/tests/fm-fleet-sync.test.sh +721 -0
- package/.agents/tests/fm-gate-refuse.test.sh +356 -0
- package/.agents/tests/fm-gitignore-config.test.sh +89 -0
- package/.agents/tests/fm-gotmp.test.sh +230 -0
- package/.agents/tests/fm-grok-continuity-live-e2e.test.sh +112 -0
- package/.agents/tests/fm-grok-harness.test.sh +117 -0
- package/.agents/tests/fm-grok-stop-live-e2e.test.sh +219 -0
- package/.agents/tests/fm-guard-stale-banner.test.sh +709 -0
- package/.agents/tests/fm-harness-adapter-instructions-live-e2e.test.sh +134 -0
- package/.agents/tests/fm-harness-adapter-references.test.sh +30 -0
- package/.agents/tests/fm-harness-liveness-drift-live-e2e.test.sh +151 -0
- package/.agents/tests/fm-herdr-lab.test.sh +243 -0
- package/.agents/tests/fm-herdr-session-cleanup-e2e.test.sh +146 -0
- package/.agents/tests/fm-herdr-session-cleanup.test.sh +329 -0
- package/.agents/tests/fm-herdr-submit-confirm-live-e2e.test.sh +125 -0
- package/.agents/tests/fm-herdr-version-floor-live-e2e.test.sh +144 -0
- package/.agents/tests/fm-home-summary-refresh.test.sh +971 -0
- package/.agents/tests/fm-inactive-reconcile.test.sh +485 -0
- package/.agents/tests/fm-kimi-harness.test.sh +687 -0
- package/.agents/tests/fm-lint-workflows.test.sh +532 -0
- package/.agents/tests/fm-lint.test.sh +1023 -0
- package/.agents/tests/fm-muse-harness.test.sh +932 -0
- package/.agents/tests/fm-muse-signals-live-e2e.test.sh +205 -0
- package/.agents/tests/fm-no-mistakes-required.test.sh +72 -0
- package/.agents/tests/fm-on.test.sh +507 -0
- package/.agents/tests/fm-opencode-primary-live-e2e.test.sh +357 -0
- package/.agents/tests/fm-operational-input.test.sh +160 -0
- package/.agents/tests/fm-peek-remote.test.sh +110 -0
- package/.agents/tests/fm-pending-reply.test.sh +1288 -0
- package/.agents/tests/fm-pi-branch-extension.test.sh +3182 -0
- package/.agents/tests/fm-pi-branch-live-e2e.test.sh +527 -0
- package/.agents/tests/fm-pi-primary-live-e2e.test.sh +345 -0
- package/.agents/tests/fm-pi-primary-types.test.sh +68 -0
- package/.agents/tests/fm-pi-watch-extension.test.sh +2843 -0
- package/.agents/tests/fm-pr-check-security.test.sh +2140 -0
- package/.agents/tests/fm-pr-merge.test.sh +2135 -0
- package/.agents/tests/fm-procevent-quota.test.sh +225 -0
- package/.agents/tests/fm-procevent-when.test.sh +406 -0
- package/.agents/tests/fm-procevent.test.sh +1745 -0
- package/.agents/tests/fm-project-origin.test.sh +110 -0
- package/.agents/tests/fm-public-followup.test.sh +2272 -0
- package/.agents/tests/fm-quota-array-dispatch-live-e2e.test.sh +526 -0
- package/.agents/tests/fm-quota-choose.test.sh +626 -0
- package/.agents/tests/fm-remote-backlog-handoff.test.sh +453 -0
- package/.agents/tests/fm-remote-doctor.test.sh +625 -0
- package/.agents/tests/fm-remote-entrypoint.test.sh +59 -0
- package/.agents/tests/fm-remote-job-orphan-reap.test.sh +205 -0
- package/.agents/tests/fm-remote-job.test.sh +768 -0
- package/.agents/tests/fm-remote-reply.test.sh +520 -0
- package/.agents/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +1256 -0
- package/.agents/tests/fm-remote-secondmate-parent-binding.test.sh +326 -0
- package/.agents/tests/fm-remote-secondmate-trace-context.test.sh +310 -0
- package/.agents/tests/fm-remote-transport-lanes.test.sh +431 -0
- package/.agents/tests/fm-review-diff.test.sh +176 -0
- package/.agents/tests/fm-secondmate-harness.test.sh +2600 -0
- package/.agents/tests/fm-secondmate-lifecycle-e2e.test.sh +259 -0
- package/.agents/tests/fm-secondmate-liveness.test.sh +559 -0
- package/.agents/tests/fm-secondmate-reconcile.test.sh +732 -0
- package/.agents/tests/fm-secondmate-safety.test.sh +2972 -0
- package/.agents/tests/fm-secondmate-sync.test.sh +913 -0
- package/.agents/tests/fm-send-inbox-doorbell-live-e2e.test.sh +206 -0
- package/.agents/tests/fm-send-inbox.test.sh +352 -0
- package/.agents/tests/fm-send-popup-settle.test.sh +167 -0
- package/.agents/tests/fm-send-remote-delivery.test.sh +796 -0
- package/.agents/tests/fm-send-resolve-key.test.sh +551 -0
- package/.agents/tests/fm-send-secondmate-marker-herdr-e2e.test.sh +187 -0
- package/.agents/tests/fm-send-secondmate-marker.test.sh +278 -0
- package/.agents/tests/fm-send-settle.test.sh +147 -0
- package/.agents/tests/fm-send-strict.test.sh +241 -0
- package/.agents/tests/fm-session-lock-ancestry.test.sh +365 -0
- package/.agents/tests/fm-session-start.test.sh +2515 -0
- package/.agents/tests/fm-sessionstart-hook-live-e2e.test.sh +631 -0
- package/.agents/tests/fm-sessionstart-instruction-refresh-live-e2e.test.sh +230 -0
- package/.agents/tests/fm-sessionstart-nudge.test.sh +1039 -0
- package/.agents/tests/fm-shared-captain-inheritance.test.sh +404 -0
- package/.agents/tests/fm-spawn-batch.test.sh +148 -0
- package/.agents/tests/fm-spawn-dispatch-profile.test.sh +827 -0
- package/.agents/tests/fm-spawn-pool-base-freshen.test.sh +440 -0
- package/.agents/tests/fm-spawn-worktree-settle.test.sh +147 -0
- package/.agents/tests/fm-startup-memory-budget.test.sh +333 -0
- package/.agents/tests/fm-startup-network.test.sh +712 -0
- package/.agents/tests/fm-stow-cascade.test.sh +370 -0
- package/.agents/tests/fm-subagent-pretool-check.test.sh +291 -0
- package/.agents/tests/fm-supervision-events.test.sh +156 -0
- package/.agents/tests/fm-supervision-instructions.test.sh +189 -0
- package/.agents/tests/fm-tangle-guard.test.sh +277 -0
- package/.agents/tests/fm-task-delivery.test.sh +420 -0
- package/.agents/tests/fm-task-inbox.test.sh +513 -0
- package/.agents/tests/fm-teardown-endpoint-safety.test.sh +374 -0
- package/.agents/tests/fm-teardown.test.sh +2665 -0
- package/.agents/tests/fm-test-fixture-cleanup.test.sh +172 -0
- package/.agents/tests/fm-test-fixtures.test.sh +132 -0
- package/.agents/tests/fm-test-isolation-proof.test.sh +278 -0
- package/.agents/tests/fm-test-run.test.sh +1196 -0
- package/.agents/tests/fm-tmux-agent-liveness.test.sh +354 -0
- package/.agents/tests/fm-tmux-submit-busy.test.sh +340 -0
- package/.agents/tests/fm-tool-update-check.test.sh +1044 -0
- package/.agents/tests/fm-trace-context-lib.test.sh +253 -0
- package/.agents/tests/fm-trace-context-spawn.test.sh +581 -0
- package/.agents/tests/fm-transition-lib.test.sh +54 -0
- package/.agents/tests/fm-turnend-guard.test.sh +1822 -0
- package/.agents/tests/fm-update.test.sh +304 -0
- package/.agents/tests/fm-vendor-auth-probe.test.sh +395 -0
- package/.agents/tests/fm-voice-relay.test.sh +4741 -0
- package/.agents/tests/fm-wake-daemon-lifecycle-e2e.test.sh +171 -0
- package/.agents/tests/fm-wake-drain-open-decisions-cursor.test.sh +356 -0
- package/.agents/tests/fm-wake-drain-open-decisions.test.sh +226 -0
- package/.agents/tests/fm-wake-drain-unread-status.test.sh +379 -0
- package/.agents/tests/fm-wake-queue.test.sh +1224 -0
- package/.agents/tests/fm-watch-arm.test.sh +821 -0
- package/.agents/tests/fm-watch-checkpoint.test.sh +87 -0
- package/.agents/tests/fm-watch-recovery-loop.test.sh +223 -0
- package/.agents/tests/fm-watch-triage.test.sh +3889 -0
- package/.agents/tests/fm-watcher-lock.test.sh +1133 -0
- package/.agents/tests/fm-x-mode.test.sh +3042 -0
- package/.agents/tests/gleipnir-guard-pretool-check.test.sh +51 -0
- package/.agents/tests/gleipnir-machine-lock.test.sh +130 -0
- package/.agents/tests/herdr-test-safety.sh +42 -0
- package/.agents/tests/lib.sh +315 -0
- package/.agents/tests/remote-herdr-fixture.sh +125 -0
- package/.agents/tests/secondmate-helpers.sh +193 -0
- package/.agents/tests/smoke.test.sh +93 -0
- package/.agents/tests/wake-helpers.sh +338 -0
- package/.agents/tests/zellij-test-safety.sh +60 -0
- package/.agents/tools/README.md +10 -0
- package/.agents/tools/bin/tasks-cli +267 -0
- package/.agents/tools/tasks-cli.ts +216 -0
- package/AGENTS.md +498 -0
- package/CHANGELOG.md +1913 -0
- package/LICENSE +1 -1
- package/NOTICE +63 -0
- package/README.md +114 -60
- package/RULES/01-domains.md +51 -0
- package/RULES/02-agents.md +78 -0
- package/RULES/03-houses.md +61 -0
- package/RULES/04-hoard.md +97 -0
- package/RULES/05-platforms.md +67 -0
- package/RULES/06-append-only.md +108 -0
- package/RULES/07-config.md +101 -0
- package/RULES/08-delivery-gate.md +75 -0
- package/RULES/09-electron.md +58 -0
- package/RULES/README.md +22 -0
- package/bin/README.md +107 -0
- package/bin/a2a-mcp.sh +133 -0
- package/bin/a2a-serve.py +103 -0
- package/bin/a2a-serve.sh +8 -0
- package/bin/a2a-talk.sh +50 -0
- package/bin/a2abridge-ensure.sh +108 -0
- package/bin/agent-run.sh +43 -0
- package/bin/agents-config.sh +256 -0
- package/bin/apodex-smoke-test.sh +105 -0
- package/bin/bifrost-bridge.sh +131 -0
- package/bin/bifrost-ingress.sh +77 -0
- package/bin/bootstrap-macos.sh +112 -0
- package/bin/bootstrap-windows.ps1 +112 -0
- package/bin/branch-guard.sh +64 -0
- package/bin/brokk +73 -0
- package/bin/brokk-classify-lib.sh +1770 -0
- package/bin/brokk-lease-lib.sh +222 -0
- package/bin/brokk-lease.sh +193 -0
- package/bin/brokk-lint.sh +65 -0
- package/bin/brokk-send.sh +51 -0
- package/bin/brokk-update.sh +9 -0
- package/bin/brokk-wake-grant.sh +127 -0
- package/bin/brokk-wake-lib.sh +1849 -0
- package/bin/changelog-assemble.sh +154 -0
- package/bin/changelog-guard.sh +197 -0
- package/bin/crash-sense.sh +173 -0
- package/bin/design-check.sh +80 -0
- package/bin/design-icon.sh +153 -0
- package/bin/desktop-place.sh +224 -0
- package/bin/docs-guard.sh +39 -0
- package/bin/editor-place.sh +225 -0
- package/bin/eindri-acclaim.sh +47 -0
- package/bin/eindri-control.sh +28 -0
- package/bin/eindri-role.sh +83 -0
- package/bin/eindri-seat.sh +83 -0
- package/bin/eindri-seen.sh +54 -0
- package/bin/eindri-send.sh +15 -0
- package/bin/eindri-start.sh +98 -0
- package/bin/eindri-watch.sh +57 -0
- package/bin/einherjar-spawn.sh +576 -0
- package/bin/eir-doctor.sh +158 -0
- package/bin/erindi-brief.sh +271 -0
- package/bin/feature-inventory.sh +73 -0
- package/bin/fleet-apply.sh +81 -0
- package/bin/github-deploy.sh +94 -0
- package/bin/gjallarhorn-expose.sh +136 -0
- package/bin/gjallarhorn-notify.sh +71 -0
- package/bin/gjallarhorn-purge.sh +67 -0
- package/bin/gjallarhorn-tunnel.sh +85 -0
- package/bin/gleipnir-lock-lib.sh +284 -0
- package/bin/groa-update.sh +130 -0
- package/bin/hall-snapshot.sh +111 -0
- package/bin/hamr-harness.sh +246 -0
- package/bin/heimdall-ssh-keys.sh +374 -0
- package/bin/herdr-agents.py +44 -0
- package/bin/herdr-ensure.sh +104 -0
- package/bin/herdr-run.sh +452 -0
- package/bin/hermes-ensure.sh +88 -0
- package/bin/hlidskjalf-agents.sh +172 -0
- package/bin/hlidskjalf-usage.sh +196 -0
- package/bin/hoard-guard.sh +135 -0
- package/bin/hoard-lib.sh +105 -0
- package/bin/hodd.sh +102 -0
- package/bin/host-sense.sh +199 -0
- package/bin/huginn-research-worker.sh +121 -0
- package/bin/local-model-lock.sh +70 -0
- package/bin/mimir-bridge.py +259 -0
- package/bin/mimir-bridge.sh +100 -0
- package/bin/mimir-ingest.sh +131 -0
- package/bin/mimir-reflect.sh +27 -0
- package/bin/mimir.sh +113 -0
- package/bin/mjollnir-webhook.sh +72 -0
- package/bin/mjollnir.sh +104 -0
- package/bin/model-hardware.sh +96 -0
- package/bin/model-resolve.sh +160 -0
- package/bin/models-detect.sh +178 -0
- package/bin/nornir-cron-start.sh +162 -0
- package/bin/nornir-job-daily-briefing.sh +191 -0
- package/bin/nornir-job-git-sync.sh +152 -0
- package/bin/nornir-job-hall-snapshot.sh +28 -0
- package/bin/nornir-job-memory-housekeeping.sh +114 -0
- package/bin/nornir-job-nsr-compliance.sh +54 -0
- package/bin/nornir-job-observer.sh +166 -0
- package/bin/nornir-job-skillopt-sleep.sh +76 -0
- package/bin/npm-publish.sh +136 -0
- package/bin/omarchy-hook-install.sh +60 -0
- package/bin/omarchy-install.sh +161 -0
- package/bin/omarchy-plugins.sh +187 -0
- package/bin/omarchy-sense.sh +162 -0
- package/bin/perm-guard.sh +30 -0
- package/bin/pi-agent.sh +68 -0
- package/bin/pi-ensure.sh +77 -0
- package/bin/pi-local.sh +37 -0
- package/bin/pi-model-agnostic.sh +71 -0
- package/bin/pi-seat.sh +63 -0
- package/bin/prereq-ensure.sh +166 -0
- package/bin/private-guard.sh +58 -0
- package/bin/project-git.sh +69 -0
- package/bin/public-guard.sh +71 -0
- package/bin/ratatoskr.sh +49 -0
- package/bin/realm-lib.sh +32 -0
- package/bin/repo-scrub.sh +62 -0
- package/bin/rodd-operational-input.sh +250 -0
- package/bin/runes-append.sh +198 -0
- package/bin/saga-session-start.sh +172 -0
- package/bin/saga-sessionstart-run.sh +79 -0
- package/bin/saga-wake-drain.sh +41 -0
- package/bin/secret-guard.sh +63 -0
- package/bin/sessrumnir-ensure.sh +100 -0
- package/bin/sessrumnir-sync.sh +237 -0
- package/bin/sessrumnir.sh +104 -0
- package/bin/skillopt-setup.sh +76 -0
- package/bin/skuld-branch-outcome.sh +203 -0
- package/bin/skuld-branch-prompt.sh +101 -0
- package/bin/smidja-bootstrap.sh +94 -0
- package/bin/smidja-observe.sh +86 -0
- package/bin/syn-arm-pretool-check.sh +57 -0
- package/bin/syn-asset-pretool-check.sh +89 -0
- package/bin/syn-cd-pretool-check.sh +28 -0
- package/bin/syn-guard-pretool-check.sh +83 -0
- package/bin/syn-spawn-pretool-check.sh +58 -0
- package/bin/syn-turnend-guard.sh +36 -0
- package/bin/syn-watch-arm.sh +99 -0
- package/bin/tailscale-sync.sh +203 -0
- package/bin/telegram-bot.sh +82 -0
- package/bin/toolchain.sh +74 -0
- package/bin/ui-truth-check.sh +75 -0
- package/bin/utgard.sh +111 -0
- package/bin/valhalla.sh +115 -0
- package/bin/valknut-load.sh +298 -0
- package/bin/vor-crew-state.sh +124 -0
- package/bin/wedge-notify.sh +60 -0
- package/bin/workspace-provision.sh +80 -0
- package/bin/workspace-rag.sh +163 -0
- package/bin/wyrd-db.sh +47 -0
- package/bin/yggdrasil.sh +152 -0
- package/bin/ymir-edit.sh +145 -0
- package/bin/ymir-install.sh +806 -0
- package/bin/ymir-invite.sh +93 -0
- package/bin/ymir-isolation.sh +88 -0
- package/bin/ymir-migrate.sh +61 -0
- package/bin/ymir-plan.sh +356 -0
- package/bin/ymir-platform.sh +348 -0
- package/bin/ymir-say.sh +110 -0
- package/bin/ymir-setup-auth.sh +119 -0
- package/bin/ymir-validate.sh +193 -0
- package/bin/ymir.js +74 -16
- package/docs/Architecture.md +339 -0
- package/docs/design.md +351 -0
- package/docs/installations/README.md +63 -0
- package/docs/installations/ymir-home.md +93 -0
- package/docs/integration.md +105 -0
- package/docs/lore.md +1060 -0
- package/docs/research/agent-harnesses.md +290 -0
- package/docs/runbooks/README.md +19 -0
- package/docs/runbooks/a2a.md +91 -0
- package/docs/runbooks/agent-permissions.md +99 -0
- package/docs/runbooks/agents.md +57 -0
- package/docs/runbooks/models.md +69 -0
- package/docs/runbooks/realm-onboarding.md +60 -0
- package/docs/runbooks/secrets-and-hoard.md +60 -0
- package/docs/runbooks/updates-and-migrations.md +110 -0
- package/docs/runbooks/worktrees-isolation.md +72 -0
- package/docs/session-start.md +105 -0
- package/docs/skillopt-integration.md +321 -0
- package/docs/sync-system.md +66 -0
- package/docs/workspaces.md +104 -0
- package/docs/ymir-rut.md +260 -0
- package/package.json +54 -14
- package/packaging/build.sh +74 -0
- package/packaging/macos/Ymir Installer.command +43 -0
- package/packaging/windows/ymir-setup.nsi +74 -0
- package/scripts/README.md +22 -0
- package/scripts/electron.sh +269 -0
- package/scripts/lower.sh +34 -0
- package/scripts/raise.sh +42 -0
- package/scripts/start.sh +217 -0
- package/scripts/stop.sh +113 -0
- package/install.sh +0 -35
|
@@ -0,0 +1,3889 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# tests/fm-watch-triage.test.sh - the always-on wake triage built into
|
|
3
|
+
# bin/fm-watch.sh and the shared classifier (bin/fm-classify-lib.sh). The watcher
|
|
4
|
+
# now absorbs the benign majority of wakes in bash and exits ONLY on an actionable
|
|
5
|
+
# wake, so firstmate's LLM re-arms once per actionable event instead of once per
|
|
6
|
+
# wake. These tests cover the classifier predicates as pure functions, then drive
|
|
7
|
+
# a real fm-watch.sh subprocess to assert the behavioral contract:
|
|
8
|
+
# provably-working no-verb wakes absorbed (no exit, no queue entry, suppressor
|
|
9
|
+
# advanced, beacon fresh), stopped-crew no-verb wakes surfaced (queue + exit),
|
|
10
|
+
# provably-working stale panes absorbed-then-escalated past the threshold,
|
|
11
|
+
# terminal-looking stale status lines overridden by an active run, the heartbeat
|
|
12
|
+
# backstop fail-safe, and afk coherence (no double-triage while the away-mode
|
|
13
|
+
# daemon owns supervision).
|
|
14
|
+
#
|
|
15
|
+
# Daemon-side classification/injection lives in fm-daemon.test.sh; watcher/lock
|
|
16
|
+
# liveness in fm-watcher-lock.test.sh; the durable-queue safety matrix in
|
|
17
|
+
# fm-wake-queue.test.sh.
|
|
18
|
+
set -u
|
|
19
|
+
|
|
20
|
+
# shellcheck source=tests/wake-helpers.sh
|
|
21
|
+
. "$(dirname "${BASH_SOURCE[0]}")/wake-helpers.sh"
|
|
22
|
+
# shellcheck source=/dev/null
|
|
23
|
+
. "$ROOT/bin/fm-classify-lib.sh"
|
|
24
|
+
|
|
25
|
+
WATCH="$ROOT/bin/fm-watch.sh"
|
|
26
|
+
DRAIN="$ROOT/bin/fm-wake-drain.sh"
|
|
27
|
+
|
|
28
|
+
TMP_ROOT=$(fm_test_tmproot fm-watch-triage-tests)
|
|
29
|
+
|
|
30
|
+
ack_stopped_cycle() { # <state>
|
|
31
|
+
local state=$1 err sequence generation
|
|
32
|
+
err="$state/.test-cycle-drain.err"
|
|
33
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2> "$err" || return 1
|
|
34
|
+
sequence=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through \([0-9][0-9]*\) --recovery-generation [A-Za-z0-9._-][A-Za-z0-9._-]*$/\1/p' "$err")
|
|
35
|
+
generation=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through [0-9][0-9]* --recovery-generation \([A-Za-z0-9._-][A-Za-z0-9._-]*\)$/\1/p' "$err")
|
|
36
|
+
rm -f "$err"
|
|
37
|
+
[ -n "$sequence" ] && [ -n "$generation" ] || return 1
|
|
38
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" --ack-through "$sequence" \
|
|
39
|
+
--recovery-generation "$generation"
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
# Common watcher knobs: tight poll/grace, no check or heartbeat cadence unless a
|
|
43
|
+
# test overrides them, so a test only exercises the path it targets. FM_CREW_STATE_BIN
|
|
44
|
+
# points at the case's hermetic fake fm-crew-state.sh (installed by make_case) so the
|
|
45
|
+
# absorb-only-when-provably-working triage reads a canned verdict; a test fixes that
|
|
46
|
+
# verdict via FM_FAKE_CREW_STATE in its environment before calling watch_bg.
|
|
47
|
+
watch_bg() { # <state> <fakebin> <out> [extra env assignments...]
|
|
48
|
+
local state=$1 fakebin=$2 out=$3
|
|
49
|
+
shift 3
|
|
50
|
+
PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
51
|
+
FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$@" "$WATCH" > "$out" &
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
# Wait up to <limit> 0.1s ticks while <pid> stays alive; 0 if still alive, 1 if it died.
|
|
55
|
+
wait_live() {
|
|
56
|
+
local pid=$1 limit=${2:-30} i=0
|
|
57
|
+
while [ "$i" -lt "$limit" ]; do
|
|
58
|
+
kill -0 "$pid" 2>/dev/null || return 1
|
|
59
|
+
sleep 0.1
|
|
60
|
+
i=$((i + 1))
|
|
61
|
+
done
|
|
62
|
+
return 0
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
# Wait until <pid>'s watcher has completed a whole poll cycle, or exited first.
|
|
66
|
+
# A fixed wait_live budget only proves the process is still ALIVE: fm-watch.sh
|
|
67
|
+
# does bounded startup work (the recovery-marker snapshot, lock acquisition)
|
|
68
|
+
# before its first stale scan, so on a loaded
|
|
69
|
+
# machine a short fixed budget can reap a round before the cycle it asserts on
|
|
70
|
+
# ever ran - and then every "no wake, no marker" assertion passes vacuously
|
|
71
|
+
# while every "marker written" assertion fails spuriously.
|
|
72
|
+
# The liveness beacon is touched at the TOP of every poll, so this drops any
|
|
73
|
+
# beacon left by an earlier round, waits for THIS watcher to write a fresh one
|
|
74
|
+
# (some poll's top), then waits for that one to advance (the next poll's top) -
|
|
75
|
+
# and the whole cycle in between is what the caller's assertions describe.
|
|
76
|
+
# 0 if the watcher is still alive after a completed cycle, 1 if it exited.
|
|
77
|
+
wait_poll_cycle() { # <state> <pid> [limit-ticks]
|
|
78
|
+
local state=$1 pid=$2 limit=${3:-300} beat first now i=0
|
|
79
|
+
beat="$state/.last-watcher-beat"
|
|
80
|
+
rm -f "$beat"
|
|
81
|
+
first=""
|
|
82
|
+
while [ "$i" -lt "$limit" ]; do
|
|
83
|
+
kill -0 "$pid" 2>/dev/null || return 1
|
|
84
|
+
first=$(file_mtime "$beat")
|
|
85
|
+
[ -n "$first" ] && break
|
|
86
|
+
sleep 0.1
|
|
87
|
+
i=$((i + 1))
|
|
88
|
+
done
|
|
89
|
+
while [ "$i" -lt "$limit" ]; do
|
|
90
|
+
kill -0 "$pid" 2>/dev/null || return 1
|
|
91
|
+
now=$(file_mtime "$beat")
|
|
92
|
+
if [ -n "$now" ] && [ "$now" != "$first" ]; then
|
|
93
|
+
return 0
|
|
94
|
+
fi
|
|
95
|
+
sleep 0.1
|
|
96
|
+
i=$((i + 1))
|
|
97
|
+
done
|
|
98
|
+
return 1
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
# Every wait_for_exit budget in this file is 100 ticks (10s), not because any
|
|
102
|
+
# watcher takes that long to decide, but because fm-watch.sh does bounded
|
|
103
|
+
# startup work before its first poll: a tighter budget reaps the process while
|
|
104
|
+
# it is still starting and reports a spurious "did not surface" failure. A
|
|
105
|
+
# generous budget can only remove that false negative - a watcher that never
|
|
106
|
+
# exits still fails the assertion when the budget runs out.
|
|
107
|
+
wait_numeric_file() {
|
|
108
|
+
local file=$1 limit=${2:-30} i=0 value
|
|
109
|
+
while [ "$i" -lt "$limit" ]; do
|
|
110
|
+
value=$(cat "$file" 2>/dev/null || true)
|
|
111
|
+
case "$value" in
|
|
112
|
+
''|*[!0-9]*) ;;
|
|
113
|
+
*) return 0 ;;
|
|
114
|
+
esac
|
|
115
|
+
sleep 0.1
|
|
116
|
+
i=$((i + 1))
|
|
117
|
+
done
|
|
118
|
+
return 1
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
# Portable mtime in epoch seconds. Platform-detected, never the `stat -f || stat -c`
|
|
122
|
+
# fallback (which writes a partial filesystem dump on Linux; see fm-watch.sh).
|
|
123
|
+
file_mtime() {
|
|
124
|
+
if [ "$(uname)" = Darwin ]; then stat -f %m "$1" 2>/dev/null; else stat -c %Y "$1" 2>/dev/null; fi
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
# Set <file>'s mtime to exactly <epoch> seconds, for aging a busy-turn marker by
|
|
128
|
+
# a precise amount (touch -t takes a local-time stamp, not an epoch, on both
|
|
129
|
+
# platforms, so convert via BSD `date -r` or GNU `date -d @`).
|
|
130
|
+
set_mtime() { # <epoch> <file>
|
|
131
|
+
local epoch=$1 f=$2 stamp
|
|
132
|
+
if stamp=$(date -r "$epoch" +%Y%m%d%H%M.%S 2>/dev/null); then
|
|
133
|
+
touch -t "$stamp" "$f"
|
|
134
|
+
else
|
|
135
|
+
stamp=$(date -d "@$epoch" +%Y%m%d%H%M.%S)
|
|
136
|
+
touch -t "$stamp" "$f"
|
|
137
|
+
fi
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
# Signature a primed .seen-* marker must hold so the per-poll signal scan does not
|
|
141
|
+
# fire on a pre-existing status (mirrors fm-watch.sh's stat_sig exactly).
|
|
142
|
+
seen_sig() {
|
|
143
|
+
local reported size ident
|
|
144
|
+
case "$1" in
|
|
145
|
+
*.status)
|
|
146
|
+
reported=$(status_observed_signature "$1")
|
|
147
|
+
size=$(size_of "$1")
|
|
148
|
+
ident=$(_fm_open_decisions_file_ident "$1")
|
|
149
|
+
printf 'v2\t%s\t%s@%s' "$reported" "$size" "$ident"
|
|
150
|
+
;;
|
|
151
|
+
*)
|
|
152
|
+
if [ "$(uname)" = Darwin ]; then stat -f '%z:%Fm' "$1" 2>/dev/null; else stat -c '%s:%Y' "$1" 2>/dev/null; fi
|
|
153
|
+
;;
|
|
154
|
+
esac
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
# Prime <file>'s .seen-* suppressor to its CURRENT signature, so the per-poll
|
|
158
|
+
# no-verb signal scan (which watches every *.turn-ended for a size:mtime change)
|
|
159
|
+
# treats a just-created or just-backdated turn-ended marker as already seen.
|
|
160
|
+
# Busy-turn-age fixtures create/backdate turn-ended directly (there is no real
|
|
161
|
+
# harness touching it), so without this the marker's own first sighting would
|
|
162
|
+
# fire an unrelated "signal:" wake and mask the busy-turn-age assertion under
|
|
163
|
+
# test. Call again after any further touch/set_mtime on the same file.
|
|
164
|
+
prime_turnend_seen() { # <file>
|
|
165
|
+
local f=$1 base
|
|
166
|
+
base=$(basename "$f" | tr '.' '_')
|
|
167
|
+
printf '%s' "$(seen_sig "$f")" > "$(dirname "$f")/.seen-$base"
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
record_pi_busy() { # <state-dir> <id>
|
|
171
|
+
local state=$1 id=$2 gen
|
|
172
|
+
gen=$("$ROOT/bin/fm-busy-event.sh" arm "$state" "$id")
|
|
173
|
+
"$ROOT/bin/fm-busy-event.sh" apply "$state" "$id" busy --gen "$gen" \
|
|
174
|
+
--source pi-ext --event agent-start
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
reap() { kill "$1" 2>/dev/null || true; wait "$1" 2>/dev/null || true; }
|
|
178
|
+
|
|
179
|
+
# --- pure classifier predicates (fm-classify-lib.sh) ------------------------
|
|
180
|
+
|
|
181
|
+
size_of() { LC_ALL=C wc -c < "$1" | tr -d '[:space:]'; }
|
|
182
|
+
|
|
183
|
+
test_status_span_actionable_classifier() {
|
|
184
|
+
local dir state offset
|
|
185
|
+
dir=$(make_case classify-signal); state="$dir/state"
|
|
186
|
+
printf 'working: step 1\nworking: step 2\n' > "$state/a.status"
|
|
187
|
+
status_span_has_actionable "$state/a.status" 0 && fail "benign working: span classified actionable"
|
|
188
|
+
printf 'working: x\nneeds-decision: pick A or B\n' > "$state/b.status"
|
|
189
|
+
status_span_has_actionable "$state/b.status" 0 || fail "captain-relevant span classified benign"
|
|
190
|
+
# A failure and a merge result are captain-relevant and must always wake.
|
|
191
|
+
printf 'failed: build broke on main\n' > "$state/d.status"
|
|
192
|
+
status_span_has_actionable "$state/d.status" 0 || fail "a failed: line was not actionable"
|
|
193
|
+
printf 'merged\n' > "$state/e.status"
|
|
194
|
+
status_span_has_actionable "$state/e.status" 0 || fail "a legacy merged line was not actionable"
|
|
195
|
+
# An offset past the whole log has nothing left to classify: an event already
|
|
196
|
+
# classified must not re-fire on the next append.
|
|
197
|
+
offset=$(size_of "$state/b.status")
|
|
198
|
+
status_span_has_actionable "$state/b.status" "$offset" \
|
|
199
|
+
&& fail "an already-classified needs-decision re-fired from its own end offset"
|
|
200
|
+
printf 'working: tidying up\n' >> "$state/b.status"
|
|
201
|
+
status_span_has_actionable "$state/b.status" "$offset" \
|
|
202
|
+
&& fail "a routine append after a classified decision was classified actionable"
|
|
203
|
+
# An unusable offset (absent, malformed, or past a truncated log) reads the
|
|
204
|
+
# whole file rather than losing the events it cannot account for.
|
|
205
|
+
status_span_has_actionable "$state/b.status" "" || fail "an empty offset did not read the whole log"
|
|
206
|
+
status_span_has_actionable "$state/b.status" "not-a-number" || fail "a malformed offset did not read the whole log"
|
|
207
|
+
status_span_has_actionable "$state/b.status" 99999 || fail "an offset past the log did not read the whole log"
|
|
208
|
+
pass "status_span_has_actionable: benign absorbed, captain events surfaced, classified events not re-fired"
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
# The reported bug, at the classifier: an actionable event followed by a ROUTINE
|
|
212
|
+
# append must stay actionable, and must be reported as ITSELF rather than as the
|
|
213
|
+
# routine line that happens to sit last.
|
|
214
|
+
test_status_span_survives_a_later_routine_append() {
|
|
215
|
+
local dir state event
|
|
216
|
+
dir=$(make_case classify-masked); state="$dir/state"
|
|
217
|
+
printf 'working: setup\nneeds-decision: pick A or B\nworking: still tidying the branch\n' \
|
|
218
|
+
> "$state/mask.status"
|
|
219
|
+
status_span_has_actionable "$state/mask.status" 0 \
|
|
220
|
+
|| fail "a needs-decision hidden behind a later working: line was classified routine"
|
|
221
|
+
event=$(status_span_first_actionable "$state/mask.status" 0)
|
|
222
|
+
[ "$event" = "needs-decision: pick A or B" ] \
|
|
223
|
+
|| fail "the span reported '$event' instead of the decision it found"
|
|
224
|
+
# The captain-reported shape: a finished release/install reported as done and
|
|
225
|
+
# then followed by routine cleanup chatter must still reach the captain.
|
|
226
|
+
printf 'working: publishing\ndone: release 1.4.0 published and installed\nworking: cleaning the build dir\nnote: cache pruned\n' \
|
|
227
|
+
> "$state/release.status"
|
|
228
|
+
status_span_has_actionable "$state/release.status" 0 \
|
|
229
|
+
|| fail "a done: completion hidden behind later routine appends was classified routine"
|
|
230
|
+
event=$(status_span_first_actionable "$state/release.status" 0)
|
|
231
|
+
[ "$event" = "done: release 1.4.0 published and installed" ] \
|
|
232
|
+
|| fail "the span reported '$event' instead of the completion it found"
|
|
233
|
+
# A blocker is the away-mode shape of the same masking.
|
|
234
|
+
printf 'blocked: cannot reach the release host\npaused: waiting for release access\n' \
|
|
235
|
+
> "$state/blocked.status"
|
|
236
|
+
status_span_has_actionable "$state/blocked.status" 0 \
|
|
237
|
+
|| fail "a blocked: event hidden behind a current wait was classified routine"
|
|
238
|
+
pass "an actionable event is not hidden by later routine appends, and is named as itself"
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
# Closure is the one thing that may retire an event inside a span, and only
|
|
242
|
+
# through status_open_decisions' own open/closed rule.
|
|
243
|
+
test_status_span_respects_decision_closure() {
|
|
244
|
+
local dir state event open
|
|
245
|
+
dir=$(make_case classify-closure); state="$dir/state"
|
|
246
|
+
printf 'needs-decision [key=api]: pick A or B\nresolved [key=api]: took A\n' > "$state/closed.status"
|
|
247
|
+
status_span_has_actionable "$state/closed.status" 0 \
|
|
248
|
+
&& fail "a decision the same span already closed was still classified actionable"
|
|
249
|
+
# Reopening the SAME key after a close must survive: the close belongs to the
|
|
250
|
+
# earlier opening, not to the one that came after it.
|
|
251
|
+
printf 'needs-decision [key=api]: pick A or B\nresolved [key=api]: took A\nneeds-decision: [key=api] pick A or B\n' \
|
|
252
|
+
> "$state/reopened.status"
|
|
253
|
+
event=$(status_span_first_actionable "$state/reopened.status" 0) \
|
|
254
|
+
|| fail "a decision reopened under a key that was closed earlier was classified routine"
|
|
255
|
+
[ "$event" = "needs-decision: [key=api] pick A or B" ] \
|
|
256
|
+
|| fail "the reopened key surfaced its closed opening instead of the live reopening: $event"
|
|
257
|
+
# A terminal event is never retired by a later closure line.
|
|
258
|
+
printf 'failed: build broke on main\nresolved [key=api]: unrelated\n' > "$state/term.status"
|
|
259
|
+
status_span_has_actionable "$state/term.status" 0 \
|
|
260
|
+
|| fail "a failed: event was retired by an unrelated closure"
|
|
261
|
+
# A live decision must survive a NEWER closure that belongs to another key.
|
|
262
|
+
printf 'needs-decision [key=api]: pick A or B\nneeds-decision [key=db]: pick a store\nresolved [key=db]: took sqlite\n' \
|
|
263
|
+
> "$state/two.status"
|
|
264
|
+
event=$(status_span_first_actionable "$state/two.status" 0) \
|
|
265
|
+
|| fail "a still-open decision was retired by a newer closure under another key"
|
|
266
|
+
[ "$event" = "needs-decision [key=api]: pick A or B" ] \
|
|
267
|
+
|| fail "the span reported '$event' instead of the decision still open"
|
|
268
|
+
printf 'needs-decision [key=pending-reply-x]: unrelated request\nworking: awaiting reconciliation\n' \
|
|
269
|
+
> "$state/rejected-reserved.status"
|
|
270
|
+
event=$(status_span_first_actionable "$state/rejected-reserved.status" 0) \
|
|
271
|
+
|| fail "a rejected reserved-key request was silently dropped"
|
|
272
|
+
[ "$event" = "reconciliation-required: needs-decision [key=pending-reply-x]: unrelated request" ] \
|
|
273
|
+
|| fail "a rejected reserved-key request was not labeled for reconciliation: $event"
|
|
274
|
+
open=$(status_open_decisions "$state/rejected-reserved.status")
|
|
275
|
+
[ -z "$open" ] \
|
|
276
|
+
|| fail "span classification treated a rejected reserved-key request as an open decision: $open"
|
|
277
|
+
pass "span classification retires closed decisions and surfaces rejected transitions for reconciliation"
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
test_malformed_seen_signature_reads_the_whole_log() {
|
|
281
|
+
local dir state f marker offset
|
|
282
|
+
dir=$(make_case malformed-seen); state="$dir/state"; f="$state/task.status"
|
|
283
|
+
printf 'needs-decision: choose the release target\nworking: cleanup\n' > "$f"
|
|
284
|
+
marker="$state/.seen-task_status"
|
|
285
|
+
printf '40' > "$marker"
|
|
286
|
+
offset=$(bash -c '. "$1"; fm_wake_signal_seen_size "$2" "$3"' _ \
|
|
287
|
+
"$ROOT/bin/fm-wake-lib.sh" "$state" "$f")
|
|
288
|
+
[ "$offset" = 0 ] \
|
|
289
|
+
|| fail "a digits-only malformed seen signature was accepted as an offset"
|
|
290
|
+
status_span_has_actionable "$f" "$offset" \
|
|
291
|
+
|| fail "a malformed seen signature skipped the actionable start of the log"
|
|
292
|
+
pass "a malformed seen signature causes the whole status log to be classified"
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
test_stale_is_terminal_classifier() {
|
|
296
|
+
local dir state
|
|
297
|
+
dir=$(make_case classify-stale); state="$dir/state"
|
|
298
|
+
printf 'done: ready in branch fm/x\n' > "$state/term.status"
|
|
299
|
+
stale_is_terminal "sess:fm-term" "$state" || fail "terminal stale status not classified terminal"
|
|
300
|
+
fm_write_meta "$state/herdr-term.meta" "window=default:w1:p2" "backend=herdr"
|
|
301
|
+
printf 'done: ready in branch fm/herdr\n' > "$state/herdr-term.status"
|
|
302
|
+
stale_is_terminal "default:w1:p2" "$state" || fail "terminal herdr stale status not resolved through metadata"
|
|
303
|
+
printf 'working: compiling\n' > "$state/nonterm.status"
|
|
304
|
+
stale_is_terminal "sess:fm-nonterm" "$state" && fail "non-terminal stale classified terminal"
|
|
305
|
+
stale_is_terminal "sess:fm-missing" "$state" && fail "stale with no status classified terminal"
|
|
306
|
+
pass "stale_is_terminal: terminal status surfaces, non-terminal and no-status are benign"
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
test_classifier_primitives() {
|
|
310
|
+
local dir state open activity
|
|
311
|
+
dir=$(make_case classify-primitives); state="$dir/state"
|
|
312
|
+
printf 'working: a\n\ndone: b\n\n' > "$state/x.status"
|
|
313
|
+
[ "$(last_status_line "$state/x.status")" = "done: b" ] || fail "last_status_line did not return the last non-blank line"
|
|
314
|
+
status_is_captain_relevant "done: b" || fail "done: not recognized as captain-relevant"
|
|
315
|
+
status_is_captain_relevant "needs-decision [key=q1]: b" || fail "keyed needs-decision not recognized as captain-relevant"
|
|
316
|
+
status_is_captain_relevant "working: b" && fail "working: wrongly recognized as captain-relevant"
|
|
317
|
+
# Incident regression: free-text "merged" inside a nonterminal working: line must
|
|
318
|
+
# not become captain-relevant (AFK false-terminal path).
|
|
319
|
+
status_is_captain_relevant \
|
|
320
|
+
"working: stage 2 setup complete on PR #74 exact source branch rebased onto merged #76; task dates preserved" \
|
|
321
|
+
&& fail "working: ... merged #N wrongly recognized as captain-relevant"
|
|
322
|
+
status_is_captain_relevant "working: rebased onto predecessor #76" \
|
|
323
|
+
&& fail "working: predecessor prose wrongly recognized as captain-relevant"
|
|
324
|
+
status_is_captain_relevant "working: PR ready checks green merged ready in branch" \
|
|
325
|
+
&& fail "working: free-text tokens wrongly recognized as captain-relevant"
|
|
326
|
+
status_is_captain_relevant "done: PR https://x/pull/76 checks green" \
|
|
327
|
+
|| fail "genuine done: checks green not captain-relevant"
|
|
328
|
+
status_is_terminal_verb "done: PR https://x/pull/76 checks green" \
|
|
329
|
+
|| fail "done: not a terminal verb"
|
|
330
|
+
status_is_terminal_verb "working: rebased onto merged #76" \
|
|
331
|
+
&& fail "working: wrongly classed as terminal verb"
|
|
332
|
+
status_is_captain_relevant "merged" || fail "legacy bare merged free-text not captain-relevant"
|
|
333
|
+
status_is_captain_relevant "PR ready https://x/pull/2" \
|
|
334
|
+
|| fail "legacy bare PR ready free-text not captain-relevant"
|
|
335
|
+
[ "$(window_to_task "sess:fm-fix-login-k3")" = "fix-login-k3" ] || fail "window_to_task did not strip session+fm- prefix"
|
|
336
|
+
fm_write_meta "$state/herdr-task.meta" "window=default:w1:p2" "backend=herdr"
|
|
337
|
+
[ "$(window_to_task "default:w1:p2" "$state")" = "herdr-task" ] || fail "window_to_task did not resolve opaque backend target through metadata"
|
|
338
|
+
FM_CAPTAIN_RE='custom-verb:' status_is_captain_relevant "custom-verb: x" || fail "FM_CAPTAIN_RE override not honored"
|
|
339
|
+
FM_CAPTAIN_RE='custom-verb:' status_is_captain_relevant "done: x" && fail "FM_CAPTAIN_RE override did not replace the default verb set"
|
|
340
|
+
FM_CAPTAIN_RE='merged|custom-verb:' status_is_captain_relevant "working: rebased onto merged #76" \
|
|
341
|
+
&& fail "FM_CAPTAIN_RE override bypassed working: suppression"
|
|
342
|
+
FM_CAPTAIN_RE='checks green|custom-verb:' status_is_captain_relevant "paused: checks green pending approval" \
|
|
343
|
+
&& fail "FM_CAPTAIN_RE override bypassed paused: suppression"
|
|
344
|
+
FM_CAPTAIN_RE='custom-verb:' status_is_captain_relevant "custom-verb: x" \
|
|
345
|
+
|| fail "nonterminal suppression weakened custom bare-line behavior"
|
|
346
|
+
printf 'needs-decision: should docs mention [key=prose]?\nneeds-decision [key=q1]: real choice\nresolved: docs still mention [key=q1]\nneeds-decision [key=bad key]: malformed\n' > "$state/keys.status"
|
|
347
|
+
open=$(status_open_decisions "$state/keys.status")
|
|
348
|
+
printf '%s' "$open" | grep -F $'q1\t' >/dev/null \
|
|
349
|
+
|| fail "a key token in resolved note prose closed the keyed decision"
|
|
350
|
+
printf '%s' "$open" | grep -F $'prose\t' >/dev/null \
|
|
351
|
+
&& fail "a key token in note prose changed the decision key"
|
|
352
|
+
printf '%s' "$open" | grep -F $'bad key\t' >/dev/null \
|
|
353
|
+
&& fail "an invalid key slug entered the open-decision set"
|
|
354
|
+
cat > "$state/activity.status" <<'EOF'
|
|
355
|
+
working [key=phase7]: Phase 7 started
|
|
356
|
+
working [key=phase6]: Phase 6 started
|
|
357
|
+
working [key=legal]: reviewing legal dependency
|
|
358
|
+
done [key=phase6]: Phase 6 completed
|
|
359
|
+
resolved [key=phase7]: Phase 7 completed and moved to Done
|
|
360
|
+
paused [key=legal]: awaiting external counsel
|
|
361
|
+
resolved [key=legal]: legal item returned to the queue
|
|
362
|
+
working [key=phase8]: Phase 8 started
|
|
363
|
+
EOF
|
|
364
|
+
activity=$(status_open_activities "$state/activity.status")
|
|
365
|
+
printf '%s' "$activity" | grep -F $'phase8\tworking\tPhase 8 started' >/dev/null \
|
|
366
|
+
|| fail "the current keyed working phase was not retained"
|
|
367
|
+
printf '%s' "$activity" | grep -F $'phase7\t' >/dev/null \
|
|
368
|
+
&& fail "a keyed resolved event did not close the older working phase"
|
|
369
|
+
printf '%s' "$activity" | grep -F $'phase6\t' >/dev/null \
|
|
370
|
+
&& fail "a same-key terminal event did not supersede the older working phase"
|
|
371
|
+
printf '%s' "$activity" | grep -F $'legal\t' >/dev/null \
|
|
372
|
+
&& fail "a keyed resolved event did not close the declared pause"
|
|
373
|
+
printf 'working: legacy start\ndone: legacy completion\n' > "$state/legacy-activity.status"
|
|
374
|
+
[ -z "$(status_open_activities "$state/legacy-activity.status")" ] \
|
|
375
|
+
|| fail "a legacy terminal event did not supersede the default working phase"
|
|
376
|
+
pass "classifier primitives: keyed decisions and activity phases, captain relevance, window-to-task, and overrides"
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
# crew_is_provably_working: the absorb-only-when-provably-working predicate. It is
|
|
380
|
+
# benign (absorb) ONLY when fm-crew-state.sh reports the crew as working from an
|
|
381
|
+
# actively-running pipeline step (source run-step) or a busy pane (source pane);
|
|
382
|
+
# everything else - a stale working: status-log line, a finished/parked/failed run,
|
|
383
|
+
# an unknown/torn-down crew, or an empty id - is NOT provable, so it surfaces. The
|
|
384
|
+
# fake fm-crew-state.sh (FM_CREW_STATE_BIN) returns a canned verdict per case.
|
|
385
|
+
test_crew_is_provably_working_classifier() {
|
|
386
|
+
local dir fakebin
|
|
387
|
+
dir=$(make_case provably-working); fakebin="$dir/fakebin"
|
|
388
|
+
# Point the predicate at this case's hermetic fake and drive its verdict per case.
|
|
389
|
+
# export marks the var for the fake subprocess; it is unset again at the end so it
|
|
390
|
+
# cannot leak into a later test (every behavioral test sets its own verdict anyway).
|
|
391
|
+
export FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh"
|
|
392
|
+
export FM_FAKE_CREW_STATE
|
|
393
|
+
FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
394
|
+
crew_is_provably_working a || fail "active run-step not treated as provably working"
|
|
395
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy'
|
|
396
|
+
crew_is_provably_working a || fail "busy pane not treated as provably working"
|
|
397
|
+
FM_FAKE_CREW_STATE='state: working · source: status-log · working: compiling'
|
|
398
|
+
! crew_is_provably_working a || fail "stale status-log working: treated as provably working"
|
|
399
|
+
FM_FAKE_CREW_STATE='state: done · source: run-step · checks green'
|
|
400
|
+
! crew_is_provably_working a || fail "finished run treated as provably working"
|
|
401
|
+
FM_FAKE_CREW_STATE='state: parked · source: run-step · parked at review'
|
|
402
|
+
! crew_is_provably_working a || fail "parked run treated as provably working"
|
|
403
|
+
FM_FAKE_CREW_STATE='state: failed · source: run-step · run failed'
|
|
404
|
+
! crew_is_provably_working a || fail "failed run treated as provably working"
|
|
405
|
+
FM_FAKE_CREW_STATE='state: unknown · source: none · worktree gone'
|
|
406
|
+
! crew_is_provably_working a || fail "unknown crew treated as provably working"
|
|
407
|
+
FM_FAKE_CREW_STATE='state: working · source: run-step · x'
|
|
408
|
+
! crew_is_provably_working "" || fail "empty id treated as provably working"
|
|
409
|
+
unset FM_FAKE_CREW_STATE
|
|
410
|
+
pass "crew_is_provably_working: only working+run-step/pane is provable; idle/finished/parked/failed/unknown surface"
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
# status_is_paused: the shared pause verb test both consumers read (so neither
|
|
414
|
+
# hardcodes the literal). Matches only the verb before the first colon, so a reason
|
|
415
|
+
# that merely mentions "paused" does not false-match, and a genuine blocker stays a
|
|
416
|
+
# blocker.
|
|
417
|
+
test_status_is_paused_classifier() {
|
|
418
|
+
status_is_paused 'paused: holding for the upstream release' || fail "paused verb not recognized"
|
|
419
|
+
status_is_paused ' paused: waiting on a rate-limit reset' || fail "leading-space paused verb not recognized"
|
|
420
|
+
status_is_paused 'blocked: the build is paused upstream' && fail "a blocked line mentioning paused false-matched"
|
|
421
|
+
status_is_paused 'working: paused the animation loop' && fail "a working line mentioning paused false-matched"
|
|
422
|
+
status_is_paused 'done: shipped' && fail "done classified as paused"
|
|
423
|
+
status_is_paused '' && fail "empty line classified as paused"
|
|
424
|
+
# A pause is deliberately NOT captain-relevant: it is a stop-nagging signal, not
|
|
425
|
+
# work to keep surfacing.
|
|
426
|
+
status_is_captain_relevant 'paused: holding for the upstream release' && fail "paused is captain-relevant (should not be)"
|
|
427
|
+
status_is_paused_or_captain_held 'paused: holding for the upstream release' \
|
|
428
|
+
|| fail "declared pause not recognized by the bounded-idle classifier"
|
|
429
|
+
status_is_paused_or_captain_held 'captain-held [key=route]: tracked by task-decision-route' \
|
|
430
|
+
|| fail "captain-held transfer not recognized by the bounded-idle classifier"
|
|
431
|
+
status_is_paused_or_captain_held 'resolved [key=route]: captain answered' \
|
|
432
|
+
&& fail "resolved decision remained classed as captain-held"
|
|
433
|
+
# The two declarations share one cadence but block on different humans, so the
|
|
434
|
+
# combined predicate cannot be the only discriminator: a recheck has to know which
|
|
435
|
+
# verb it is naming.
|
|
436
|
+
status_is_captain_held 'captain-held [key=route]: tracked by task-decision-route' \
|
|
437
|
+
|| fail "captain-held verb not recognized"
|
|
438
|
+
status_is_captain_held 'paused: holding for the upstream release' \
|
|
439
|
+
&& fail "a declared pause matched the captain-held verb"
|
|
440
|
+
status_is_captain_held 'working: the captain-held backlog item is next' \
|
|
441
|
+
&& fail "a working line mentioning captain-held false-matched"
|
|
442
|
+
status_is_captain_held '' && fail "empty line classified as captain-held"
|
|
443
|
+
pass "status_is_paused: only the leading paused verb matches, paused is not captain-relevant, and the two declared-wait verbs stay separable"
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
# crew_absorb_class: the single fm-crew-state.sh read that returns BOTH absorb
|
|
447
|
+
# reasons - working (active run/busy pane), paused (declared external wait), or none
|
|
448
|
+
# (surface it) - so the watcher's stale path gets both for one bounded call.
|
|
449
|
+
# crew_is_paused delegates to it exactly as crew_is_provably_working does.
|
|
450
|
+
test_crew_absorb_class_classifier() {
|
|
451
|
+
local dir fakebin
|
|
452
|
+
dir=$(make_case absorb-class); fakebin="$dir/fakebin"
|
|
453
|
+
export FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh"
|
|
454
|
+
export FM_FAKE_CREW_STATE
|
|
455
|
+
FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
456
|
+
[ "$(crew_absorb_class a)" = working ] || fail "active run-step not classed working"
|
|
457
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy'
|
|
458
|
+
[ "$(crew_absorb_class a)" = working ] || fail "busy pane not classed working"
|
|
459
|
+
FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting upstream'
|
|
460
|
+
[ "$(crew_absorb_class a)" = paused ] || fail "declared pause not classed paused"
|
|
461
|
+
crew_is_paused a || fail "crew_is_paused did not recognize a paused verdict"
|
|
462
|
+
! crew_is_provably_working a || fail "a paused crew was treated as provably working"
|
|
463
|
+
FM_FAKE_CREW_STATE='state: working · source: status-log · working: compiling'
|
|
464
|
+
[ "$(crew_absorb_class a)" = none ] || fail "stale working: status-log classed absorbable"
|
|
465
|
+
FM_FAKE_CREW_STATE='state: unknown · source: none · worktree gone'
|
|
466
|
+
[ "$(crew_absorb_class a)" = none ] || fail "unknown crew classed absorbable"
|
|
467
|
+
! crew_is_paused a || fail "unknown crew classed paused"
|
|
468
|
+
[ "$(crew_absorb_class "")" = none ] || fail "empty id not classed none"
|
|
469
|
+
unset FM_FAKE_CREW_STATE
|
|
470
|
+
pass "crew_absorb_class: working/paused/none from one read; crew_is_paused and crew_is_provably_working agree"
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
# The wedge detector's third liveness input: writes inside the crew's own recorded
|
|
474
|
+
# worktree. Every negative outcome must report "no evidence" so the caller keeps
|
|
475
|
+
# its existing escalation schedule, and a supervisor-side git read (which touches
|
|
476
|
+
# .git, never tracked files) must not be able to fake a positive.
|
|
477
|
+
test_crew_worktree_written_since_classifier() {
|
|
478
|
+
local dir state anchor wt home statedir_wt
|
|
479
|
+
dir=$(make_case classify-worktree-writes); state="$dir/state"
|
|
480
|
+
anchor="$state/anchor"; wt="$dir/wt"; home="$dir/mate-home"; statedir_wt="$dir/wt-with-state"
|
|
481
|
+
mkdir -p "$wt/src" "$wt/.git/objects"
|
|
482
|
+
printf 'old\n' > "$wt/src/existing.c"
|
|
483
|
+
set_mtime "$(( $(date +%s) - 300 ))" "$wt/src/existing.c"
|
|
484
|
+
: > "$anchor"
|
|
485
|
+
set_mtime "$(( $(date +%s) - 120 ))" "$anchor"
|
|
486
|
+
|
|
487
|
+
# No recorded worktree at all: absence of evidence, never a positive.
|
|
488
|
+
printf 'window=test:fm-a\nkind=ship\n' > "$state/a.meta"
|
|
489
|
+
! crew_worktree_written_since a "$state" "$anchor" \
|
|
490
|
+
|| fail "a task with no recorded worktree reported write evidence"
|
|
491
|
+
# Recorded but gone (torn down): still no evidence.
|
|
492
|
+
printf 'window=test:fm-b\nkind=ship\nworktree=%s\n' "$dir/missing" > "$state/b.meta"
|
|
493
|
+
! crew_worktree_written_since b "$state" "$anchor" \
|
|
494
|
+
|| fail "a torn-down worktree reported write evidence"
|
|
495
|
+
# Present, but nothing written since the anchor.
|
|
496
|
+
printf 'window=test:fm-c\nkind=ship\nworktree=%s\n' "$wt" > "$state/c.meta"
|
|
497
|
+
! crew_worktree_written_since c "$state" "$anchor" \
|
|
498
|
+
|| fail "a quiet worktree reported write evidence"
|
|
499
|
+
# A missing anchor cannot be compared against: no evidence.
|
|
500
|
+
! crew_worktree_written_since c "$state" "$state/absent-anchor" \
|
|
501
|
+
|| fail "a missing anchor reported write evidence"
|
|
502
|
+
# Only .git churn (what firstmate's own read-only git commands touch): pruned.
|
|
503
|
+
printf 'pack\n' > "$wt/.git/objects/fresh"
|
|
504
|
+
printf 'ref\n' > "$wt/.git/index"
|
|
505
|
+
! crew_worktree_written_since c "$state" "$anchor" \
|
|
506
|
+
|| fail ".git churn alone reported write evidence (a supervisor read could fake liveness)"
|
|
507
|
+
# A real file written after the anchor: positive evidence.
|
|
508
|
+
printf 'new\n' > "$wt/src/new.c"
|
|
509
|
+
crew_worktree_written_since c "$state" "$anchor" \
|
|
510
|
+
|| fail "a file written after the anchor was not reported as write evidence"
|
|
511
|
+
# An empty id is never evidence.
|
|
512
|
+
! crew_worktree_written_since "" "$state" "$anchor" || fail "an empty id reported write evidence"
|
|
513
|
+
|
|
514
|
+
# A secondmate records a provisioned firstmate home, not a code tree, and such a
|
|
515
|
+
# home supervises itself: its own watcher beacon, pane hashes, and heartbeats keep
|
|
516
|
+
# its state/ churning whether or not the mate produced anything.
|
|
517
|
+
mkdir -p "$home/state"
|
|
518
|
+
printf 'sm-classify-1\n' > "$home/.fm-secondmate-home"
|
|
519
|
+
printf 'beat\n' > "$home/state/.last-watcher-beat"
|
|
520
|
+
printf 'window=remote:sm\nkind=secondmate\nworktree=%s\n' "$home" > "$state/sm.meta"
|
|
521
|
+
! crew_worktree_written_since sm "$state" "$anchor" \
|
|
522
|
+
|| fail "a secondmate's own home supervision churn reported crew write evidence"
|
|
523
|
+
# The home marker alone is enough, even when the record does not say secondmate.
|
|
524
|
+
printf 'window=test:fm-sm2\nkind=ship\nworktree=%s\n' "$home" > "$state/sm2.meta"
|
|
525
|
+
! crew_worktree_written_since sm2 "$state" "$anchor" \
|
|
526
|
+
|| fail "a marked firstmate home reported crew write evidence"
|
|
527
|
+
# But an ordinary worktree that merely holds a directory named state is real
|
|
528
|
+
# work: only the home is excluded, never a source directory of that name.
|
|
529
|
+
mkdir -p "$statedir_wt/state"
|
|
530
|
+
printf 'machine\n' > "$statedir_wt/state/machine.go"
|
|
531
|
+
printf 'window=test:fm-d\nkind=ship\nworktree=%s\n' "$statedir_wt" > "$state/d.meta"
|
|
532
|
+
crew_worktree_written_since d "$state" "$anchor" \
|
|
533
|
+
|| fail "a source directory named state was hidden from the write probe"
|
|
534
|
+
pass "crew_worktree_written_since: real writes are evidence; no worktree, no anchor, quiet trees, .git churn and a mate's own home are not"
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
# FM_WORKTREE_WRITE_PRUNE is a skip list, so clearing it skips nothing and is the
|
|
538
|
+
# obvious way to widen the probe to the whole depth-bounded tree. An empty list must
|
|
539
|
+
# therefore widen the walk rather than report no evidence at all, which would
|
|
540
|
+
# quietly cost the wedge detector its third liveness input on a home that cleared
|
|
541
|
+
# the knob to get more coverage, not less.
|
|
542
|
+
test_empty_write_prune_widens_the_probe() {
|
|
543
|
+
local dir state anchor wt saved
|
|
544
|
+
dir=$(make_case classify-empty-write-prune); state="$dir/state"
|
|
545
|
+
anchor="$state/anchor"; wt="$dir/wt"
|
|
546
|
+
mkdir -p "$wt/src" "$wt/.git"
|
|
547
|
+
: > "$anchor"
|
|
548
|
+
set_mtime "$(( $(date +%s) - 120 ))" "$anchor"
|
|
549
|
+
printf 'window=test:fm-e\nkind=ship\nworktree=%s\n' "$wt" > "$state/e.meta"
|
|
550
|
+
saved=$FM_WORKTREE_WRITE_PRUNE
|
|
551
|
+
FM_WORKTREE_WRITE_PRUNE=''
|
|
552
|
+
# A quiet tree is still no evidence, so the caller's schedule is untouched.
|
|
553
|
+
! crew_worktree_written_since e "$state" "$anchor" \
|
|
554
|
+
|| fail "an empty prune list reported write evidence for a quiet worktree"
|
|
555
|
+
printf 'new\n' > "$wt/src/new.c"
|
|
556
|
+
crew_worktree_written_since e "$state" "$anchor" \
|
|
557
|
+
|| fail "an empty prune list disabled the probe instead of widening it"
|
|
558
|
+
# Widened means nothing is skipped, including what the default list prunes.
|
|
559
|
+
set_mtime "$(( $(date +%s) - 900 ))" "$wt/src/new.c"
|
|
560
|
+
printf 'pack\n' > "$wt/.git/index"
|
|
561
|
+
crew_worktree_written_since e "$state" "$anchor" \
|
|
562
|
+
|| fail "an empty prune list still skipped a directory the default list prunes"
|
|
563
|
+
# Restoring the default prunes .git again, so a supervisor's own read-only git
|
|
564
|
+
# command still cannot fake liveness.
|
|
565
|
+
FM_WORKTREE_WRITE_PRUNE=$saved
|
|
566
|
+
! crew_worktree_written_since e "$state" "$anchor" \
|
|
567
|
+
|| fail "the default prune list stopped keeping .git out of the probe"
|
|
568
|
+
pass "an empty FM_WORKTREE_WRITE_PRUNE widens the probe to the whole depth-bounded tree instead of disabling it"
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
# The same widening, reached the way a home actually configures it: through the
|
|
572
|
+
# process ENVIRONMENT, not an in-process assignment made after the library was
|
|
573
|
+
# sourced. An empty exported value must survive as empty, because defaulting it with
|
|
574
|
+
# the colon form reads "explicitly cleared" as "never set" and hands the default skip
|
|
575
|
+
# list straight back to the one home that asked for a wider walk.
|
|
576
|
+
# shellcheck disable=SC2016 # single quotes are deliberate: the library path, state dir, and anchor expand inside the bash -c child, not here
|
|
577
|
+
test_empty_write_prune_from_the_environment_widens_the_probe() {
|
|
578
|
+
local dir state anchor wt
|
|
579
|
+
dir=$(make_case classify-empty-write-prune-env); state="$dir/state"
|
|
580
|
+
anchor="$state/anchor"; wt="$dir/wt"
|
|
581
|
+
mkdir -p "$wt/.git/objects"
|
|
582
|
+
: > "$anchor"
|
|
583
|
+
set_mtime "$(( $(date +%s) - 120 ))" "$anchor"
|
|
584
|
+
printf 'window=test:fm-wenv\nkind=ship\nworktree=%s\n' "$wt" > "$state/wenv.meta"
|
|
585
|
+
# The one thing written since the anchor sits exactly where the DEFAULT list prunes.
|
|
586
|
+
printf 'pack\n' > "$wt/.git/objects/fresh"
|
|
587
|
+
env -u FM_WORKTREE_WRITE_PRUNE \
|
|
588
|
+
bash -c '. "$1"; crew_worktree_written_since wenv "$2" "$3"' _ \
|
|
589
|
+
"$ROOT/bin/fm-classify-lib.sh" "$state" "$anchor" \
|
|
590
|
+
&& fail "the default skip list let .git churn count as write evidence"
|
|
591
|
+
FM_WORKTREE_WRITE_PRUNE='' \
|
|
592
|
+
bash -c '. "$1"; crew_worktree_written_since wenv "$2" "$3"' _ \
|
|
593
|
+
"$ROOT/bin/fm-classify-lib.sh" "$state" "$anchor" \
|
|
594
|
+
|| fail "an empty FM_WORKTREE_WRITE_PRUNE in the environment fell back to the default skip list instead of widening the probe"
|
|
595
|
+
pass "an empty FM_WORKTREE_WRITE_PRUNE exported into the environment prunes nothing, widening the probe"
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
# The probe's walk runs synchronously inside the poll that was about to escalate, so
|
|
599
|
+
# it must be wall-clock bounded: -xdev keeps it out of a nested mount, but a worktree
|
|
600
|
+
# root that is ITSELF on a hung mount would otherwise stall the very supervisor that
|
|
601
|
+
# exists to notice a wedge. A fake find that never returns in time stands in for that
|
|
602
|
+
# mount. Hitting the bound must read as NO evidence, exactly like every other
|
|
603
|
+
# negative outcome, so the caller's escalation schedule is untouched.
|
|
604
|
+
test_worktree_write_probe_is_wall_clock_bounded() {
|
|
605
|
+
local dir state anchor wt slowbin fastbin started elapsed
|
|
606
|
+
dir=$(make_case classify-write-probe-bound); state="$dir/state"
|
|
607
|
+
anchor="$state/anchor"; wt="$dir/wt"; slowbin="$dir/slowbin"; fastbin="$dir/fastbin"
|
|
608
|
+
mkdir -p "$wt/src" "$slowbin" "$fastbin"
|
|
609
|
+
: > "$anchor"
|
|
610
|
+
set_mtime "$(( $(date +%s) - 120 ))" "$anchor"
|
|
611
|
+
printf 'window=test:fm-slow\nkind=ship\nworktree=%s\n' "$wt" > "$state/slow.meta"
|
|
612
|
+
# Both stand-ins report the same hit; only one of them takes longer than the bound
|
|
613
|
+
# to do it, so the prompt one shows what a positive outcome looks like and the
|
|
614
|
+
# bounded assertion below cannot pass merely because the fake failed.
|
|
615
|
+
cat > "$fastbin/find" <<'SH'
|
|
616
|
+
#!/usr/bin/env bash
|
|
617
|
+
set -u
|
|
618
|
+
printf '%s\n' "$1/hit"
|
|
619
|
+
SH
|
|
620
|
+
cat > "$slowbin/find" <<'SH'
|
|
621
|
+
#!/usr/bin/env bash
|
|
622
|
+
set -u
|
|
623
|
+
sleep 30
|
|
624
|
+
printf '%s\n' "$1/hit"
|
|
625
|
+
SH
|
|
626
|
+
chmod +x "$fastbin/find" "$slowbin/find"
|
|
627
|
+
PATH="$fastbin:$PATH" \
|
|
628
|
+
bash -c '. "$1"; crew_worktree_written_since slow "$2" "$3"' _ \
|
|
629
|
+
"$ROOT/bin/fm-classify-lib.sh" "$state" "$anchor" \
|
|
630
|
+
|| fail "a walk that reported a hit inside its bound was not read as write evidence"
|
|
631
|
+
started=$(date +%s)
|
|
632
|
+
PATH="$slowbin:$PATH" FM_WORKTREE_WRITE_TIMEOUT=1 \
|
|
633
|
+
bash -c '. "$1"; crew_worktree_written_since slow "$2" "$3"' _ \
|
|
634
|
+
"$ROOT/bin/fm-classify-lib.sh" "$state" "$anchor" \
|
|
635
|
+
&& fail "a walk that outlived its bound was reported as write evidence"
|
|
636
|
+
elapsed=$(( $(date +%s) - started ))
|
|
637
|
+
[ "$elapsed" -lt 10 ] \
|
|
638
|
+
|| fail "the worktree write probe was not wall-clock bounded: one walk held the caller for ${elapsed}s"
|
|
639
|
+
pass "the worktree write probe is wall-clock bounded, and hitting the bound reads as no write evidence"
|
|
640
|
+
}
|
|
641
|
+
|
|
642
|
+
# signal_crew_provably_working: a no-verb "signal:" wake is benign ONLY when EVERY
|
|
643
|
+
# task it references is provably working; if any crew has stopped, or no task can be
|
|
644
|
+
# resolved, it surfaces. Files map to ids by stripping .status / .turn-ended.
|
|
645
|
+
test_signal_crew_provably_working_classifier() {
|
|
646
|
+
local dir fakebin state
|
|
647
|
+
dir=$(make_case signal-provably-working); fakebin="$dir/fakebin"; state="$dir/state"
|
|
648
|
+
export FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh"
|
|
649
|
+
export FM_FAKE_CREW_STATE_a='state: working · source: run-step · running'
|
|
650
|
+
export FM_FAKE_CREW_STATE_b='state: done · source: run-step · run passed'
|
|
651
|
+
signal_crew_provably_working "$state/a.status" "$state/a.turn-ended" \
|
|
652
|
+
|| fail "a single provably-working crew (status+turn-end) was not benign"
|
|
653
|
+
! signal_crew_provably_working "$state/a.status" "$state/b.turn-ended" \
|
|
654
|
+
|| fail "a coalesced batch including a stopped crew was treated as benign"
|
|
655
|
+
! signal_crew_provably_working "$state/b.turn-ended" \
|
|
656
|
+
|| fail "a stopped crew's bare turn-end was treated as benign"
|
|
657
|
+
! signal_crew_provably_working "$state/a.meta" \
|
|
658
|
+
|| fail "a non-signal file resolved to a benign verdict"
|
|
659
|
+
! signal_crew_provably_working \
|
|
660
|
+
|| fail "an empty signal file list was treated as benign"
|
|
661
|
+
unset FM_FAKE_CREW_STATE_a FM_FAKE_CREW_STATE_b
|
|
662
|
+
pass "signal_crew_provably_working: benign only when every referenced crew is provably working"
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
test_secondmate_status_signal_never_absorbed_classifier() {
|
|
666
|
+
local dir fakebin state
|
|
667
|
+
dir=$(make_case secondmate-signal-classify); fakebin="$dir/fakebin"; state="$dir/state"
|
|
668
|
+
export FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh"
|
|
669
|
+
# Even PROVABLY working, a secondmate's .status signal is its routed-reply
|
|
670
|
+
# channel and must surface; its bare turn-ended keeps the ordinary absorb.
|
|
671
|
+
export FM_FAKE_CREW_STATE_sm='state: working · source: run-step · running'
|
|
672
|
+
printf 'kind=secondmate\n' > "$state/sm.meta"
|
|
673
|
+
printf 'working: routed reply for the parent\n' > "$state/sm.status"
|
|
674
|
+
! signal_crew_provably_working "$state/sm.status" \
|
|
675
|
+
|| fail "a working secondmate's status signal was treated as absorbable"
|
|
676
|
+
signal_crew_provably_working "$state/sm.turn-ended" \
|
|
677
|
+
|| fail "a working secondmate's bare turn-end lost its ordinary absorb"
|
|
678
|
+
# An ordinary crewmate with the same verdict stays absorbable: the rule is
|
|
679
|
+
# keyed on recorded kind, not on task naming or content guessing.
|
|
680
|
+
export FM_FAKE_CREW_STATE_crew='state: working · source: run-step · running'
|
|
681
|
+
printf 'kind=ship\n' > "$state/crew.meta"
|
|
682
|
+
printf 'working: progress\n' > "$state/crew.status"
|
|
683
|
+
signal_crew_provably_working "$state/crew.status" \
|
|
684
|
+
|| fail "the secondmate rule leaked onto an ordinary crewmate status"
|
|
685
|
+
unset FM_FAKE_CREW_STATE_sm FM_FAKE_CREW_STATE_crew
|
|
686
|
+
pass "a secondmate's status signal is never absorbed as provably working; crewmates are unaffected"
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
# --- benign wakes are absorbed ONLY when the crew is provably working ---------
|
|
690
|
+
|
|
691
|
+
test_provably_working_signal_absorbed() {
|
|
692
|
+
local dir state fakebin out status_file pid
|
|
693
|
+
dir=$(make_case provably-working-signal); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out"
|
|
694
|
+
status_file="$state/task.status"
|
|
695
|
+
printf 'working: compiling step 2\n' > "$status_file"
|
|
696
|
+
# The crew's pipeline is in an actively-running step: positive evidence it is
|
|
697
|
+
# still working, so a no-verb working: signal is absorbed (the original low-churn
|
|
698
|
+
# case during a long validation).
|
|
699
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
700
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
701
|
+
pid=$!
|
|
702
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
703
|
+
reap "$pid"; fail "watcher exited for a working: signal whose crew is provably working (should absorb): $(cat "$out")"
|
|
704
|
+
fi
|
|
705
|
+
[ ! -s "$out" ] || fail "provably-working signal printed a wake reason: $(cat "$out")"
|
|
706
|
+
[ ! -s "$state/.wake-queue" ] || fail "provably-working signal enqueued a durable wake record"
|
|
707
|
+
[ -s "$state/.seen-task_status" ] || fail "provably-working signal did not advance its .seen-* suppressor"
|
|
708
|
+
[ -e "$state/.last-watcher-beat" ] || fail "watcher beacon was not touched while absorbing"
|
|
709
|
+
reap "$pid"
|
|
710
|
+
pass "a no-verb signal whose crew is provably working is absorbed (no exit, no queue, suppressor advanced, beacon present)"
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
test_turn_ended_provably_working_absorbed() {
|
|
714
|
+
local dir state fakebin out pid
|
|
715
|
+
dir=$(make_case turn-ended-working); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out"
|
|
716
|
+
: > "$state/task.turn-ended"
|
|
717
|
+
# A busy pane is the second form of positive evidence (covers a queued
|
|
718
|
+
# continuation right after the turn-end).
|
|
719
|
+
export FM_FAKE_CREW_STATE='state: working · source: pane · harness busy'
|
|
720
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
721
|
+
pid=$!
|
|
722
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
723
|
+
reap "$pid"; fail "watcher exited for a turn-end whose crew is provably working (should absorb): $(cat "$out")"
|
|
724
|
+
fi
|
|
725
|
+
[ ! -s "$out" ] || fail "provably-working turn-end printed a wake reason: $(cat "$out")"
|
|
726
|
+
[ ! -s "$state/.wake-queue" ] || fail "provably-working turn-end enqueued a durable wake record"
|
|
727
|
+
reap "$pid"
|
|
728
|
+
pass "a bare turn-end whose crew is provably working (busy pane) is absorbed"
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
# --- a no-verb signal whose crew is NOT provably working SURFACES -------------
|
|
732
|
+
# This is the swallowed-finish fix: a crew that finished (or stopped and waits)
|
|
733
|
+
# reports its final turn-end with no captain-relevant status and no running
|
|
734
|
+
# pipeline, so the wake must surface instead of being absorbed.
|
|
735
|
+
|
|
736
|
+
test_turn_ended_not_working_surfaced() {
|
|
737
|
+
local dir state fakebin out drain_out pid
|
|
738
|
+
dir=$(make_case turn-ended-stopped); state="$dir/state"; fakebin="$dir/fakebin"
|
|
739
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
740
|
+
: > "$state/task.turn-ended"
|
|
741
|
+
# No running pipeline, no busy pane: the crew has stopped (e.g. it finished via
|
|
742
|
+
# an interactive menu and wrote no done: status). Default unknown verdict.
|
|
743
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available'
|
|
744
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
745
|
+
pid=$!
|
|
746
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end whose crew is not provably working"
|
|
747
|
+
grep -F "signal: $state/task.turn-ended" "$out" >/dev/null || fail "watcher did not print the surfaced turn-end signal"
|
|
748
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the surfaced turn-end failed"
|
|
749
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/task.turn-ended" >/dev/null || fail "surfaced turn-end was not queued"
|
|
750
|
+
pass "a bare turn-end whose crew is not provably working is surfaced (the swallowed-finish fix)"
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
# --- bare turn-end, unverifiable harness: pane churn is the third proof --------
|
|
754
|
+
# A harness whose semantic busy state has no verified source (codex) can never
|
|
755
|
+
# report working, so the two proofs above are unreachable for it and EVERY worker
|
|
756
|
+
# turn boundary woke firstmate. Pane content that changed since the previous poll
|
|
757
|
+
# is harness-independent positive evidence the crew is still executing - the same
|
|
758
|
+
# liveness input the stale backbone already trusts - so a bare turn-end from a
|
|
759
|
+
# churning pane is benign. The pane going quiet afterwards is still caught by that
|
|
760
|
+
# backbone, which is why this widens the proof rather than bounding the wake rate.
|
|
761
|
+
|
|
762
|
+
# The pane-churn turn-end absorb is opt-in per home, so every case that exercises
|
|
763
|
+
# it (whether it expects an absorb or one of the guards that must still surface)
|
|
764
|
+
# points the watcher at a case-local config dir holding the flag. A case that must
|
|
765
|
+
# NOT have it points at an empty one, so no developer's real config can leak in.
|
|
766
|
+
churn_config() { # <dir> [off]
|
|
767
|
+
local cfg="$1/config"
|
|
768
|
+
mkdir -p "$cfg"
|
|
769
|
+
[ "${2:-}" = off ] || : > "$cfg/turnend-churn-absorb"
|
|
770
|
+
printf '%s\n' "$cfg"
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
# Wait until the watcher records an absorbed wake matching <needle> in its triage
|
|
774
|
+
# log. 1 if the watcher exits first (i.e. it surfaced the wake instead), which is
|
|
775
|
+
# exactly the unfixed behavior this case exists to catch. Polls the log rather
|
|
776
|
+
# than a poll cycle so the assertion lands inside the FIRST poll, long before an
|
|
777
|
+
# unchanging fixture pane could reach the stale backbone.
|
|
778
|
+
wait_for_absorbed() { # <state> <pid> <needle>
|
|
779
|
+
local state=$1 pid=$2 needle=$3 i=0
|
|
780
|
+
while [ "$i" -lt 100 ]; do
|
|
781
|
+
grep -Fq "$needle" "$state/.watch-triage.log" 2>/dev/null && return 0
|
|
782
|
+
kill -0 "$pid" 2>/dev/null || return 1
|
|
783
|
+
sleep 0.1
|
|
784
|
+
i=$((i + 1))
|
|
785
|
+
done
|
|
786
|
+
return 1
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
test_turn_ended_churning_pane_absorbed() {
|
|
790
|
+
local dir state fakebin out capture_file window key pid
|
|
791
|
+
dir=$(make_case turn-ended-churning); state="$dir/state"; fakebin="$dir/fakebin"
|
|
792
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
793
|
+
window="test:fm-codexer"
|
|
794
|
+
: > "$state/codexer.turn-ended"
|
|
795
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexer.meta"
|
|
796
|
+
printf 'apply_patch: writing bin/thing.sh' > "$capture_file"
|
|
797
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
798
|
+
# The previous poll recorded DIFFERENT pane content, so this poll's capture is
|
|
799
|
+
# churn: the crew rendered output between the two polls.
|
|
800
|
+
printf '%s' "$(hash_text 'reading the brief')" > "$state/.hash-$key"
|
|
801
|
+
printf '0\n' > "$state/.count-$key"
|
|
802
|
+
# The codex verdict verbatim: a verified dispatch adapter with no verified
|
|
803
|
+
# semantic busy source, so crew_is_provably_working can never be satisfied.
|
|
804
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
805
|
+
# A slow poll leaves the first cycle's absorb assertion many ticks clear of the
|
|
806
|
+
# stale backbone, which this static fixture pane would otherwise reach.
|
|
807
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
808
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
809
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
810
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
811
|
+
pid=$!
|
|
812
|
+
wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \
|
|
813
|
+
|| { reap "$pid"; fail "a bare turn-end from a churning pane was not absorbed: $(cat "$out")"; }
|
|
814
|
+
[ ! -s "$out" ] || fail "an absorbed churning-pane turn-end printed a wake reason: $(cat "$out")"
|
|
815
|
+
[ ! -s "$state/.wake-queue" ] || fail "an absorbed churning-pane turn-end enqueued a durable wake record"
|
|
816
|
+
[ -s "$state/.churn-since-$key" ] \
|
|
817
|
+
|| { reap "$pid"; fail "an absorbed churning-pane turn-end did not open a bounded deferral window"; }
|
|
818
|
+
reap "$pid"
|
|
819
|
+
unset FM_FAKE_CREW_STATE
|
|
820
|
+
pass "a bare turn-end from a pane that churned since the previous poll is absorbed"
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
test_turn_ended_churn_resets_prior_stale_classification() {
|
|
824
|
+
local dir state fakebin out capture_file window key old_hash active_hash pid i
|
|
825
|
+
dir=$(make_case turn-ended-churn-resets-stale); state="$dir/state"; fakebin="$dir/fakebin"
|
|
826
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
827
|
+
window="test:fm-codexreturned"
|
|
828
|
+
: > "$state/codexreturned.turn-ended"
|
|
829
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexreturned.meta"
|
|
830
|
+
old_hash=$(hash_text 'idle prompt from an earlier turn')
|
|
831
|
+
active_hash=$(hash_text 'rendering a new turn')
|
|
832
|
+
printf 'rendering a new turn' > "$capture_file"
|
|
833
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
834
|
+
printf '%s' "$old_hash" > "$state/.hash-$key"
|
|
835
|
+
printf '1\n' > "$state/.count-$key"
|
|
836
|
+
printf '%s' "$old_hash" > "$state/.stale-$key"
|
|
837
|
+
date +%s > "$state/.stale-since-$key"
|
|
838
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
839
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
840
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
841
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 \
|
|
842
|
+
FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
843
|
+
pid=$!
|
|
844
|
+
wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \
|
|
845
|
+
|| { reap "$pid"; fail "a churning turn-end with prior stale state was not absorbed: $(cat "$out")"; }
|
|
846
|
+
i=0
|
|
847
|
+
while [ "$i" -lt 100 ] && [ "$(cat "$state/.hash-$key" 2>/dev/null || true)" != "$active_hash" ]; do
|
|
848
|
+
kill -0 "$pid" 2>/dev/null || { reap "$pid"; fail "watcher exited before recording the active pane"; }
|
|
849
|
+
sleep 0.1
|
|
850
|
+
i=$((i + 1))
|
|
851
|
+
done
|
|
852
|
+
[ "$(cat "$state/.hash-$key" 2>/dev/null || true)" = "$active_hash" ] \
|
|
853
|
+
|| { reap "$pid"; fail "watcher did not record the active pane after absorbing its turn-end"; }
|
|
854
|
+
|
|
855
|
+
# The worker stops on bytes that happened to be stale in an earlier turn.
|
|
856
|
+
# This is a new quiet interval, so it must surface through ordinary staleness
|
|
857
|
+
# instead of inheriting the earlier interval's wedge timer.
|
|
858
|
+
printf 'idle prompt from an earlier turn' > "$capture_file"
|
|
859
|
+
wait_for_exit "$pid" 100 \
|
|
860
|
+
|| { reap "$pid"; fail "a stopped pane matching an earlier stale render waited for the wedge timeout"; }
|
|
861
|
+
grep -Fx "stale: $window" "$out" >/dev/null \
|
|
862
|
+
|| fail "the returned stale render did not surface through ordinary staleness"
|
|
863
|
+
grep -F "possible wedge" "$out" >/dev/null \
|
|
864
|
+
&& fail "the returned stale render inherited the earlier quiet interval's wedge classification"
|
|
865
|
+
unset FM_FAKE_CREW_STATE
|
|
866
|
+
pass "pane churn starts a fresh stale-classification interval before a stopped render returns"
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
test_turn_ended_churn_resets_wedge_state_before_stale_poll() {
|
|
870
|
+
local dir state fakebin out capture_file capture_count window key pid
|
|
871
|
+
dir=$(make_case turn-ended-churn-resets-wedge); state="$dir/state"; fakebin="$dir/fakebin"
|
|
872
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; capture_count="$dir/capture.count"
|
|
873
|
+
window="test:fm-codexfreshinterval"
|
|
874
|
+
: > "$state/codexfreshinterval.turn-ended"
|
|
875
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexfreshinterval.meta"
|
|
876
|
+
printf 'rendering a new turn' > "$capture_file"
|
|
877
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
878
|
+
printf '%s' "$(hash_text 'idle output from the prior interval')" > "$state/.hash-$key"
|
|
879
|
+
printf '2\n' > "$state/.wedge-escalations-$key"
|
|
880
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
881
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
882
|
+
FM_FAKE_TMUX_CAPTURE_COUNT_FILE="$capture_count" FM_FAKE_TMUX_CAPTURE_FAIL_AFTER=1 \
|
|
883
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
884
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
885
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
886
|
+
pid=$!
|
|
887
|
+
wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \
|
|
888
|
+
|| { reap "$pid"; fail "a churning turn-end was not absorbed before the stale-path capture failed: $(cat "$out")"; }
|
|
889
|
+
[ ! -e "$state/.wedge-escalations-$key" ] \
|
|
890
|
+
|| { reap "$pid"; fail "churn retained the prior quiet interval's wedge-escalation count"; }
|
|
891
|
+
[ ! -s "$state/.wake-queue" ] \
|
|
892
|
+
|| { reap "$pid"; fail "the absorbed churn fixture queued an unexpected wake"; }
|
|
893
|
+
reap "$pid"
|
|
894
|
+
unset FM_FAKE_CREW_STATE
|
|
895
|
+
pass "pane churn resets prior wedge escalation state before the stale-path poll"
|
|
896
|
+
}
|
|
897
|
+
|
|
898
|
+
# The safety half: the same unverifiable harness, the same fixture, but the pane
|
|
899
|
+
# has NOT changed since the previous poll. There is no positive evidence, so the
|
|
900
|
+
# wake must still surface - a stopped worker is exactly what the turn-end marker
|
|
901
|
+
# earns its keep detecting, and widening the proof must not cost that.
|
|
902
|
+
test_turn_ended_still_pane_surfaced() {
|
|
903
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
904
|
+
dir=$(make_case turn-ended-still); state="$dir/state"; fakebin="$dir/fakebin"
|
|
905
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
906
|
+
window="test:fm-codexstopped"
|
|
907
|
+
: > "$state/codexstopped.turn-ended"
|
|
908
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexstopped.meta"
|
|
909
|
+
printf 'apply_patch: writing bin/thing.sh' > "$capture_file"
|
|
910
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
911
|
+
# The previous poll recorded THIS pane content: nothing rendered since.
|
|
912
|
+
printf '%s' "$(hash_text 'apply_patch: writing bin/thing.sh')" > "$state/.hash-$key"
|
|
913
|
+
printf '0\n' > "$state/.count-$key"
|
|
914
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
915
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
916
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
917
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
918
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
919
|
+
pid=$!
|
|
920
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a bare turn-end from an unchanged pane"
|
|
921
|
+
grep -F "signal: $state/codexstopped.turn-ended" "$out" >/dev/null \
|
|
922
|
+
|| fail "watcher did not print the surfaced still-pane turn-end signal"
|
|
923
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the still-pane turn-end failed"
|
|
924
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexstopped.turn-ended" >/dev/null \
|
|
925
|
+
|| fail "surfaced still-pane turn-end was not queued"
|
|
926
|
+
unset FM_FAKE_CREW_STATE
|
|
927
|
+
pass "a bare turn-end from a pane unchanged since the previous poll still surfaces"
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
test_turn_ended_malformed_prior_hash_surfaced() {
|
|
931
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
932
|
+
dir=$(make_case turn-ended-malformed-hash); state="$dir/state"; fakebin="$dir/fakebin"
|
|
933
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
934
|
+
window="test:fm-codexmalformed"
|
|
935
|
+
: > "$state/codexmalformed.turn-ended"
|
|
936
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexmalformed.meta"
|
|
937
|
+
printf 'stopped after rendering this' > "$capture_file"
|
|
938
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
939
|
+
printf 'x' > "$state/.hash-$key"
|
|
940
|
+
printf '0\n' > "$state/.count-$key"
|
|
941
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
942
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
943
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
944
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
945
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
946
|
+
pid=$!
|
|
947
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a turn-end backed by a malformed prior hash"
|
|
948
|
+
grep -F "signal: $state/codexmalformed.turn-ended" "$out" >/dev/null \
|
|
949
|
+
|| fail "watcher did not print the surfaced malformed-hash turn-end"
|
|
950
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
951
|
+
|| fail "drain after the malformed-hash turn-end failed"
|
|
952
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexmalformed.turn-ended" >/dev/null \
|
|
953
|
+
|| fail "malformed-hash turn-end was not queued"
|
|
954
|
+
unset FM_FAKE_CREW_STATE
|
|
955
|
+
pass "a bare turn-end backed by a malformed prior hash surfaces"
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
test_turn_ended_trailing_newline_prior_hash_surfaced() {
|
|
959
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
960
|
+
dir=$(make_case turn-ended-newline-hash); state="$dir/state"; fakebin="$dir/fakebin"
|
|
961
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
962
|
+
window="test:fm-codexnewline"
|
|
963
|
+
: > "$state/codexnewline.turn-ended"
|
|
964
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexnewline.meta"
|
|
965
|
+
printf 'rendered after the prior poll' > "$capture_file"
|
|
966
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
967
|
+
printf '%s\n' "$(hash_text 'the previous render')" > "$state/.hash-$key"
|
|
968
|
+
printf '0\n' > "$state/.count-$key"
|
|
969
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
970
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
971
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
972
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
973
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
974
|
+
pid=$!
|
|
975
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a turn-end backed by a newline-terminated prior hash"
|
|
976
|
+
grep -F "signal: $state/codexnewline.turn-ended" "$out" >/dev/null \
|
|
977
|
+
|| fail "watcher did not print the surfaced newline-hash turn-end"
|
|
978
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
979
|
+
|| fail "drain after the newline-hash turn-end failed"
|
|
980
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexnewline.turn-ended" >/dev/null \
|
|
981
|
+
|| fail "newline-hash turn-end was not queued"
|
|
982
|
+
[ ! -e "$state/.churn-since-$key" ] \
|
|
983
|
+
|| fail "a newline-terminated prior hash opened a deferral window"
|
|
984
|
+
unset FM_FAKE_CREW_STATE
|
|
985
|
+
pass "a bare turn-end backed by a newline-terminated prior hash surfaces"
|
|
986
|
+
}
|
|
987
|
+
|
|
988
|
+
test_secondmate_turn_ended_churning_pane_surfaced() {
|
|
989
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
990
|
+
dir=$(make_case secondmate-turn-ended-churning); state="$dir/state"; fakebin="$dir/fakebin"
|
|
991
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
992
|
+
window="test:fm-mate-churning"
|
|
993
|
+
: > "$state/mate.turn-ended"
|
|
994
|
+
printf 'window=%s\nkind=secondmate\nharness=pi\n' "$window" > "$state/mate.meta"
|
|
995
|
+
printf 'working on the next routed item' > "$capture_file"
|
|
996
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
997
|
+
printf '%s' "$(hash_text 'waiting for work')" > "$state/.hash-$key"
|
|
998
|
+
printf '0\n' > "$state/.count-$key"
|
|
999
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable'
|
|
1000
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1001
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1002
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1003
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1004
|
+
pid=$!
|
|
1005
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a churning secondmate turn-end"
|
|
1006
|
+
grep -F "signal: $state/mate.turn-ended" "$out" >/dev/null \
|
|
1007
|
+
|| fail "watcher did not print the surfaced churning secondmate turn-end"
|
|
1008
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1009
|
+
|| fail "drain after the churning secondmate turn-end failed"
|
|
1010
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/mate.turn-ended" >/dev/null \
|
|
1011
|
+
|| fail "churning secondmate turn-end was not queued"
|
|
1012
|
+
unset FM_FAKE_CREW_STATE
|
|
1013
|
+
pass "a churning secondmate turn-end surfaces without a stale resurface path"
|
|
1014
|
+
}
|
|
1015
|
+
|
|
1016
|
+
test_turn_ended_colliding_window_key_surfaced() {
|
|
1017
|
+
local dir state fakebin out drain_out capture_file window colliding key pid
|
|
1018
|
+
dir=$(make_case turn-ended-colliding-key); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1019
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1020
|
+
window="test:fm-a.b"; colliding="test:fm-a_b"
|
|
1021
|
+
: > "$state/a.b.turn-ended"
|
|
1022
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/a.b.meta"
|
|
1023
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$colliding" > "$state/a_b.meta"
|
|
1024
|
+
printf 'rendered after the prior poll' > "$capture_file"
|
|
1025
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1026
|
+
printf '%s' "$(hash_text 'the other window pane')" > "$state/.hash-$key"
|
|
1027
|
+
printf '0\n' > "$state/.count-$key"
|
|
1028
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1029
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1030
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1031
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1032
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1033
|
+
pid=$!
|
|
1034
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with an ambiguous pane marker"
|
|
1035
|
+
grep -F "signal: $state/a.b.turn-ended" "$out" >/dev/null \
|
|
1036
|
+
|| fail "watcher did not print the surfaced ambiguous-marker turn-end"
|
|
1037
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1038
|
+
|| fail "drain after the ambiguous-marker turn-end failed"
|
|
1039
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/a.b.turn-ended" >/dev/null \
|
|
1040
|
+
|| fail "ambiguous-marker turn-end was not queued"
|
|
1041
|
+
unset FM_FAKE_CREW_STATE
|
|
1042
|
+
pass "a turn-end whose marker key matches another recorded endpoint surfaces"
|
|
1043
|
+
}
|
|
1044
|
+
|
|
1045
|
+
test_turn_ended_duplicate_endpoint_records_surfaced() {
|
|
1046
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
1047
|
+
dir=$(make_case turn-ended-duplicate-endpoint); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1048
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1049
|
+
window="test:fm-shared"
|
|
1050
|
+
: > "$state/first.turn-ended"
|
|
1051
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/first.meta"
|
|
1052
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/second.meta"
|
|
1053
|
+
printf 'rendered after the prior poll' > "$capture_file"
|
|
1054
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1055
|
+
printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key"
|
|
1056
|
+
printf '0\n' > "$state/.count-$key"
|
|
1057
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1058
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1059
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1060
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1061
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1062
|
+
pid=$!
|
|
1063
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a turn-end shared by two endpoint records"
|
|
1064
|
+
grep -F "signal: $state/first.turn-ended" "$out" >/dev/null \
|
|
1065
|
+
|| fail "watcher did not print the surfaced duplicate-endpoint turn-end"
|
|
1066
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1067
|
+
|| fail "drain after the duplicate-endpoint turn-end failed"
|
|
1068
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/first.turn-ended" >/dev/null \
|
|
1069
|
+
|| fail "duplicate-endpoint turn-end was not queued"
|
|
1070
|
+
[ ! -e "$state/.churn-since-$key" ] \
|
|
1071
|
+
|| fail "duplicate endpoint records opened a deferral window"
|
|
1072
|
+
unset FM_FAKE_CREW_STATE
|
|
1073
|
+
pass "two metadata records sharing one endpoint make churn evidence ambiguous"
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
test_turn_ended_mixed_positive_evidence_batch_absorbed() {
|
|
1077
|
+
local dir state fakebin out capture_file first_window second_window first_key second_key pid
|
|
1078
|
+
dir=$(make_case turn-ended-mixed-evidence); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1079
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
1080
|
+
first_window="test:fm-first"; second_window="test:fm-second"
|
|
1081
|
+
: > "$state/first.turn-ended"
|
|
1082
|
+
: > "$state/second.turn-ended"
|
|
1083
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$first_window" > "$state/first.meta"
|
|
1084
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/second.meta"
|
|
1085
|
+
printf 'second task rendered after the prior poll' > "$capture_file"
|
|
1086
|
+
first_key=$(printf '%s' "$first_window" | tr ':/.' '___')
|
|
1087
|
+
second_key=$(printf '%s' "$second_window" | tr ':/.' '___')
|
|
1088
|
+
printf '%s' "$(hash_text 'first task static pane')" > "$state/.hash-$first_key"
|
|
1089
|
+
printf '%s' "$(hash_text 'second task previous render')" > "$state/.hash-$second_key"
|
|
1090
|
+
printf '0\n' > "$state/.count-$first_key"
|
|
1091
|
+
printf '0\n' > "$state/.count-$second_key"
|
|
1092
|
+
export FM_FAKE_CREW_STATE_first='state: working · source: run-step · running'
|
|
1093
|
+
export FM_FAKE_CREW_STATE_second='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1094
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-first\nfm-second')" \
|
|
1095
|
+
FM_FAKE_TMUX_CAPTURE="$capture_file" FM_FAKE_TMUX_FORBIDDEN_TARGET="$first_window" \
|
|
1096
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1097
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1098
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1099
|
+
pid=$!
|
|
1100
|
+
wait_for_absorbed "$state" "$pid" "absorbed benign signal:" \
|
|
1101
|
+
|| { reap "$pid"; fail "a mixed authoritative-and-churn batch was not absorbed: $(cat "$out")"; }
|
|
1102
|
+
[ ! -s "$out" ] || fail "an absorbed mixed-evidence batch printed a wake reason: $(cat "$out")"
|
|
1103
|
+
[ ! -s "$state/.wake-queue" ] || fail "an absorbed mixed-evidence batch enqueued a durable wake record"
|
|
1104
|
+
[ ! -e "$state/.churn-since-$first_key" ] \
|
|
1105
|
+
|| fail "an authoritatively working task opened a pane-churn deadline"
|
|
1106
|
+
[ -s "$state/.churn-since-$second_key" ] \
|
|
1107
|
+
|| fail "the churn-proven task did not open its bounded deferral window"
|
|
1108
|
+
reap "$pid"
|
|
1109
|
+
unset FM_FAKE_CREW_STATE_first FM_FAKE_CREW_STATE_second
|
|
1110
|
+
pass "a batch may satisfy positive evidence independently per task"
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
test_turn_ended_mixed_positive_evidence_batch_default_off() {
|
|
1114
|
+
local dir state fakebin out drain_out capture_file first_window second_window first_key second_key pid
|
|
1115
|
+
dir=$(make_case turn-ended-mixed-evidence-off); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1116
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1117
|
+
first_window="test:fm-firstoff"; second_window="test:fm-secondoff"
|
|
1118
|
+
: > "$state/firstoff.turn-ended"
|
|
1119
|
+
: > "$state/secondoff.turn-ended"
|
|
1120
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$first_window" > "$state/firstoff.meta"
|
|
1121
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/secondoff.meta"
|
|
1122
|
+
printf 'second task rendered after the prior poll' > "$capture_file"
|
|
1123
|
+
first_key=$(printf '%s' "$first_window" | tr ':/.' '___')
|
|
1124
|
+
second_key=$(printf '%s' "$second_window" | tr ':/.' '___')
|
|
1125
|
+
printf '%s' "$(hash_text 'first task static pane')" > "$state/.hash-$first_key"
|
|
1126
|
+
printf '%s' "$(hash_text 'second task previous render')" > "$state/.hash-$second_key"
|
|
1127
|
+
printf '0\n' > "$state/.count-$first_key"
|
|
1128
|
+
printf '0\n' > "$state/.count-$second_key"
|
|
1129
|
+
export FM_FAKE_CREW_STATE_firstoff='state: working · source: run-step · running'
|
|
1130
|
+
export FM_FAKE_CREW_STATE_secondoff='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1131
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-firstoff\nfm-secondoff')" \
|
|
1132
|
+
FM_FAKE_TMUX_CAPTURE="$capture_file" FM_CONFIG_OVERRIDE="$(churn_config "$dir" off)" \
|
|
1133
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1134
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1135
|
+
pid=$!
|
|
1136
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a mixed-evidence batch without the opt-in flag"
|
|
1137
|
+
grep -F "$state/firstoff.turn-ended" "$out" >/dev/null \
|
|
1138
|
+
|| fail "watcher did not print the first default-off turn-end"
|
|
1139
|
+
grep -F "$state/secondoff.turn-ended" "$out" >/dev/null \
|
|
1140
|
+
|| fail "watcher did not print the second default-off turn-end"
|
|
1141
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1142
|
+
|| fail "drain after the default-off mixed-evidence batch failed"
|
|
1143
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/firstoff.turn-ended" >/dev/null \
|
|
1144
|
+
|| fail "the first default-off turn-end was not queued"
|
|
1145
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/secondoff.turn-ended" >/dev/null \
|
|
1146
|
+
|| fail "the second default-off turn-end was not queued"
|
|
1147
|
+
[ ! -e "$state/.churn-since-$first_key" ] && [ ! -e "$state/.churn-since-$second_key" ] \
|
|
1148
|
+
|| fail "the default-off mixed-evidence batch opened a deferral window"
|
|
1149
|
+
unset FM_FAKE_CREW_STATE_firstoff FM_FAKE_CREW_STATE_secondoff
|
|
1150
|
+
pass "per-task evidence composition stays off until the home opts in"
|
|
1151
|
+
}
|
|
1152
|
+
|
|
1153
|
+
test_status_and_turn_end_batch_never_uses_churn_evidence() {
|
|
1154
|
+
local dir state fakebin out drain_out capture_file first_window second_window second_key pid
|
|
1155
|
+
dir=$(make_case status-and-turn-ended-churn); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1156
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1157
|
+
first_window="test:fm-firststatus"; second_window="test:fm-secondturn"
|
|
1158
|
+
printf 'working: authoritative task still running\n' > "$state/firststatus.status"
|
|
1159
|
+
: > "$state/secondturn.turn-ended"
|
|
1160
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$first_window" > "$state/firststatus.meta"
|
|
1161
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/secondturn.meta"
|
|
1162
|
+
printf 'second task rendered after the prior poll' > "$capture_file"
|
|
1163
|
+
second_key=$(printf '%s' "$second_window" | tr ':/.' '___')
|
|
1164
|
+
printf '%s' "$(hash_text 'second task previous render')" > "$state/.hash-$second_key"
|
|
1165
|
+
printf '0\n' > "$state/.count-$second_key"
|
|
1166
|
+
export FM_FAKE_CREW_STATE_firststatus='state: working · source: run-step · running'
|
|
1167
|
+
export FM_FAKE_CREW_STATE_secondturn='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1168
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-firststatus\nfm-secondturn')" \
|
|
1169
|
+
FM_FAKE_TMUX_CAPTURE="$capture_file" FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1170
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1171
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1172
|
+
pid=$!
|
|
1173
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a status-and-turn-end batch on churn evidence"
|
|
1174
|
+
grep -F "$state/firststatus.status" "$out" >/dev/null \
|
|
1175
|
+
|| fail "watcher did not print the status file from the surfaced mixed batch"
|
|
1176
|
+
grep -F "$state/secondturn.turn-ended" "$out" >/dev/null \
|
|
1177
|
+
|| fail "watcher did not print the turn-end from the surfaced mixed batch"
|
|
1178
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1179
|
+
|| fail "drain after the surfaced status-and-turn-end batch failed"
|
|
1180
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/firststatus.status" >/dev/null \
|
|
1181
|
+
|| fail "the status file from the surfaced mixed batch was not queued"
|
|
1182
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/secondturn.turn-ended" >/dev/null \
|
|
1183
|
+
|| fail "the turn-end from the surfaced mixed batch was not queued"
|
|
1184
|
+
[ ! -e "$state/.churn-since-$second_key" ] \
|
|
1185
|
+
|| fail "a status-bearing batch opened a pane-churn deadline"
|
|
1186
|
+
unset FM_FAKE_CREW_STATE_firststatus FM_FAKE_CREW_STATE_secondturn
|
|
1187
|
+
pass "a status-bearing batch never falls through to pane-churn evidence"
|
|
1188
|
+
}
|
|
1189
|
+
|
|
1190
|
+
# The opt-in half. Pane churn infers execution from rendered bytes rather than
|
|
1191
|
+
# from a verdict the harness vouches for, so a home that has not asked for it must
|
|
1192
|
+
# see exactly the pre-change triage: the same churning fixture that absorbs above
|
|
1193
|
+
# surfaces here purely because the flag is absent.
|
|
1194
|
+
test_turn_ended_churn_absorb_off_by_default() {
|
|
1195
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
1196
|
+
dir=$(make_case turn-ended-churn-default-off); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1197
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1198
|
+
window="test:fm-codexdefault"
|
|
1199
|
+
: > "$state/codexdefault.turn-ended"
|
|
1200
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexdefault.meta"
|
|
1201
|
+
printf 'apply_patch: writing bin/thing.sh' > "$capture_file"
|
|
1202
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1203
|
+
printf '%s' "$(hash_text 'reading the brief')" > "$state/.hash-$key"
|
|
1204
|
+
printf '0\n' > "$state/.count-$key"
|
|
1205
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1206
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1207
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir" off)" \
|
|
1208
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1209
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1210
|
+
pid=$!
|
|
1211
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a churning turn-end without the opt-in flag"
|
|
1212
|
+
grep -F "signal: $state/codexdefault.turn-ended" "$out" >/dev/null \
|
|
1213
|
+
|| fail "watcher did not print the surfaced default-off churning turn-end"
|
|
1214
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1215
|
+
|| fail "drain after the default-off churning turn-end failed"
|
|
1216
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexdefault.turn-ended" >/dev/null \
|
|
1217
|
+
|| fail "default-off churning turn-end was not queued"
|
|
1218
|
+
[ ! -e "$state/.churn-since-$key" ] \
|
|
1219
|
+
|| fail "the default-off path opened a bounded deferral window"
|
|
1220
|
+
unset FM_FAKE_CREW_STATE
|
|
1221
|
+
pass "pane-churn turn-end absorb is off until a home opts in"
|
|
1222
|
+
}
|
|
1223
|
+
|
|
1224
|
+
# The bound. Churn and pane staleness read the same pane, so a pane that renders
|
|
1225
|
+
# continuously (a clock, a spinner, a harness that leaves a background renderer
|
|
1226
|
+
# alive after its agent yields) never reaches the staleness backbone's two
|
|
1227
|
+
# identical hashes either. Without a bound on the churn absorb a worker that had
|
|
1228
|
+
# genuinely stopped behind such a renderer would have no path left to surface at
|
|
1229
|
+
# all, so an exhausted deferral window must surface and restart.
|
|
1230
|
+
test_turn_ended_churn_absorb_bounded() {
|
|
1231
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
1232
|
+
dir=$(make_case turn-ended-churn-bounded); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1233
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1234
|
+
window="test:fm-codexclock"
|
|
1235
|
+
: > "$state/codexclock.turn-ended"
|
|
1236
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexclock.meta"
|
|
1237
|
+
printf 'a background renderer that never stops' > "$capture_file"
|
|
1238
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1239
|
+
printf '%s' "$(hash_text 'the previous frame')" > "$state/.hash-$key"
|
|
1240
|
+
printf '0\n' > "$state/.count-$key"
|
|
1241
|
+
# This endpoint has already been riding churn evidence longer than the bound.
|
|
1242
|
+
printf '%s' "$(( $(date +%s) - 600 ))" > "$state/.churn-since-$key"
|
|
1243
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1244
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1245
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" FM_TURNEND_CHURN_ABSORB_SECS=60 \
|
|
1246
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1247
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1248
|
+
pid=$!
|
|
1249
|
+
wait_for_exit "$pid" 100 \
|
|
1250
|
+
|| fail "a perpetually churning pane deferred its turn-end past the absorb bound"
|
|
1251
|
+
grep -F "signal: $state/codexclock.turn-ended" "$out" >/dev/null \
|
|
1252
|
+
|| fail "watcher did not print the turn-end surfaced by the exhausted absorb bound"
|
|
1253
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1254
|
+
|| fail "drain after the bounded churn turn-end failed"
|
|
1255
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexclock.turn-ended" >/dev/null \
|
|
1256
|
+
|| fail "the turn-end surfaced by the exhausted absorb bound was not queued"
|
|
1257
|
+
[ ! -e "$state/.churn-since-$key" ] \
|
|
1258
|
+
|| fail "an exhausted deferral window was not restarted after surfacing"
|
|
1259
|
+
unset FM_FAKE_CREW_STATE
|
|
1260
|
+
pass "a perpetually churning pane surfaces once its bounded deferral window is spent"
|
|
1261
|
+
}
|
|
1262
|
+
|
|
1263
|
+
test_turn_ended_churn_timer_write_failure_surfaced() {
|
|
1264
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
1265
|
+
dir=$(make_case turn-ended-churn-timer-write-failure); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1266
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1267
|
+
window="test:fm-codextimer"
|
|
1268
|
+
: > "$state/codextimer.turn-ended"
|
|
1269
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codextimer.meta"
|
|
1270
|
+
printf 'rendered after the previous poll' > "$capture_file"
|
|
1271
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1272
|
+
printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key"
|
|
1273
|
+
printf '0\n' > "$state/.count-$key"
|
|
1274
|
+
mkdir "$state/.churn-since-$key"
|
|
1275
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1276
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1277
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1278
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1279
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null &
|
|
1280
|
+
pid=$!
|
|
1281
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a churning turn-end without recording its deadline"
|
|
1282
|
+
grep -F "signal: $state/codextimer.turn-ended" "$out" >/dev/null \
|
|
1283
|
+
|| fail "watcher did not print the turn-end whose churn deadline could not be recorded"
|
|
1284
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1285
|
+
|| fail "drain after the failed churn deadline write failed"
|
|
1286
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codextimer.turn-ended" >/dev/null \
|
|
1287
|
+
|| fail "turn-end with an unrecordable churn deadline was not queued"
|
|
1288
|
+
unset FM_FAKE_CREW_STATE
|
|
1289
|
+
pass "an unrecordable pane-churn deadline surfaces the turn-end"
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
test_turn_ended_invalid_churn_bound_surfaced() {
|
|
1293
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
1294
|
+
dir=$(make_case turn-ended-invalid-churn-bound); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1295
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1296
|
+
window="test:fm-codexbound"
|
|
1297
|
+
: > "$state/codexbound.turn-ended"
|
|
1298
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexbound.meta"
|
|
1299
|
+
printf 'rendered after the previous poll' > "$capture_file"
|
|
1300
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1301
|
+
printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key"
|
|
1302
|
+
printf '0\n' > "$state/.count-$key"
|
|
1303
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1304
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1305
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" FM_TURNEND_CHURN_ABSORB_SECS=bogus \
|
|
1306
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1307
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null &
|
|
1308
|
+
pid=$!
|
|
1309
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with an invalid churn bound"
|
|
1310
|
+
grep -F "signal: $state/codexbound.turn-ended" "$out" >/dev/null \
|
|
1311
|
+
|| fail "watcher terminated before printing the invalid-bound turn-end"
|
|
1312
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1313
|
+
|| fail "drain after the invalid churn bound failed"
|
|
1314
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexbound.turn-ended" >/dev/null \
|
|
1315
|
+
|| fail "turn-end with an invalid churn bound was not queued"
|
|
1316
|
+
[ ! -e "$state/.churn-since-$key" ] \
|
|
1317
|
+
|| fail "an invalid churn bound opened a deferral window"
|
|
1318
|
+
unset FM_FAKE_CREW_STATE
|
|
1319
|
+
pass "an invalid pane-churn bound surfaces the turn-end"
|
|
1320
|
+
}
|
|
1321
|
+
|
|
1322
|
+
test_turn_ended_oversized_churn_bound_surfaced() {
|
|
1323
|
+
local dir state fakebin out drain_out capture_file window key pid
|
|
1324
|
+
dir=$(make_case turn-ended-oversized-churn-bound); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1325
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1326
|
+
window="test:fm-codexoversized"
|
|
1327
|
+
: > "$state/codexoversized.turn-ended"
|
|
1328
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexoversized.meta"
|
|
1329
|
+
printf 'rendered after the previous poll' > "$capture_file"
|
|
1330
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1331
|
+
printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key"
|
|
1332
|
+
printf '0\n' > "$state/.count-$key"
|
|
1333
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1334
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1335
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" FM_TURNEND_CHURN_ABSORB_SECS=999999999999999999999999999999999999 \
|
|
1336
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1337
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null &
|
|
1338
|
+
pid=$!
|
|
1339
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with an oversized churn bound"
|
|
1340
|
+
grep -F "signal: $state/codexoversized.turn-ended" "$out" >/dev/null \
|
|
1341
|
+
|| fail "watcher terminated before printing the oversized-bound turn-end"
|
|
1342
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1343
|
+
|| fail "drain after the oversized churn bound failed"
|
|
1344
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexoversized.turn-ended" >/dev/null \
|
|
1345
|
+
|| fail "turn-end with an oversized churn bound was not queued"
|
|
1346
|
+
[ ! -e "$state/.churn-since-$key" ] \
|
|
1347
|
+
|| fail "an oversized churn bound opened a deferral window"
|
|
1348
|
+
unset FM_FAKE_CREW_STATE
|
|
1349
|
+
pass "an oversized pane-churn bound surfaces the turn-end"
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1352
|
+
test_turn_ended_invalid_churn_deadline_surfaced() {
|
|
1353
|
+
local variant value dir state fakebin out drain_out capture_file window key marker pid
|
|
1354
|
+
for variant in empty leading-zero nonnumeric future overflow; do
|
|
1355
|
+
dir=$(make_case "turn-ended-invalid-churn-deadline-$variant")
|
|
1356
|
+
state="$dir/state"; fakebin="$dir/fakebin"
|
|
1357
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1358
|
+
window="test:fm-codexdeadline"
|
|
1359
|
+
: > "$state/codexdeadline.turn-ended"
|
|
1360
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$window" > "$state/codexdeadline.meta"
|
|
1361
|
+
printf 'rendered after the previous poll' > "$capture_file"
|
|
1362
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1363
|
+
marker="$state/.churn-since-$key"
|
|
1364
|
+
printf '%s' "$(hash_text 'the previous render')" > "$state/.hash-$key"
|
|
1365
|
+
printf '0\n' > "$state/.count-$key"
|
|
1366
|
+
case "$variant" in
|
|
1367
|
+
empty) value='' ;;
|
|
1368
|
+
leading-zero) value=09 ;;
|
|
1369
|
+
nonnumeric) value=bogus ;;
|
|
1370
|
+
future) value=$(( $(date +%s) + 600 )) ;;
|
|
1371
|
+
overflow) value=999999999999999999999999999999999999 ;;
|
|
1372
|
+
esac
|
|
1373
|
+
printf '%s' "$value" > "$marker"
|
|
1374
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1375
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1376
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1377
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1378
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null &
|
|
1379
|
+
pid=$!
|
|
1380
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a turn-end with a $variant churn deadline"
|
|
1381
|
+
grep -F "signal: $state/codexdeadline.turn-ended" "$out" >/dev/null \
|
|
1382
|
+
|| fail "watcher terminated before printing the $variant-deadline turn-end"
|
|
1383
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1384
|
+
|| fail "drain after the $variant churn deadline failed"
|
|
1385
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/codexdeadline.turn-ended" >/dev/null \
|
|
1386
|
+
|| fail "turn-end with a $variant churn deadline was not queued"
|
|
1387
|
+
[ "$(cat "$marker")" = "$value" ] \
|
|
1388
|
+
|| fail "the $variant churn deadline was rewritten"
|
|
1389
|
+
done
|
|
1390
|
+
unset FM_FAKE_CREW_STATE
|
|
1391
|
+
pass "invalid existing pane-churn deadlines surface without mutation"
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
test_turn_ended_surfaced_batch_opens_no_partial_deadline() {
|
|
1395
|
+
local dir state fakebin out drain_out capture_file first_window second_window first_key second_key pid
|
|
1396
|
+
dir=$(make_case turn-ended-no-partial-churn-deadline); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1397
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1398
|
+
first_window="test:fm-codexfirst"; second_window="test:fm-codexsecond"
|
|
1399
|
+
: > "$state/first.turn-ended"
|
|
1400
|
+
: > "$state/second.turn-ended"
|
|
1401
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$first_window" > "$state/first.meta"
|
|
1402
|
+
printf 'window=%s\nkind=ship\nharness=codex\n' "$second_window" > "$state/second.meta"
|
|
1403
|
+
printf 'rendered after the previous poll' > "$capture_file"
|
|
1404
|
+
first_key=$(printf '%s' "$first_window" | tr ':/.' '___')
|
|
1405
|
+
second_key=$(printf '%s' "$second_window" | tr ':/.' '___')
|
|
1406
|
+
printf '%s' "$(hash_text 'first previous render')" > "$state/.hash-$first_key"
|
|
1407
|
+
printf '%s' "$(hash_text 'second previous render')" > "$state/.hash-$second_key"
|
|
1408
|
+
printf '0\n' > "$state/.count-$first_key"
|
|
1409
|
+
printf '0\n' > "$state/.count-$second_key"
|
|
1410
|
+
printf 'bogus' > "$state/.churn-since-$second_key"
|
|
1411
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: pane · harness state unavailable (unknown codex-unverified)'
|
|
1412
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOWS="$(printf 'fm-codexfirst\nfm-codexsecond')" \
|
|
1413
|
+
FM_FAKE_TMUX_CAPTURE="$capture_file" FM_CONFIG_OVERRIDE="$(churn_config "$dir")" \
|
|
1414
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=3 FM_SIGNAL_GRACE=1 \
|
|
1415
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" 2>/dev/null &
|
|
1416
|
+
pid=$!
|
|
1417
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a batch containing an invalid churn deadline"
|
|
1418
|
+
grep -F "$state/first.turn-ended" "$out" >/dev/null \
|
|
1419
|
+
|| fail "watcher did not print the first turn-end from the surfaced batch"
|
|
1420
|
+
grep -F "$state/second.turn-ended" "$out" >/dev/null \
|
|
1421
|
+
|| fail "watcher did not print the second turn-end from the surfaced batch"
|
|
1422
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null \
|
|
1423
|
+
|| fail "drain after the surfaced churn batch failed"
|
|
1424
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/first.turn-ended" >/dev/null \
|
|
1425
|
+
|| fail "the first turn-end from the surfaced batch was not queued"
|
|
1426
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/second.turn-ended" >/dev/null \
|
|
1427
|
+
|| fail "the second turn-end from the surfaced batch was not queued"
|
|
1428
|
+
[ ! -e "$state/.churn-since-$first_key" ] \
|
|
1429
|
+
|| fail "a surfaced batch opened a partial churn deadline"
|
|
1430
|
+
[ "$(cat "$state/.churn-since-$second_key")" = bogus ] \
|
|
1431
|
+
|| fail "the invalid churn deadline in a surfaced batch was rewritten"
|
|
1432
|
+
unset FM_FAKE_CREW_STATE
|
|
1433
|
+
pass "a surfaced batch opens no partial pane-churn deadline"
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
test_working_note_not_working_surfaced() {
|
|
1437
|
+
local dir state fakebin out drain_out status_file pid
|
|
1438
|
+
dir=$(make_case working-note-stopped); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1439
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
1440
|
+
status_file="$state/task.status"
|
|
1441
|
+
printf 'working: compiling step 2\n' > "$status_file"
|
|
1442
|
+
# A non-no-mistakes crew (no run) whose pane went idle: fm-crew-state falls back
|
|
1443
|
+
# to the stale working: status-log line. That is NOT positive evidence, so the
|
|
1444
|
+
# wake must surface - these users must never be left hanging.
|
|
1445
|
+
export FM_FAKE_CREW_STATE='state: working · source: status-log · working: compiling step 2'
|
|
1446
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1447
|
+
pid=$!
|
|
1448
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a working: note whose crew has no running pipeline and an idle pane"
|
|
1449
|
+
grep -F "signal: $status_file" "$out" >/dev/null || fail "watcher did not print the surfaced working: signal"
|
|
1450
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the surfaced working: note failed"
|
|
1451
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$status_file" >/dev/null || fail "surfaced working: note was not queued"
|
|
1452
|
+
[ -s "$state/.seen-task_status" ] || fail "surfaced working: note did not advance its .seen-* suppressor"
|
|
1453
|
+
pass "a no-verb working: note whose crew is idle with no running pipeline is surfaced"
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
test_secondmate_status_note_surfaced_despite_busy_agent() {
|
|
1457
|
+
local dir state fakebin out drain_out pid
|
|
1458
|
+
dir=$(make_case secondmate-note-surfaced); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1459
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
1460
|
+
printf 'kind=secondmate\n' > "$state/mate.meta"
|
|
1461
|
+
printf 'working: routed reply landed in the parent stream\n' > "$state/mate.status"
|
|
1462
|
+
# Busy evidence that would absorb an ordinary crewmate's no-verb note must
|
|
1463
|
+
# not absorb a secondmate's: its status stream is the routed-reply channel.
|
|
1464
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · running'
|
|
1465
|
+
FM_CONFIG_OVERRIDE="$(churn_config "$dir")" watch_bg "$state" "$fakebin" "$out"
|
|
1466
|
+
pid=$!
|
|
1467
|
+
wait_for_exit "$pid" 100 || fail "watcher absorbed a busy secondmate's routed status note"
|
|
1468
|
+
grep -F "signal: $state/mate.status" "$out" >/dev/null \
|
|
1469
|
+
|| fail "watcher did not print the surfaced secondmate note"
|
|
1470
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the surfaced note failed"
|
|
1471
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$state/mate.status" >/dev/null \
|
|
1472
|
+
|| fail "surfaced secondmate note was not queued"
|
|
1473
|
+
pass "a secondmate's status note surfaces even while its own agent is busy"
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
test_self_announced_close_does_not_rewake_but_next_note_does() {
|
|
1477
|
+
local dir state fakebin out status_file pid rc
|
|
1478
|
+
dir=$(make_case self-close-quiet); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out"
|
|
1479
|
+
status_file="$state/task.status"
|
|
1480
|
+
printf 'needs-decision [key=k1]: pick one\n' > "$status_file"
|
|
1481
|
+
prime_status_seen "$state" "$status_file" || fail "could not prime the announced baseline"
|
|
1482
|
+
# The home's own bookkeeping close, written through the guarded
|
|
1483
|
+
# self-announced append this home's answerers use.
|
|
1484
|
+
rc=0
|
|
1485
|
+
FM_STATE_OVERRIDE="$state" bash -c '
|
|
1486
|
+
. "$1"
|
|
1487
|
+
fm_wake_status_append_self_announced "$2" "$3" "resolved [key=k1]: answered: closed by this home"
|
|
1488
|
+
' _ "$ROOT/bin/fm-wake-lib.sh" "$state" "$status_file" || rc=$?
|
|
1489
|
+
[ "$rc" -eq 0 ] || fail "the bookkeeping close was not self-announced (rc=$rc)"
|
|
1490
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: none · idle worker'
|
|
1491
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1492
|
+
pid=$!
|
|
1493
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
1494
|
+
reap "$pid"; fail "the home's own bookkeeping close re-woke its own watcher: $(cat "$out")"
|
|
1495
|
+
fi
|
|
1496
|
+
[ ! -s "$out" ] || { reap "$pid"; fail "self-announced close printed a wake reason: $(cat "$out")"; }
|
|
1497
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "self-announced close enqueued a durable wake"; }
|
|
1498
|
+
# A later, different note on the SAME task still wakes: dedup is keyed on the
|
|
1499
|
+
# exact announced bytes, never on task identity.
|
|
1500
|
+
printf 'needs-decision [key=k2]: a genuinely new decision\n' >> "$status_file"
|
|
1501
|
+
wait_for_exit "$pid" 100 || fail "a later different note after a self-announced close was swallowed"
|
|
1502
|
+
grep -F "signal: $status_file" "$out" >/dev/null \
|
|
1503
|
+
|| fail "the later note did not surface as a signal"
|
|
1504
|
+
pass "a self-announced close never wakes its own home, and the next real note still does"
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
# --- actionable wakes are surfaced (queue + exit) ---------------------------
|
|
1508
|
+
|
|
1509
|
+
test_actionable_signal_surfaced() {
|
|
1510
|
+
local dir state fakebin out drain_out status_file pid
|
|
1511
|
+
dir=$(make_case actionable-signal); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1512
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
1513
|
+
status_file="$state/task.status"
|
|
1514
|
+
printf 'working: setup\nneeds-decision: pick A or B\n' > "$status_file"
|
|
1515
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1516
|
+
pid=$!
|
|
1517
|
+
wait_for_exit "$pid" 100 || fail "watcher did not exit for an actionable needs-decision signal"
|
|
1518
|
+
grep -F "signal: $status_file" "$out" >/dev/null || fail "watcher did not print the actionable signal reason"
|
|
1519
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the actionable signal failed"
|
|
1520
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$status_file" >/dev/null || fail "actionable signal was not queued"
|
|
1521
|
+
[ -s "$state/.hb-surfaced-task" ] || fail "actionable signal did not record the surfaced marker"
|
|
1522
|
+
pass "captain-relevant signal is surfaced (queue + exit) and marked surfaced"
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
# The reported bug, end to end through a real watcher: a crew reports something
|
|
1526
|
+
# the captain must act on and then keeps appending routine progress, which is
|
|
1527
|
+
# ordinary while the watcher lingers its signal grace window to coalesce a status
|
|
1528
|
+
# write with the same turn's turn-end. Classifying only the last line reads the
|
|
1529
|
+
# batch as routine, and because the crew IS provably working the no-verb fallback
|
|
1530
|
+
# absorbs it too - the .seen-* suppressor then advances and nothing ever re-reads
|
|
1531
|
+
# the event, so the work stalls with the captain never told.
|
|
1532
|
+
test_actionable_signal_survives_a_later_routine_append() {
|
|
1533
|
+
local dir state fakebin out drain_out status_file sig pid
|
|
1534
|
+
dir=$(make_case actionable-masked); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1535
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
1536
|
+
status_file="$state/task.status"
|
|
1537
|
+
# Everything through "working: setup" was already classified, so this asserts
|
|
1538
|
+
# the newly appended span, not merely a whole-file re-read.
|
|
1539
|
+
printf 'working: setup\n' > "$status_file"
|
|
1540
|
+
sig=$(seen_sig "$status_file"); printf '%s' "$sig" > "$state/.seen-task_status"
|
|
1541
|
+
printf 'needs-decision: pick A or B\nworking: still tidying the branch\n' >> "$status_file"
|
|
1542
|
+
# Positive evidence the crew is still working, so the no-verb fallback cannot
|
|
1543
|
+
# rescue the wake: only reading the event itself can surface it.
|
|
1544
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
1545
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1546
|
+
pid=$!
|
|
1547
|
+
wait_for_exit "$pid" 100 \
|
|
1548
|
+
|| { reap "$pid"; fail "watcher absorbed a needs-decision hidden behind a later working: line"; }
|
|
1549
|
+
grep -F "signal: $status_file" "$out" >/dev/null || fail "watcher did not print the actionable signal reason"
|
|
1550
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the masked signal failed"
|
|
1551
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$status_file" >/dev/null \
|
|
1552
|
+
|| fail "the masked actionable signal was not queued"
|
|
1553
|
+
unset FM_FAKE_CREW_STATE
|
|
1554
|
+
pass "a captain event hidden behind a later routine append is still surfaced (queue + exit)"
|
|
1555
|
+
}
|
|
1556
|
+
|
|
1557
|
+
# The captain-reported completion shape of the same masking, end to end.
|
|
1558
|
+
test_release_completion_survives_a_later_routine_append() {
|
|
1559
|
+
local dir state fakebin out drain_out status_file sig pid
|
|
1560
|
+
dir=$(make_case release-masked); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1561
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
1562
|
+
status_file="$state/task.status"
|
|
1563
|
+
printf 'working: publishing\n' > "$status_file"
|
|
1564
|
+
sig=$(seen_sig "$status_file"); printf '%s' "$sig" > "$state/.seen-task_status"
|
|
1565
|
+
printf 'done: release 1.4.0 published and installed\nworking: cleaning the build dir\n' >> "$status_file"
|
|
1566
|
+
export FM_FAKE_CREW_STATE='state: working · source: pane · harness busy'
|
|
1567
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1568
|
+
pid=$!
|
|
1569
|
+
wait_for_exit "$pid" 100 \
|
|
1570
|
+
|| { reap "$pid"; fail "watcher absorbed a release/install completion hidden behind later cleanup chatter"; }
|
|
1571
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the masked completion failed"
|
|
1572
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$status_file" >/dev/null \
|
|
1573
|
+
|| fail "the masked completion was not queued"
|
|
1574
|
+
unset FM_FAKE_CREW_STATE
|
|
1575
|
+
pass "a finished release reported before routine cleanup chatter is still surfaced"
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1578
|
+
# The other direction: the fix must not turn ordinary progress into wakes.
|
|
1579
|
+
test_routine_appends_after_a_classified_event_stay_absorbed() {
|
|
1580
|
+
local dir state fakebin out status_file sig pid
|
|
1581
|
+
dir=$(make_case actionable-classified); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1582
|
+
out="$dir/watch.out"
|
|
1583
|
+
status_file="$state/task.status"
|
|
1584
|
+
# The decision is BEHIND the classified position, so only the new routine line
|
|
1585
|
+
# is in the span. A supervisor that re-read the whole log would wake again here.
|
|
1586
|
+
printf 'working: setup\nneeds-decision: pick A or B\n' > "$status_file"
|
|
1587
|
+
sig=$(seen_sig "$status_file"); printf '%s' "$sig" > "$state/.seen-task_status"
|
|
1588
|
+
printf 'working: still tidying the branch\n' >> "$status_file"
|
|
1589
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
1590
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1591
|
+
pid=$!
|
|
1592
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
1593
|
+
reap "$pid"; fail "watcher re-surfaced a decision it had already classified: $(cat "$out")"
|
|
1594
|
+
fi
|
|
1595
|
+
[ ! -s "$state/.wake-queue" ] || fail "a routine append after a classified decision enqueued a wake"
|
|
1596
|
+
reap "$pid"
|
|
1597
|
+
unset FM_FAKE_CREW_STATE
|
|
1598
|
+
pass "a routine append after an already-classified event is absorbed (no re-wake)"
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
test_unreadable_status_reports_once_per_file_state() {
|
|
1602
|
+
local dir state fakebin out status_file target marker sig pid
|
|
1603
|
+
dir=$(make_case unreadable-status); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1604
|
+
out="$dir/watch.out"; status_file="$state/task.status"; target="$dir/missing-status-target"
|
|
1605
|
+
ln -s "$target" "$status_file"
|
|
1606
|
+
marker="$state/.seen-task_status"
|
|
1607
|
+
|
|
1608
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1609
|
+
pid=$!
|
|
1610
|
+
wait_for_exit "$pid" 100 || { reap "$pid"; fail "a dangling status symlink was not reported"; }
|
|
1611
|
+
grep -Fx "signal: $status_file" "$out" >/dev/null \
|
|
1612
|
+
|| fail "a dangling status symlink did not use the immediate signal path: $(cat "$out")"
|
|
1613
|
+
sig=$(status_observed_signature "$status_file")
|
|
1614
|
+
status_presentation_marker_reported_matches "$marker" "$sig" \
|
|
1615
|
+
|| fail "the unreadable status report did not advance its wake signature"
|
|
1616
|
+
[ "$(status_presentation_marker_offset "$marker" "$status_file")" = 0 ] \
|
|
1617
|
+
|| fail "the unreadable status report advanced its classification position"
|
|
1618
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the first unreadable-status wake"
|
|
1619
|
+
touch "$state/.last-check" "$state/.last-heartbeat"
|
|
1620
|
+
|
|
1621
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1622
|
+
pid=$!
|
|
1623
|
+
wait_poll_cycle "$state" "$pid" \
|
|
1624
|
+
|| { reap "$pid"; fail "an unchanged unreadable status reported again after restart: $(cat "$out")"; }
|
|
1625
|
+
reap "$pid"
|
|
1626
|
+
|
|
1627
|
+
printf 'blocked: changed target state with a longer path\n' > "$dir/status-target-two-longer"
|
|
1628
|
+
ln -snf "$dir/status-target-two-longer" "$status_file"
|
|
1629
|
+
target="$dir/status-target-two-longer"
|
|
1630
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1631
|
+
pid=$!
|
|
1632
|
+
wait_for_exit "$pid" 100 || { reap "$pid"; fail "a changed unreadable status did not report again"; }
|
|
1633
|
+
[ "$(status_presentation_marker_offset "$marker" "$status_file")" = 0 ] \
|
|
1634
|
+
|| fail "a changed unreadable status advanced its classification position"
|
|
1635
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the changed unreadable-status wake"
|
|
1636
|
+
|
|
1637
|
+
rm -f "$status_file"
|
|
1638
|
+
cp "$target" "$status_file"
|
|
1639
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1640
|
+
pid=$!
|
|
1641
|
+
wait_for_exit "$pid" 100 || { reap "$pid"; fail "a readable replacement did not surface preserved content"; }
|
|
1642
|
+
[ "$(status_presentation_marker_offset "$marker" "$status_file")" = "$(size_of "$status_file")" ] \
|
|
1643
|
+
|| fail "readable recovery did not classify content written before the failure"
|
|
1644
|
+
pass "unreadable status reports are bounded without advancing classification"
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
test_permission_recovery_surfaces_preserved_status() {
|
|
1648
|
+
local dir state fakebin out status_file marker before_ident after_ident pid
|
|
1649
|
+
dir=$(make_case permission-recovery); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1650
|
+
out="$dir/watch.out"; status_file="$state/task.status"; marker="$state/.seen-task_status"
|
|
1651
|
+
printf 'blocked: release approval required\nworking: preserving context\n' > "$status_file"
|
|
1652
|
+
before_ident=$(_fm_open_decisions_file_ident "$status_file")
|
|
1653
|
+
chmod 000 "$status_file"
|
|
1654
|
+
if [ -r "$status_file" ]; then
|
|
1655
|
+
chmod 600 "$status_file"
|
|
1656
|
+
pass "permission recovery skipped because permissions cannot deny reads"
|
|
1657
|
+
return
|
|
1658
|
+
fi
|
|
1659
|
+
|
|
1660
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1661
|
+
pid=$!
|
|
1662
|
+
wait_for_exit "$pid" 100 || { reap "$pid"; chmod 600 "$status_file"; fail "an unreadable regular status was not reported"; }
|
|
1663
|
+
[ "$(status_presentation_marker_offset "$marker" "$status_file")" = 0 ] \
|
|
1664
|
+
|| { chmod 600 "$status_file"; fail "an unreadable regular status advanced its classification position"; }
|
|
1665
|
+
ack_stopped_cycle "$state" || { chmod 600 "$status_file"; fail "could not acknowledge the unreadable regular-status wake"; }
|
|
1666
|
+
touch "$state/.last-check" "$state/.last-heartbeat"
|
|
1667
|
+
|
|
1668
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
1669
|
+
pid=$!
|
|
1670
|
+
wait_poll_cycle "$state" "$pid" \
|
|
1671
|
+
|| { reap "$pid"; chmod 600 "$status_file"; fail "an unchanged unreadable regular status reported again"; }
|
|
1672
|
+
|
|
1673
|
+
chmod 600 "$status_file"
|
|
1674
|
+
after_ident=$(_fm_open_decisions_file_ident "$status_file")
|
|
1675
|
+
[ "$after_ident" = "$before_ident" ] || { reap "$pid"; fail "the permission-only recovery changed file identity"; }
|
|
1676
|
+
wait_for_exit "$pid" 100 || { reap "$pid"; fail "readability recovery did not surface preserved content"; }
|
|
1677
|
+
grep -Fx "signal: $status_file" "$out" >/dev/null \
|
|
1678
|
+
|| fail "readability recovery did not use the actionable signal path: $(cat "$out")"
|
|
1679
|
+
[ "$(status_presentation_marker_offset "$marker" "$status_file")" = "$(size_of "$status_file")" ] \
|
|
1680
|
+
|| fail "readability recovery did not classify from the unadvanced position"
|
|
1681
|
+
pass "permission recovery surfaces content from the unadvanced position"
|
|
1682
|
+
}
|
|
1683
|
+
|
|
1684
|
+
test_terminal_stale_surfaced() {
|
|
1685
|
+
local dir state fakebin out drain_out capture_file window key pane_hash sig pid
|
|
1686
|
+
dir=$(make_case terminal-stale); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1687
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1688
|
+
window="test:fm-done"
|
|
1689
|
+
printf 'finished, awaiting review' > "$capture_file"
|
|
1690
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/done.meta"
|
|
1691
|
+
printf 'done: PR https://example.test/pr/3\n' > "$state/done.status"
|
|
1692
|
+
sig=$(seen_sig "$state/done.status"); printf '%s' "$sig" > "$state/.seen-done_status"
|
|
1693
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1694
|
+
pane_hash=$(hash_text "finished, awaiting review")
|
|
1695
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
1696
|
+
printf '1\n' > "$state/.count-$key"
|
|
1697
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1698
|
+
FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1699
|
+
pid=$!
|
|
1700
|
+
wait_for_exit "$pid" 100 || fail "watcher did not exit for a stale pane on a terminal status"
|
|
1701
|
+
grep -Fx "stale: $window" "$out" >/dev/null || fail "watcher did not print the terminal stale wake"
|
|
1702
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the terminal stale failed"
|
|
1703
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "terminal stale was not queued"
|
|
1704
|
+
pass "a stale pane sitting on a terminal status is surfaced (queue + exit)"
|
|
1705
|
+
}
|
|
1706
|
+
|
|
1707
|
+
# --- stale pane, STALE terminal status overridden by an active run: absorbed ---
|
|
1708
|
+
# Regression for the 2026-07 herdr false-surface incidents: a crew's own status
|
|
1709
|
+
# log gets no new entry once firstmate hands it to a no-mistakes validation
|
|
1710
|
+
# (AGENTS.md's sparse status-reporting contract), so the log keeps showing its
|
|
1711
|
+
# pre-validation "done:" line as the LAST line for the run's entire (possibly
|
|
1712
|
+
# many-minutes) duration. stale_is_terminal alone has no run-step awareness and
|
|
1713
|
+
# would treat that leftover as still-current every time the pane goes quiet,
|
|
1714
|
+
# immediately surfacing a crew that is actively validating. crew_is_provably_working
|
|
1715
|
+
# must get a chance to override a captain-relevant-but-stale status line, exactly
|
|
1716
|
+
# as it already does for a plain non-terminal one.
|
|
1717
|
+
test_stale_terminal_status_overridden_by_active_run() {
|
|
1718
|
+
local dir state fakebin out drain_out capture_file window key pane_hash sig pid
|
|
1719
|
+
dir=$(make_case terminal-stale-overridden); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1720
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1721
|
+
window="test:fm-validating"
|
|
1722
|
+
printf 'no-mistakes axi run: validating...' > "$capture_file"
|
|
1723
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/validating.meta"
|
|
1724
|
+
# The crew reported done BEFORE firstmate triggered no-mistakes validation;
|
|
1725
|
+
# this line never gets superseded by a newer status-log entry while the
|
|
1726
|
+
# pipeline itself runs.
|
|
1727
|
+
printf 'done: implementation complete, ready to validate\n' > "$state/validating.status"
|
|
1728
|
+
sig=$(seen_sig "$state/validating.status"); printf '%s' "$sig" > "$state/.seen-validating_status"
|
|
1729
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1730
|
+
pane_hash=$(hash_text "no-mistakes axi run: validating...")
|
|
1731
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
1732
|
+
printf '1\n' > "$state/.count-$key"
|
|
1733
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
1734
|
+
|
|
1735
|
+
# Phase A: a high escalation threshold means the first sighting is absorbed,
|
|
1736
|
+
# not surfaced, despite the captain-relevant "done:" status-log line.
|
|
1737
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1738
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1739
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1740
|
+
pid=$!
|
|
1741
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
1742
|
+
reap "$pid"; fail "watcher exited for a stale terminal-looking status the run-step overrides (should absorb): $(cat "$out")"
|
|
1743
|
+
fi
|
|
1744
|
+
[ ! -s "$out" ] || fail "the overridden stale terminal status printed a wake reason during absorb"
|
|
1745
|
+
[ ! -s "$state/.wake-queue" ] || fail "the overridden stale terminal status enqueued a wake during absorb"
|
|
1746
|
+
[ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on absorb"
|
|
1747
|
+
[ -s "$state/.stale-since-$key" ] || fail "stale-since escalation timer was not recorded on absorb"
|
|
1748
|
+
[ ! -e "$state/.hb-surfaced-validating" ] || fail "an absorbed wake must not mark the status line as surfaced"
|
|
1749
|
+
reap "$pid"
|
|
1750
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional phase-A watcher stop"
|
|
1751
|
+
|
|
1752
|
+
# Phase B: backdate the idle timer past the threshold; the run genuinely
|
|
1753
|
+
# wedges and the next poll escalates exactly like the non-terminal case.
|
|
1754
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
1755
|
+
: > "$out"
|
|
1756
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1757
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1758
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1759
|
+
pid=$!
|
|
1760
|
+
wait_for_exit "$pid" 100 || fail "watcher did not escalate an overridden stale terminal status past the threshold"
|
|
1761
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "escalation did not print a stale wake"
|
|
1762
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "escalation did not flag a possible wedge"
|
|
1763
|
+
unset FM_FAKE_CREW_STATE
|
|
1764
|
+
pass "a stale terminal-looking status is overridden and absorbed while a run is actively working, then wedge-escalated"
|
|
1765
|
+
}
|
|
1766
|
+
|
|
1767
|
+
# --- non-terminal stale, crew provably working: absorbed, then wedge-escalated ---
|
|
1768
|
+
# A provably-working crew (an actively-running pipeline) legitimately sits on a
|
|
1769
|
+
# static pane (e.g. waiting on CI), so a non-terminal stale is absorbed and only
|
|
1770
|
+
# the wedge timer eventually escalates it - the low-churn behavior preserved.
|
|
1771
|
+
|
|
1772
|
+
test_nonterminal_stale_provably_working_absorbed_then_escalated() {
|
|
1773
|
+
local dir state fakebin out drain_out capture_file window key pane_hash sig pid
|
|
1774
|
+
dir=$(make_case nonterminal-stale-working); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1775
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1776
|
+
window="test:fm-quiet"
|
|
1777
|
+
printf 'idle building output' > "$capture_file"
|
|
1778
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/quiet.meta"
|
|
1779
|
+
# Non-terminal status, and prime .seen-* so the signal scan does not pre-empt
|
|
1780
|
+
# the stale path.
|
|
1781
|
+
printf 'working: still compiling\n' > "$state/quiet.status"
|
|
1782
|
+
sig=$(seen_sig "$state/quiet.status"); printf '%s' "$sig" > "$state/.seen-quiet_status"
|
|
1783
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1784
|
+
pane_hash=$(hash_text "idle building output")
|
|
1785
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
1786
|
+
printf '1\n' > "$state/.count-$key"
|
|
1787
|
+
# The crew's pipeline is actively running: a static pane is normal (waiting on CI).
|
|
1788
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · ci running'
|
|
1789
|
+
|
|
1790
|
+
# Phase A: a high escalation threshold means the first sighting is absorbed.
|
|
1791
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1792
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1793
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1794
|
+
pid=$!
|
|
1795
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
1796
|
+
reap "$pid"; fail "watcher exited for a fresh provably-working non-terminal stale (should absorb): $(cat "$out")"
|
|
1797
|
+
fi
|
|
1798
|
+
[ ! -s "$out" ] || fail "fresh provably-working stale printed a wake reason during absorb"
|
|
1799
|
+
[ ! -s "$state/.wake-queue" ] || fail "fresh provably-working stale enqueued a wake during absorb"
|
|
1800
|
+
[ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on absorb"
|
|
1801
|
+
[ -s "$state/.stale-since-$key" ] || fail "stale-since escalation timer was not recorded on absorb"
|
|
1802
|
+
reap "$pid"
|
|
1803
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional phase-A watcher stop"
|
|
1804
|
+
|
|
1805
|
+
# Phase B: backdate the idle timer past the threshold; the next run escalates.
|
|
1806
|
+
# (The subsequent-sight timer path does not re-read the crew state.)
|
|
1807
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
1808
|
+
: > "$out"
|
|
1809
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1810
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1811
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1812
|
+
pid=$!
|
|
1813
|
+
wait_for_exit "$pid" 100 || fail "watcher did not escalate a provably-working non-terminal stale past the threshold"
|
|
1814
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "escalation did not print a stale wake"
|
|
1815
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "escalation did not flag a possible wedge"
|
|
1816
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "stale-since timer was not cleared after escalation"
|
|
1817
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the wedge escalation failed"
|
|
1818
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "wedge escalation was not queued"
|
|
1819
|
+
pass "provably-working non-terminal stale is absorbed on first sight, then wedge-escalated past the threshold"
|
|
1820
|
+
}
|
|
1821
|
+
|
|
1822
|
+
# --- non-terminal stale, crew NOT provably working: surfaced immediately ------
|
|
1823
|
+
# The key requirement: a crew with no running pipeline that has gone quiet (and is
|
|
1824
|
+
# not busy) has stopped - it may be done via interactive menus, waiting, or wedged.
|
|
1825
|
+
# It must surface at once, never wait out the wedge timer, so these users (a
|
|
1826
|
+
# non-no-mistakes crew, or any crew with no running pipeline) are never left hanging.
|
|
1827
|
+
|
|
1828
|
+
test_nonterminal_stale_not_working_surfaced() {
|
|
1829
|
+
local dir state fakebin out drain_out capture_file window key pane_hash sig pid
|
|
1830
|
+
dir=$(make_case nonterminal-stale-stopped); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1831
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1832
|
+
window="test:fm-stopped"
|
|
1833
|
+
printf 'idle prompt, finished' > "$capture_file"
|
|
1834
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/stopped.meta"
|
|
1835
|
+
# Non-terminal status (the crew never wrote a captain-relevant verb), .seen-*
|
|
1836
|
+
# primed so the signal scan does not pre-empt the stale path.
|
|
1837
|
+
printf 'working: implementing\n' > "$state/stopped.status"
|
|
1838
|
+
sig=$(seen_sig "$state/stopped.status"); printf '%s' "$sig" > "$state/.seen-stopped_status"
|
|
1839
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1840
|
+
pane_hash=$(hash_text "idle prompt, finished")
|
|
1841
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
1842
|
+
printf '1\n' > "$state/.count-$key"
|
|
1843
|
+
# No running pipeline; the pane is idle. NOT provably working.
|
|
1844
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available'
|
|
1845
|
+
|
|
1846
|
+
# Even with a high wedge threshold, a not-provably-working stale surfaces at once.
|
|
1847
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1848
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1849
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1850
|
+
pid=$!
|
|
1851
|
+
wait_for_exit "$pid" 100 || fail "watcher did not surface a not-provably-working non-terminal stale at once"
|
|
1852
|
+
grep -Fx "stale: $window" "$out" >/dev/null || fail "watcher did not print the immediate stale wake"
|
|
1853
|
+
grep -F "possible wedge" "$out" >/dev/null && fail "an immediate stopped-crew stale was mislabeled a wedge"
|
|
1854
|
+
[ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor was not advanced on surface"
|
|
1855
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "stale-since timer should not be set when surfacing immediately"
|
|
1856
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the immediate stale failed"
|
|
1857
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "immediate stale wake was not queued"
|
|
1858
|
+
pass "a not-provably-working non-terminal stale is surfaced immediately (never left to wait out the timer)"
|
|
1859
|
+
}
|
|
1860
|
+
|
|
1861
|
+
# --- non-terminal stale, crew DECLARED a pause: absorbed, re-surfaced on a long
|
|
1862
|
+
# cadence, never wedge-escalated ------------------------------------------
|
|
1863
|
+
# The live 2026-07-09/10 case: a crew intentionally held awaiting an upstream tool
|
|
1864
|
+
# release (paused: ...) whose idle pane tripped repeated possible-wedge escalations
|
|
1865
|
+
# all day. With the paused verb, its stale is absorbed like a working crew but never
|
|
1866
|
+
# uses the wedge timer; it re-surfaces once past PAUSE_RESURFACE_SECS (anchored on
|
|
1867
|
+
# the pause's own status-file age, so a churny idle pane cannot reset the cadence)
|
|
1868
|
+
# for a recheck, so a forgotten pause cannot rot invisibly.
|
|
1869
|
+
test_nonterminal_stale_paused_absorbed_then_resurfaced() {
|
|
1870
|
+
local dir state fakebin out drain_out capture_file window key pane_hash sig pid back statusf
|
|
1871
|
+
dir=$(make_case nonterminal-stale-paused); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1872
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
1873
|
+
window="test:fm-held"
|
|
1874
|
+
printf 'idle, holding for upstream' > "$capture_file"
|
|
1875
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/held.meta"
|
|
1876
|
+
statusf="$state/held.status"
|
|
1877
|
+
# A DECLARED pause (not captain-relevant), .seen-* primed so the signal scan does
|
|
1878
|
+
# not pre-empt the stale path.
|
|
1879
|
+
printf 'paused: holding for the upstream tool release\n' > "$statusf"
|
|
1880
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-held_status"
|
|
1881
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1882
|
+
pane_hash=$(hash_text "idle, holding for upstream")
|
|
1883
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
1884
|
+
printf '1\n' > "$state/.count-$key"
|
|
1885
|
+
# crew_absorb_class reads the declared pause from fm-crew-state.sh.
|
|
1886
|
+
export FM_FAKE_CREW_STATE='state: paused · source: status-log · holding for the upstream tool release'
|
|
1887
|
+
|
|
1888
|
+
# Phase A: a fresh pause (status file just written) under a high re-surface
|
|
1889
|
+
# threshold is absorbed - no wake, no wedge timer.
|
|
1890
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1891
|
+
FM_FAKE_TMUX_CURRENT_COMMAND=zsh \
|
|
1892
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1893
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1894
|
+
pid=$!
|
|
1895
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
1896
|
+
reap "$pid"; fail "watcher exited for a fresh declared pause (should absorb): $(cat "$out")"
|
|
1897
|
+
fi
|
|
1898
|
+
[ ! -s "$out" ] || fail "fresh paused stale printed a wake reason during absorb"
|
|
1899
|
+
[ ! -s "$state/.wake-queue" ] || fail "fresh paused stale enqueued a wake during absorb"
|
|
1900
|
+
[ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] || fail "stale suppressor not advanced on paused absorb"
|
|
1901
|
+
[ -e "$state/.paused-$key" ] || fail "paused flag not recorded on absorb"
|
|
1902
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "a paused absorb must not start the wedge timer"
|
|
1903
|
+
reap "$pid"
|
|
1904
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional paused phase-A stop"
|
|
1905
|
+
|
|
1906
|
+
# Phase B: age the pause past the (now normal) threshold by backdating its
|
|
1907
|
+
# status file, re-prime .seen-* to the new signature so the signal scan stays
|
|
1908
|
+
# quiet, and confirm it re-surfaces as a paused recheck - never a wedge.
|
|
1909
|
+
back=$(( $(date +%s) - 500 ))
|
|
1910
|
+
if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf"
|
|
1911
|
+
else touch -m -d "@$back" "$statusf"; fi
|
|
1912
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-held_status"
|
|
1913
|
+
: > "$out"
|
|
1914
|
+
printf 'idle, holding for upstream (token 2)' > "$capture_file"
|
|
1915
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1916
|
+
FM_FAKE_TMUX_CURRENT_COMMAND=zsh \
|
|
1917
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1918
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
1919
|
+
pid=$!
|
|
1920
|
+
wait_for_exit "$pid" 100 || fail "watcher did not re-surface a declared pause past the threshold"
|
|
1921
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "re-surface did not print a stale wake"
|
|
1922
|
+
grep -F "awaiting external" "$out" >/dev/null || fail "re-surface was not labeled a paused/awaiting-external recheck"
|
|
1923
|
+
grep -F "possible wedge" "$out" >/dev/null && fail "a declared pause was mislabeled a possible wedge"
|
|
1924
|
+
[ -e "$state/.paused-resurfaced-$key" ] || fail "the paused re-surface throttle marker was not recorded"
|
|
1925
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "a paused re-surface must not use the wedge timer"
|
|
1926
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the paused re-surface failed"
|
|
1927
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "paused re-surface was not queued"
|
|
1928
|
+
pass "a declared pause is absorbed on first sight, then re-surfaced as a recheck past the threshold, never wedge-escalated"
|
|
1929
|
+
}
|
|
1930
|
+
|
|
1931
|
+
# A captain-held crew can leave a stable backend endpoint after its agent exits.
|
|
1932
|
+
# fm-crew-state then authoritatively reports stopped rather than paused, but the
|
|
1933
|
+
# confirmed-dead agent plus the declared wait or captain-held transfer must retain
|
|
1934
|
+
# bounded pause handling.
|
|
1935
|
+
# A still-live agent at an external-decision gate is the disconfirming case: it
|
|
1936
|
+
# must surface once, while the unchanged hash must not append the same wake on
|
|
1937
|
+
# every watcher re-arm.
|
|
1938
|
+
test_exited_declared_pause_is_bounded_but_live_gate_surfaces() {
|
|
1939
|
+
local dir state fakebin out capture_file statusf window key pane_hash sig pid back round wakes bare
|
|
1940
|
+
dir=$(make_case exited-declared-pause); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1941
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/held.status"
|
|
1942
|
+
window="test:fm-held"
|
|
1943
|
+
printf 'idle bare shell after agent exit\n' > "$capture_file"
|
|
1944
|
+
printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/held.meta"
|
|
1945
|
+
printf 'paused: held per captain while an external decision is pending\n' > "$statusf"
|
|
1946
|
+
back=$(( $(date +%s) - 500 ))
|
|
1947
|
+
if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf"
|
|
1948
|
+
else touch -m -d "@$back" "$statusf"; fi
|
|
1949
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-held_status"
|
|
1950
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1951
|
+
pane_hash=$(hash_text "idle bare shell after agent exit")
|
|
1952
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
1953
|
+
printf '1\n' > "$state/.count-$key"
|
|
1954
|
+
|
|
1955
|
+
round=1
|
|
1956
|
+
while [ "$round" -le 6 ]; do
|
|
1957
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
1958
|
+
FM_FAKE_TMUX_CURRENT_COMMAND=zsh FM_FAKE_CREW_STATE='state: stopped · source: pane · bare shell' \
|
|
1959
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
1960
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" &
|
|
1961
|
+
pid=$!
|
|
1962
|
+
if wait_poll_cycle "$state" "$pid"; then
|
|
1963
|
+
reap "$pid"
|
|
1964
|
+
elif kill -0 "$pid" 2>/dev/null; then
|
|
1965
|
+
reap "$pid"
|
|
1966
|
+
fail "dead-agent watcher round $round timed out before completing a poll cycle"
|
|
1967
|
+
else
|
|
1968
|
+
wait "$pid" || fail "dead-agent watcher round $round failed"
|
|
1969
|
+
fi
|
|
1970
|
+
round=$((round + 1))
|
|
1971
|
+
done
|
|
1972
|
+
# A watcher that queues nothing never creates .wake-queue, so these counts
|
|
1973
|
+
# read a path that may legitimately be absent. awk aborts on a missing file
|
|
1974
|
+
# before END runs, which collapses the count to the empty string and turns the
|
|
1975
|
+
# next comparison into an "integer expression expected" error - reported as a
|
|
1976
|
+
# flood of an unprintable number of wakes instead of the real contract breach
|
|
1977
|
+
# the grep below names. No queue means no wakes, per the drain-count read at
|
|
1978
|
+
# the end of this file.
|
|
1979
|
+
wakes=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w { n++ } END { print n + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0)
|
|
1980
|
+
bare=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w && $5 == "stale: " w { n++ } END { print n + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0)
|
|
1981
|
+
[ "$wakes" -le 1 ] || fail "dead-agent declared pause flooded $wakes stale wakes across six unchanged polls"
|
|
1982
|
+
[ "$bare" -eq 0 ] || fail "dead-agent declared pause surfaced as $bare bare stopped-crew wakes"
|
|
1983
|
+
grep -F "awaiting external" "$state/.wake-queue" >/dev/null \
|
|
1984
|
+
|| fail "dead-agent declared pause did not use the bounded paused recheck"
|
|
1985
|
+
|
|
1986
|
+
dir=$(make_case exited-captain-held); state="$dir/state"; fakebin="$dir/fakebin"
|
|
1987
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/held.status"
|
|
1988
|
+
window="test:fm-held"
|
|
1989
|
+
printf 'idle bare shell after captain-held transfer\n' > "$capture_file"
|
|
1990
|
+
printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/held.meta"
|
|
1991
|
+
printf 'captain-held [key=route]: tracked by held-decision-route\n' > "$statusf"
|
|
1992
|
+
back=$(( $(date +%s) - 500 ))
|
|
1993
|
+
if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf"
|
|
1994
|
+
else touch -m -d "@$back" "$statusf"; fi
|
|
1995
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-held_status"
|
|
1996
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
1997
|
+
pane_hash=$(hash_text "idle bare shell after captain-held transfer")
|
|
1998
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
1999
|
+
printf '1\n' > "$state/.count-$key"
|
|
2000
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2001
|
+
FM_FAKE_TMUX_CURRENT_COMMAND=zsh FM_FAKE_CREW_STATE='state: stopped · source: pane · bare shell' \
|
|
2002
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2003
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2004
|
+
pid=$!
|
|
2005
|
+
wait_for_exit "$pid" 100 || fail "captain-held dead-agent pane did not re-surface on the bounded cadence"
|
|
2006
|
+
grep -F "awaiting the captain" "$state/.wake-queue" >/dev/null \
|
|
2007
|
+
|| fail "captain-held dead-agent pane surfaced as a stopped crew instead of a captain-owned recheck: $(cat "$state/.wake-queue")"
|
|
2008
|
+
grep -F "awaiting external" "$state/.wake-queue" >/dev/null \
|
|
2009
|
+
&& fail "captain-held dead-agent pane borrowed the pause verb's external-wait wording"
|
|
2010
|
+
|
|
2011
|
+
dir=$(make_case alive-decision-gate); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2012
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/gate.status"
|
|
2013
|
+
window="test:fm-gate"
|
|
2014
|
+
printf 'idle external-decision gate\n' > "$capture_file"
|
|
2015
|
+
printf 'window=%s\nkind=ship\nharness=grok\nbackend=tmux\n' "$window" > "$state/gate.meta"
|
|
2016
|
+
printf 'paused: waiting at an active external-decision gate\n' > "$statusf"
|
|
2017
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-gate_status"
|
|
2018
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2019
|
+
pane_hash=$(hash_text "idle external-decision gate")
|
|
2020
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2021
|
+
printf '1\n' > "$state/.count-$key"
|
|
2022
|
+
|
|
2023
|
+
# First sight must surface promptly so a live external-decision gate is not
|
|
2024
|
+
# hidden behind the pause cadence.
|
|
2025
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2026
|
+
FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting at an active external-decision gate' \
|
|
2027
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2028
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" &
|
|
2029
|
+
pid=$!
|
|
2030
|
+
wait_for_exit "$pid" 100 || fail "live external-decision gate did not surface immediately"
|
|
2031
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the immediate external-decision surface"
|
|
2032
|
+
|
|
2033
|
+
# Re-arm with the stale timer already beyond the wedge threshold. This is the
|
|
2034
|
+
# exact unchanged-hash fallback after the immediate surface: it must retain
|
|
2035
|
+
# the pause cadence and discard any residual wedge timer instead of emitting
|
|
2036
|
+
# a second possible-wedge wake.
|
|
2037
|
+
printf '%s\n' $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2038
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2039
|
+
FM_FAKE_TMUX_CURRENT_COMMAND=grok FM_FAKE_CREW_STATE='state: paused · source: status-log · waiting at an active external-decision gate' \
|
|
2040
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2041
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" >> "$out" &
|
|
2042
|
+
pid=$!
|
|
2043
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2044
|
+
reap "$pid"
|
|
2045
|
+
fail "live external-decision gate escalated on the wedge timer after its immediate surface: $(cat "$out")"
|
|
2046
|
+
fi
|
|
2047
|
+
[ -e "$state/.paused-$key" ] || { reap "$pid"; fail "live external-decision gate lost its pause cadence marker"; }
|
|
2048
|
+
[ ! -e "$state/.stale-since-$key" ] || { reap "$pid"; fail "live external-decision gate retained the wedge timer"; }
|
|
2049
|
+
reap "$pid"
|
|
2050
|
+
wakes=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w { n++ } END { print n + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0)
|
|
2051
|
+
bare=$(awk -F '\t' -v w="$window" '$3 == "stale" && $4 == w && $5 == "stale: " w { n++ } END { print n + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0)
|
|
2052
|
+
[ "$wakes" -eq 0 ] || fail "acknowledged external-decision surface replayed $wakes wakes"
|
|
2053
|
+
[ "$bare" -eq 0 ] || fail "acknowledged external-decision bare stale remained queued"
|
|
2054
|
+
pass "exited declared-pause and captain-held panes use bounded pause cadence while a live decision gate still surfaces once"
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
test_secondmate_paused_resurfaces_in_normal_mode() {
|
|
2058
|
+
local dir state fakebin out capture_file statusf window key pane_hash sig pid back
|
|
2059
|
+
dir=$(make_case secondmate-paused-resurface); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2060
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/secondmate-held.status"
|
|
2061
|
+
window="test:fm-secondmate-held"
|
|
2062
|
+
printf 'idle awaiting external\n' > "$capture_file"
|
|
2063
|
+
printf 'window=%s\nkind=secondmate\n' "$window" > "$state/secondmate-held.meta"
|
|
2064
|
+
printf 'paused: awaiting the upstream release\n' > "$statusf"
|
|
2065
|
+
back=$(( $(date +%s) - 500 ))
|
|
2066
|
+
if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf"
|
|
2067
|
+
else touch -m -d "@$back" "$statusf"; fi
|
|
2068
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-secondmate-held_status"
|
|
2069
|
+
key=$(printf '%s' "$window" | tr '.:/' '___')
|
|
2070
|
+
pane_hash=$(hash_text "idle awaiting external")
|
|
2071
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2072
|
+
printf '1\n' > "$state/.count-$key"
|
|
2073
|
+
export FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting the upstream release'
|
|
2074
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2075
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2076
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2077
|
+
pid=$!
|
|
2078
|
+
wait_for_exit "$pid" 100 || fail "watcher did not re-surface a paused secondmate"
|
|
2079
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "paused secondmate did not emit a stale recheck"
|
|
2080
|
+
grep -F "awaiting external" "$out" >/dev/null || fail "paused secondmate recheck omitted its external-wait reason"
|
|
2081
|
+
grep -F "awaiting the captain" "$out" >/dev/null && fail "paused secondmate recheck named the captain instead of its external dependency"
|
|
2082
|
+
grep -F "possible wedge" "$out" >/dev/null && fail "paused secondmate was mislabeled a wedge"
|
|
2083
|
+
unset FM_FAKE_CREW_STATE
|
|
2084
|
+
pass "a declared paused secondmate re-surfaces on the bounded normal-mode cadence"
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
# A captain hold is the other declared wait, but unlike paused: it has no
|
|
2088
|
+
# current-state mapping, so a held mate reports `unknown` rather than `paused`.
|
|
2089
|
+
# The bounded re-surface must still reach it, or a mate's hold rots invisibly:
|
|
2090
|
+
# nothing else re-reads a quiet mate's endpoint.
|
|
2091
|
+
test_secondmate_captain_held_resurfaces_in_normal_mode() {
|
|
2092
|
+
local dir state fakebin out capture_file statusf window key pane_hash sig pid back
|
|
2093
|
+
dir=$(make_case secondmate-held-resurface); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2094
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/secondmate-hold.status"
|
|
2095
|
+
window="test:fm-secondmate-hold"
|
|
2096
|
+
printf 'idle awaiting the captain\n' > "$capture_file"
|
|
2097
|
+
printf 'window=%s\nkind=secondmate\n' "$window" > "$state/secondmate-hold.meta"
|
|
2098
|
+
printf 'captain-held [key=route]: tracked by task-decision-route\n' > "$statusf"
|
|
2099
|
+
back=$(( $(date +%s) - 500 ))
|
|
2100
|
+
if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf"
|
|
2101
|
+
else touch -m -d "@$back" "$statusf"; fi
|
|
2102
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-secondmate-hold_status"
|
|
2103
|
+
key=$(printf '%s' "$window" | tr '.:/' '___')
|
|
2104
|
+
pane_hash=$(hash_text "idle awaiting the captain")
|
|
2105
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2106
|
+
printf '1\n' > "$state/.count-$key"
|
|
2107
|
+
export FM_FAKE_CREW_STATE='state: unknown · source: none · no current-state source available'
|
|
2108
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2109
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2110
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2111
|
+
pid=$!
|
|
2112
|
+
wait_for_exit "$pid" 100 || fail "watcher did not re-surface a captain-held secondmate"
|
|
2113
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "captain-held secondmate did not emit a stale recheck"
|
|
2114
|
+
grep -F "awaiting the captain" "$out" >/dev/null || fail "captain-held secondmate recheck did not name the captain as the blocker: $(cat "$out")"
|
|
2115
|
+
grep -F "awaiting external" "$out" >/dev/null && fail "captain-held secondmate recheck claimed an external wait"
|
|
2116
|
+
grep -F "possible wedge" "$out" >/dev/null && fail "captain-held secondmate was mislabeled a wedge"
|
|
2117
|
+
unset FM_FAKE_CREW_STATE
|
|
2118
|
+
pass "a captain-held secondmate re-surfaces on the bounded normal-mode cadence"
|
|
2119
|
+
}
|
|
2120
|
+
|
|
2121
|
+
test_secondmate_nonpaused_stale_remains_suppressed() {
|
|
2122
|
+
local dir state fakebin out capture_file statusf window key pane_hash sig pid
|
|
2123
|
+
dir=$(make_case secondmate-stale-suppressed); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2124
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; statusf="$state/secondmate-working.status"
|
|
2125
|
+
window="test:fm-secondmate-working"
|
|
2126
|
+
printf 'idle while the parent supervises\n' > "$capture_file"
|
|
2127
|
+
printf 'window=%s\nkind=secondmate\n' "$window" > "$state/secondmate-working.meta"
|
|
2128
|
+
printf 'working: the parent supervises this secondmate\n' > "$statusf"
|
|
2129
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-secondmate-working_status"
|
|
2130
|
+
key=$(printf '%s' "$window" | tr '.:/' '___')
|
|
2131
|
+
pane_hash=$(hash_text "idle while the parent supervises")
|
|
2132
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2133
|
+
printf '1\n' > "$state/.count-$key"
|
|
2134
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2135
|
+
FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2136
|
+
pid=$!
|
|
2137
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2138
|
+
reap "$pid"; fail "watcher surfaced an ordinary secondmate stale pane: $(cat "$out")"
|
|
2139
|
+
fi
|
|
2140
|
+
[ ! -s "$out" ] || { reap "$pid"; fail "ordinary secondmate stale pane printed a wake reason: $(cat "$out")"; }
|
|
2141
|
+
reap "$pid"
|
|
2142
|
+
pass "a non-paused secondmate retains normal stale suppression"
|
|
2143
|
+
}
|
|
2144
|
+
|
|
2145
|
+
test_secondmate_unpause_clears_pause_tracking() {
|
|
2146
|
+
local dir state fakebin out statusf window key pid
|
|
2147
|
+
dir=$(make_case secondmate-unpause-clears); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2148
|
+
out="$dir/watch.out"; statusf="$state/secondmate-resumed.status"; window="test:fm-secondmate-resumed"
|
|
2149
|
+
printf 'window=%s\nkind=secondmate\n' "$window" > "$state/secondmate-resumed.meta"
|
|
2150
|
+
printf 'working: upstream landed\n' > "$statusf"
|
|
2151
|
+
printf '%s' "$(seen_sig "$statusf")" > "$state/.seen-secondmate-resumed_status"
|
|
2152
|
+
key=${window//:/_}
|
|
2153
|
+
key=${key//\//_}
|
|
2154
|
+
key=${key//./_}
|
|
2155
|
+
: > "$state/.paused-$key"
|
|
2156
|
+
: > "$state/.paused-rechecked-$key"
|
|
2157
|
+
: > "$state/.paused-resurfaced-$key"
|
|
2158
|
+
: > "$state/.stale-$key"
|
|
2159
|
+
: > "$state/.stale-since-$key"
|
|
2160
|
+
: > "$state/.wedge-escalations-$key"
|
|
2161
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
2162
|
+
pid=$!
|
|
2163
|
+
wait_poll_cycle "$state" "$pid" || fail "watcher exited while reconciling a resumed secondmate: $(cat "$out")"
|
|
2164
|
+
[ ! -e "$state/.paused-$key" ] || { reap "$pid"; fail "resumed secondmate retained the pause marker"; }
|
|
2165
|
+
[ ! -e "$state/.stale-$key" ] || { reap "$pid"; fail "resumed secondmate retained stale tracking"; }
|
|
2166
|
+
[ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "resumed secondmate retained wedge tracking"; }
|
|
2167
|
+
reap "$pid"
|
|
2168
|
+
pass "a resumed secondmate clears pause and stale tracking before stale exemption"
|
|
2169
|
+
}
|
|
2170
|
+
|
|
2171
|
+
test_nonterminal_stale_pause_transitions_reclassify_unchanged_hash() {
|
|
2172
|
+
local dir state fakebin out capture_file window key pane_hash sig pid i
|
|
2173
|
+
dir=$(make_case nonterminal-stale-pause-transition); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2174
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-transition"
|
|
2175
|
+
printf 'idle awaiting external\n' > "$capture_file"
|
|
2176
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/transition.meta"
|
|
2177
|
+
printf 'paused: awaiting the upstream release\n' > "$state/transition.status"
|
|
2178
|
+
sig=$(seen_sig "$state/transition.status"); printf '%s' "$sig" > "$state/.seen-transition_status"
|
|
2179
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2180
|
+
pane_hash=$(hash_text "idle awaiting external")
|
|
2181
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2182
|
+
printf '%s' "$pane_hash" > "$state/.stale-$key"
|
|
2183
|
+
printf '1\n' > "$state/.count-$key"
|
|
2184
|
+
printf '%s\n' $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2185
|
+
export FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting the upstream release'
|
|
2186
|
+
|
|
2187
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2188
|
+
FM_FAKE_TMUX_CURRENT_COMMAND=zsh \
|
|
2189
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2190
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2191
|
+
pid=$!
|
|
2192
|
+
i=0
|
|
2193
|
+
while [ "$i" -lt 100 ] && kill -0 "$pid" 2>/dev/null; do
|
|
2194
|
+
[ -e "$state/.paused-$key" ] && [ ! -e "$state/.stale-since-$key" ] && break
|
|
2195
|
+
sleep 0.1
|
|
2196
|
+
i=$((i + 1))
|
|
2197
|
+
done
|
|
2198
|
+
kill -0 "$pid" 2>/dev/null || { reap "$pid"; fail "a stale hash that entered pause was wedge-escalated: $(cat "$out")"; }
|
|
2199
|
+
[ -e "$state/.paused-$key" ] || { reap "$pid"; fail "unchanged stale hash did not enter paused mode"; }
|
|
2200
|
+
[ ! -e "$state/.stale-since-$key" ] || { reap "$pid"; fail "pause transition retained its wedge timer"; }
|
|
2201
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a stale hash that entered pause was wedge-escalated: $(cat "$out")"; }
|
|
2202
|
+
reap "$pid"
|
|
2203
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional entered-pause watcher stop"
|
|
2204
|
+
|
|
2205
|
+
printf 'working: upstream landed, resuming\n' > "$state/transition.status"
|
|
2206
|
+
sig=$(seen_sig "$state/transition.status"); printf '%s' "$sig" > "$state/.seen-transition_status"
|
|
2207
|
+
FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
2208
|
+
: > "$out"
|
|
2209
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2210
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2211
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2212
|
+
pid=$!
|
|
2213
|
+
i=0
|
|
2214
|
+
while [ "$i" -lt 100 ] && kill -0 "$pid" 2>/dev/null; do
|
|
2215
|
+
[ ! -e "$state/.paused-$key" ] && [ -s "$state/.stale-since-$key" ] && break
|
|
2216
|
+
sleep 0.1
|
|
2217
|
+
i=$((i + 1))
|
|
2218
|
+
done
|
|
2219
|
+
kill -0 "$pid" 2>/dev/null || { reap "$pid"; fail "a stale hash that left pause did not resume wedge tracking: $(cat "$out")"; }
|
|
2220
|
+
[ ! -e "$state/.paused-$key" ] || { reap "$pid"; fail "unchanged stale hash retained paused mode after resume"; }
|
|
2221
|
+
[ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "unchanged stale hash did not restart wedge tracking after resume"; }
|
|
2222
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a stale hash that left pause did not resume wedge tracking: $(cat "$out")"; }
|
|
2223
|
+
reap "$pid"
|
|
2224
|
+
unset FM_FAKE_CREW_STATE
|
|
2225
|
+
pass "unchanged stale hashes reclassify when a crew enters or leaves pause"
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2228
|
+
test_nonterminal_paused_rechecks_authoritative_state() {
|
|
2229
|
+
local dir state fakebin out capture_file window key pane_hash sig pid
|
|
2230
|
+
dir=$(make_case nonterminal-paused-recheck); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2231
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-pause-recheck"
|
|
2232
|
+
printf 'idle awaiting external\n' > "$capture_file"
|
|
2233
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/pause-recheck.meta"
|
|
2234
|
+
printf 'paused: awaiting the upstream release\n' > "$state/pause-recheck.status"
|
|
2235
|
+
sig=$(seen_sig "$state/pause-recheck.status"); printf '%s' "$sig" > "$state/.seen-pause-recheck_status"
|
|
2236
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2237
|
+
pane_hash=$(hash_text "idle awaiting external")
|
|
2238
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2239
|
+
printf '%s' "$pane_hash" > "$state/.stale-$key"
|
|
2240
|
+
printf '1\n' > "$state/.count-$key"
|
|
2241
|
+
: > "$state/.paused-$key"
|
|
2242
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
2243
|
+
|
|
2244
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2245
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2246
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2247
|
+
pid=$!
|
|
2248
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2249
|
+
reap "$pid"; fail "an active run behind a declared pause surfaced instead of resuming wedge tracking: $(cat "$out")"
|
|
2250
|
+
fi
|
|
2251
|
+
[ ! -e "$state/.paused-$key" ] || { reap "$pid"; fail "authoritative active run retained paused mode"; }
|
|
2252
|
+
[ -s "$state/.stale-since-$key" ] || { reap "$pid"; fail "authoritative active run did not resume wedge tracking"; }
|
|
2253
|
+
reap "$pid"
|
|
2254
|
+
unset FM_FAKE_CREW_STATE
|
|
2255
|
+
pass "a declared pause is periodically rechecked against authoritative active-run state"
|
|
2256
|
+
}
|
|
2257
|
+
|
|
2258
|
+
test_paused_authoritative_working_preserves_wedge_timer() {
|
|
2259
|
+
local dir state fakebin out capture_file window key pane_hash sig pid since
|
|
2260
|
+
dir=$(make_case paused-working-preserves-wedge-timer); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2261
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-paused-working"
|
|
2262
|
+
printf 'idle awaiting external\n' > "$capture_file"
|
|
2263
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/paused-working.meta"
|
|
2264
|
+
printf 'paused: awaiting the upstream release\n' > "$state/paused-working.status"
|
|
2265
|
+
sig=$(seen_sig "$state/paused-working.status"); printf '%s' "$sig" > "$state/.seen-paused-working_status"
|
|
2266
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2267
|
+
pane_hash=$(hash_text "idle awaiting external")
|
|
2268
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2269
|
+
printf '%s' "$pane_hash" > "$state/.stale-$key"
|
|
2270
|
+
printf '1\n' > "$state/.count-$key"
|
|
2271
|
+
: > "$state/.paused-$key"
|
|
2272
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
2273
|
+
|
|
2274
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2275
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2276
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2277
|
+
pid=$!
|
|
2278
|
+
wait_numeric_file "$state/.stale-since-$key" 30 || { reap "$pid"; fail "authoritative working state did not start wedge tracking"; }
|
|
2279
|
+
since=$(cat "$state/.stale-since-$key")
|
|
2280
|
+
sleep 2
|
|
2281
|
+
[ "$(cat "$state/.stale-since-$key" 2>/dev/null || true)" = "$since" ] \
|
|
2282
|
+
|| { reap "$pid"; fail "repeat authoritative working recheck reset the wedge timer"; }
|
|
2283
|
+
reap "$pid"
|
|
2284
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional authoritative-working stop"
|
|
2285
|
+
|
|
2286
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2287
|
+
: > "$out"
|
|
2288
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2289
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2290
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2291
|
+
pid=$!
|
|
2292
|
+
wait_for_exit "$pid" 100 || fail "authoritative working state did not wedge-escalate past the threshold"
|
|
2293
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "authoritative working wedge escalation omitted its reason"
|
|
2294
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "wedge timer remained after authoritative working escalation"
|
|
2295
|
+
unset FM_FAKE_CREW_STATE
|
|
2296
|
+
pass "a paused status overridden by authoritative working preserves its wedge timer and escalates"
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
# --- consecutive wedge escalations on the same pane demand deep inspection ----
|
|
2300
|
+
# Root cause of the PR #252 incident's ~20 minutes of unnoticed green: each
|
|
2301
|
+
# wedge escalation fires, gets classified as "still validating" one poll later
|
|
2302
|
+
# (the timer restarts, see wedge_timer_check), and repeats forever on a pane
|
|
2303
|
+
# that never changes. A single escalation reason looks identical every round,
|
|
2304
|
+
# so nothing in the payload itself signals "this has now happened N times in a
|
|
2305
|
+
# row" - that judgment call was left entirely to the supervisor noticing the
|
|
2306
|
+
# repetition on its own. This is the safety-net fix: past
|
|
2307
|
+
# FM_WEDGE_DEMAND_INSPECT_COUNT consecutive escalations on the SAME pane, the
|
|
2308
|
+
# wake reason itself carries a "demand-deep-inspection" marker.
|
|
2309
|
+
|
|
2310
|
+
test_wedge_escalation_marks_demand_deep_inspection_after_threshold() {
|
|
2311
|
+
local dir state fakebin out capture_file window key pane_hash sig pid n
|
|
2312
|
+
dir=$(make_case wedge-escalation); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2313
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
2314
|
+
window="test:fm-wedged"
|
|
2315
|
+
printf 'idle building output' > "$capture_file"
|
|
2316
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/wedged.meta"
|
|
2317
|
+
printf 'working: still monitoring ci\n' > "$state/wedged.status"
|
|
2318
|
+
sig=$(seen_sig "$state/wedged.status"); printf '%s' "$sig" > "$state/.seen-wedged_status"
|
|
2319
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2320
|
+
pane_hash=$(hash_text "idle building output")
|
|
2321
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2322
|
+
printf '1\n' > "$state/.count-$key"
|
|
2323
|
+
# The crew's pipeline is actively running: a static pane is normal (waiting on CI).
|
|
2324
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
2325
|
+
|
|
2326
|
+
# Priming round: first sighting of this stale hash classifies and absorbs it
|
|
2327
|
+
# (establishing .stale-$key and starting the wedge timer) without going
|
|
2328
|
+
# through wedge_timer_check at all - mirrors the existing wedge tests' Phase A.
|
|
2329
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2330
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2331
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2332
|
+
pid=$!
|
|
2333
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2334
|
+
reap "$pid"; fail "watcher exited on the priming round (should absorb): $(cat "$out")"
|
|
2335
|
+
fi
|
|
2336
|
+
reap "$pid"
|
|
2337
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional wedge priming stop"
|
|
2338
|
+
|
|
2339
|
+
n=1
|
|
2340
|
+
while [ "$n" -le 3 ]; do
|
|
2341
|
+
# Backdate the wedge timer past the threshold before each round, mirroring
|
|
2342
|
+
# the existing wedge-escalation tests' Phase B (the subsequent-sight timer
|
|
2343
|
+
# path does not re-read the crew state).
|
|
2344
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2345
|
+
: > "$out"
|
|
2346
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2347
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2348
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2349
|
+
pid=$!
|
|
2350
|
+
wait_for_exit "$pid" 100 || fail "watcher did not escalate on consecutive wedge round $n: $(cat "$out")"
|
|
2351
|
+
grep -F "escalation $n" "$out" >/dev/null || fail "round $n did not report escalation count $n: $(cat "$out")"
|
|
2352
|
+
if [ "$n" -lt 3 ]; then
|
|
2353
|
+
grep -F "demand-deep-inspection" "$out" >/dev/null && fail "round $n escalated to demand-deep-inspection before the threshold: $(cat "$out")"
|
|
2354
|
+
else
|
|
2355
|
+
grep -F "demand-deep-inspection" "$out" >/dev/null || fail "round $n (threshold) did not demand deep inspection: $(cat "$out")"
|
|
2356
|
+
fi
|
|
2357
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge wedge escalation round $n"
|
|
2358
|
+
n=$((n + 1))
|
|
2359
|
+
done
|
|
2360
|
+
[ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || echo 0)" = 3 ] || fail "escalation counter did not persist across consecutive rounds"
|
|
2361
|
+
unset FM_FAKE_CREW_STATE
|
|
2362
|
+
pass "consecutive wedge escalations on the same pane accumulate and demand deep inspection at the threshold"
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
test_wedge_escalation_resets_when_pane_becomes_active() {
|
|
2366
|
+
local dir state fakebin out capture_file window key pane_hash sig pid
|
|
2367
|
+
dir=$(make_case wedge-escalation-reset); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2368
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
2369
|
+
window="test:fm-wedged-reset"
|
|
2370
|
+
printf 'idle building output' > "$capture_file"
|
|
2371
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/wedged-reset.meta"
|
|
2372
|
+
printf 'working: still monitoring ci\n' > "$state/wedged-reset.status"
|
|
2373
|
+
sig=$(seen_sig "$state/wedged-reset.status"); printf '%s' "$sig" > "$state/.seen-wedged-reset_status"
|
|
2374
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2375
|
+
pane_hash=$(hash_text "idle building output")
|
|
2376
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2377
|
+
printf '1\n' > "$state/.count-$key"
|
|
2378
|
+
# Pre-seed one escalation as if a prior wedge round already fired.
|
|
2379
|
+
printf '1\n' > "$state/.wedge-escalations-$key"
|
|
2380
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
2381
|
+
|
|
2382
|
+
# The pane content changes (the crew is active again): the hash no longer
|
|
2383
|
+
# matches, so the watcher resets escalation bookkeeping instead of escalating.
|
|
2384
|
+
printf 'new output, crew active again' > "$capture_file"
|
|
2385
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2386
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2387
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2388
|
+
pid=$!
|
|
2389
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2390
|
+
reap "$pid"; fail "watcher exited on a fresh (changed) pane hash: $(cat "$out")"
|
|
2391
|
+
fi
|
|
2392
|
+
[ ! -e "$state/.wedge-escalations-$key" ] || fail "a changed pane hash did not reset the wedge-escalation counter"
|
|
2393
|
+
reap "$pid"
|
|
2394
|
+
unset FM_FAKE_CREW_STATE
|
|
2395
|
+
pass "a pane becoming active again resets the consecutive wedge-escalation counter"
|
|
2396
|
+
}
|
|
2397
|
+
|
|
2398
|
+
# --- busy pane duration bound: a completed-turn age gate on top of busy -----
|
|
2399
|
+
# 2026-07 hibit-agent-focus-nonsteal-r1 incident: a busy pane (herdr "working"
|
|
2400
|
+
# and/or the harness's rendered busy footer) is unconditional, unbounded proof
|
|
2401
|
+
# of liveness in every existing classifier, so a genuinely hung foreground tool
|
|
2402
|
+
# call behind a busy signature ran undetected for 25h. BUSY_TURN_MAX_SECS bounds
|
|
2403
|
+
# how long a busy pane may run with no completed turn (state/<id>.turn-ended, or
|
|
2404
|
+
# the task's spawn record before any turn completes); past the bound, panes
|
|
2405
|
+
# without a declared external wait or verified captain-held transfer take the
|
|
2406
|
+
# SAME wedge_timer_check already used for a provably-working non-busy stale.
|
|
2407
|
+
# Escalation reuses the identical stale reason, escalation counter, and
|
|
2408
|
+
# demand-deep-inspection marker - never an
|
|
2409
|
+
# automatic interrupt or restart.
|
|
2410
|
+
|
|
2411
|
+
test_busy_pane_below_turn_age_bound_is_absorbed() {
|
|
2412
|
+
local dir state fakebin out capture_file window key sig pid
|
|
2413
|
+
dir=$(make_case busy-below-turn-age); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2414
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-busy-fresh"
|
|
2415
|
+
printf 'Working... (12.3s)' > "$capture_file"
|
|
2416
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/busy-fresh.meta"
|
|
2417
|
+
record_pi_busy "$state" busy-fresh
|
|
2418
|
+
printf 'working: setup complete\n' > "$state/busy-fresh.status"
|
|
2419
|
+
sig=$(seen_sig "$state/busy-fresh.status"); printf '%s' "$sig" > "$state/.seen-busy-fresh_status"
|
|
2420
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2421
|
+
touch "$state/busy-fresh.turn-ended"
|
|
2422
|
+
prime_turnend_seen "$state/busy-fresh.turn-ended"
|
|
2423
|
+
|
|
2424
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2425
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=999 FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2426
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2427
|
+
pid=$!
|
|
2428
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2429
|
+
reap "$pid"; fail "a busy pane below the turn-age bound was escalated: $(cat "$out")"
|
|
2430
|
+
fi
|
|
2431
|
+
[ ! -s "$out" ] || fail "a busy pane below the turn-age bound printed a wake reason"
|
|
2432
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "a busy pane below the turn-age bound started a wedge timer"
|
|
2433
|
+
reap "$pid"
|
|
2434
|
+
pass "a busy worker below the turn-age bound remains working with no escalation"
|
|
2435
|
+
}
|
|
2436
|
+
|
|
2437
|
+
test_busy_pane_stable_hash_escalates_past_turn_age_bound() {
|
|
2438
|
+
local dir state fakebin out capture_file window key pane_hash sig pid
|
|
2439
|
+
dir=$(make_case busy-stable-hash-turn-age); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2440
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-busy-stable"
|
|
2441
|
+
printf 'Working...' > "$capture_file"
|
|
2442
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/busy-stable.meta"
|
|
2443
|
+
record_pi_busy "$state" busy-stable
|
|
2444
|
+
printf 'working: setup complete\n' > "$state/busy-stable.status"
|
|
2445
|
+
sig=$(seen_sig "$state/busy-stable.status"); printf '%s' "$sig" > "$state/.seen-busy-stable_status"
|
|
2446
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2447
|
+
pane_hash=$(hash_text "Working...")
|
|
2448
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2449
|
+
printf '1\n' > "$state/.count-$key"
|
|
2450
|
+
# No completed turn ever recorded for this task: age the spawn record itself.
|
|
2451
|
+
touch -t 200001010000 "$state/busy-stable.meta"
|
|
2452
|
+
|
|
2453
|
+
# Phase A: past the bound, the stable-hash busy pane is absorbed but starts
|
|
2454
|
+
# the wedge timer (mirrors the existing provably-working-stale Phase A/B).
|
|
2455
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2456
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2457
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2458
|
+
pid=$!
|
|
2459
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2460
|
+
reap "$pid"; fail "a stable-hash busy pane past the turn-age bound escalated before the wedge threshold: $(cat "$out")"
|
|
2461
|
+
fi
|
|
2462
|
+
[ -s "$state/.stale-since-$key" ] || fail "a stable-hash busy pane past the turn-age bound did not start a wedge timer"
|
|
2463
|
+
reap "$pid"
|
|
2464
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional stable-hash phase-A stop"
|
|
2465
|
+
|
|
2466
|
+
# Phase B: backdate the wedge timer past the threshold; the next poll escalates.
|
|
2467
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2468
|
+
: > "$out"
|
|
2469
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2470
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2471
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2472
|
+
pid=$!
|
|
2473
|
+
wait_for_exit "$pid" 100 || fail "a stable-hash busy pane did not wedge-escalate past the turn-age bound"
|
|
2474
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "busy turn-age escalation did not print the stale wake"
|
|
2475
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "busy turn-age escalation did not flag a possible wedge"
|
|
2476
|
+
pass "a busy worker with a stable pane hash still escalates once its completed-turn age reaches the bound"
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
# Regression fixture for the incident's actual masking condition: Pi's rendered
|
|
2480
|
+
# elapsed-time footer changes every poll, so the pane hash never repeats and the
|
|
2481
|
+
# watcher always takes the "new hash" branch, never the stable-hash one above.
|
|
2482
|
+
test_busy_pane_changing_hash_escalates_past_turn_age_bound() {
|
|
2483
|
+
local dir state fakebin out capture_file window key pid
|
|
2484
|
+
dir=$(make_case busy-changing-hash-turn-age); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2485
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-busy-ticking"
|
|
2486
|
+
printf 'Working... (3600.1s)' > "$capture_file"
|
|
2487
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/busy-ticking.meta"
|
|
2488
|
+
record_pi_busy "$state" busy-ticking
|
|
2489
|
+
printf 'working: setup complete\n' > "$state/busy-ticking.status"
|
|
2490
|
+
sig=$(seen_sig "$state/busy-ticking.status"); printf '%s' "$sig" > "$state/.seen-busy-ticking_status"
|
|
2491
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2492
|
+
touch -t 200001010000 "$state/busy-ticking.meta"
|
|
2493
|
+
# No pre-seeded .hash-<key>: with a real ticking elapsed footer, every poll
|
|
2494
|
+
# lands here (h != prev) - the reproduction's actual masking condition.
|
|
2495
|
+
|
|
2496
|
+
# Phase A: first sight past the bound absorbs and starts the wedge timer,
|
|
2497
|
+
# without ever needing the "genuinely stale" hash-match path.
|
|
2498
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2499
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2500
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2501
|
+
pid=$!
|
|
2502
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2503
|
+
reap "$pid"; fail "a changing-hash busy pane past the turn-age bound escalated before the wedge threshold: $(cat "$out")"
|
|
2504
|
+
fi
|
|
2505
|
+
[ -s "$state/.stale-since-$key" ] || fail "a changing-hash busy pane past the turn-age bound did not start a wedge timer"
|
|
2506
|
+
reap "$pid"
|
|
2507
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional changing-hash phase-A stop"
|
|
2508
|
+
|
|
2509
|
+
# Phase B: another tick (still a fresh, never-before-seen hash) plus a
|
|
2510
|
+
# backdated wedge timer escalates exactly as the stable-hash case does.
|
|
2511
|
+
printf 'Working... (3601.2s)' > "$capture_file"
|
|
2512
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2513
|
+
: > "$out"
|
|
2514
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2515
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2516
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2517
|
+
pid=$!
|
|
2518
|
+
wait_for_exit "$pid" 100 || fail "a changing-hash busy pane did not wedge-escalate past the turn-age bound"
|
|
2519
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "busy turn-age escalation (changing hash) did not print the stale wake"
|
|
2520
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "busy turn-age escalation (changing hash) did not flag a possible wedge"
|
|
2521
|
+
pass "a busy worker whose pane hash changes every poll still escalates once its completed-turn age reaches the bound"
|
|
2522
|
+
}
|
|
2523
|
+
|
|
2524
|
+
test_busy_pane_turn_end_touch_resets_age() {
|
|
2525
|
+
local dir state fakebin out capture_file window key pane_hash sig pid
|
|
2526
|
+
dir=$(make_case busy-turn-end-resets-age); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2527
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-busy-reset"
|
|
2528
|
+
printf 'Working...' > "$capture_file"
|
|
2529
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/busy-reset.meta"
|
|
2530
|
+
record_pi_busy "$state" busy-reset
|
|
2531
|
+
printf 'working: setup complete\n' > "$state/busy-reset.status"
|
|
2532
|
+
sig=$(seen_sig "$state/busy-reset.status"); printf '%s' "$sig" > "$state/.seen-busy-reset_status"
|
|
2533
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2534
|
+
pane_hash=$(hash_text "Working...")
|
|
2535
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2536
|
+
printf '1\n' > "$state/.count-$key"
|
|
2537
|
+
# A wedge is already mid-escalation, as if several over-age polls already ran.
|
|
2538
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2539
|
+
printf '1\n' > "$state/.wedge-escalations-$key"
|
|
2540
|
+
# The worker's most recent turn just completed: touching turn-ended resets age.
|
|
2541
|
+
touch "$state/busy-reset.turn-ended"
|
|
2542
|
+
prime_turnend_seen "$state/busy-reset.turn-ended"
|
|
2543
|
+
|
|
2544
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2545
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=3600 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2546
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2547
|
+
pid=$!
|
|
2548
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2549
|
+
reap "$pid"; fail "a freshly completed turn on a busy pane was still escalated: $(cat "$out")"
|
|
2550
|
+
fi
|
|
2551
|
+
[ ! -s "$out" ] || fail "a freshly completed turn on a busy pane printed a wake reason"
|
|
2552
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "a freshly completed turn did not clear the wedge timer"
|
|
2553
|
+
[ ! -e "$state/.wedge-escalations-$key" ] || fail "a freshly completed turn did not clear the escalation counter"
|
|
2554
|
+
reap "$pid"
|
|
2555
|
+
pass "touching a busy worker's completed-turn marker resets the age and prevents an old-age escalation"
|
|
2556
|
+
}
|
|
2557
|
+
|
|
2558
|
+
test_busy_pane_repeated_escalation_reaches_demand_deep_inspection() {
|
|
2559
|
+
local dir state fakebin out capture_file window key pane_hash sig pid n
|
|
2560
|
+
dir=$(make_case busy-turn-age-demand-inspect); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2561
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-busy-demand-inspect"
|
|
2562
|
+
printf 'Working...' > "$capture_file"
|
|
2563
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/busy-demand.meta"
|
|
2564
|
+
record_pi_busy "$state" busy-demand
|
|
2565
|
+
printf 'working: setup complete\n' > "$state/busy-demand.status"
|
|
2566
|
+
sig=$(seen_sig "$state/busy-demand.status"); printf '%s' "$sig" > "$state/.seen-busy-demand_status"
|
|
2567
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2568
|
+
pane_hash=$(hash_text "Working...")
|
|
2569
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2570
|
+
printf '1\n' > "$state/.count-$key"
|
|
2571
|
+
touch -t 200001010000 "$state/busy-demand.turn-ended"
|
|
2572
|
+
prime_turnend_seen "$state/busy-demand.turn-ended"
|
|
2573
|
+
|
|
2574
|
+
# Priming round: first sighting past the turn-age bound absorbs and starts
|
|
2575
|
+
# the wedge timer, mirroring the existing provably-working wedge tests.
|
|
2576
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2577
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2578
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2579
|
+
pid=$!
|
|
2580
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2581
|
+
reap "$pid"; fail "priming round for busy turn-age escalation was not absorbed: $(cat "$out")"
|
|
2582
|
+
fi
|
|
2583
|
+
reap "$pid"
|
|
2584
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional busy-wedge priming stop"
|
|
2585
|
+
|
|
2586
|
+
n=1
|
|
2587
|
+
while [ "$n" -le 3 ]; do
|
|
2588
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2589
|
+
: > "$out"
|
|
2590
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2591
|
+
FM_STATE_OVERRIDE="$state" FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2592
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2593
|
+
pid=$!
|
|
2594
|
+
wait_for_exit "$pid" 100 || fail "busy turn-age escalation round $n did not escalate: $(cat "$out")"
|
|
2595
|
+
grep -F "escalation $n" "$out" >/dev/null || fail "busy turn-age round $n did not report escalation count $n: $(cat "$out")"
|
|
2596
|
+
if [ "$n" -lt 3 ]; then
|
|
2597
|
+
grep -F "demand-deep-inspection" "$out" >/dev/null && fail "busy turn-age round $n escalated to demand-deep-inspection before the threshold: $(cat "$out")"
|
|
2598
|
+
else
|
|
2599
|
+
grep -F "demand-deep-inspection" "$out" >/dev/null || fail "busy turn-age round $n (threshold) did not demand deep inspection: $(cat "$out")"
|
|
2600
|
+
fi
|
|
2601
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge busy turn-age escalation round $n"
|
|
2602
|
+
n=$((n + 1))
|
|
2603
|
+
done
|
|
2604
|
+
[ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || echo 0)" = 3 ] || fail "busy turn-age escalation counter did not persist across consecutive rounds"
|
|
2605
|
+
pass "repeated busy turn-age escalations reuse the existing escalation counter and demand deep inspection at the threshold"
|
|
2606
|
+
}
|
|
2607
|
+
|
|
2608
|
+
# --- declared pause + busy pane: the busy-turn bound must honor the declaration
|
|
2609
|
+
# A single foreground call can keep a declared external wait semantically busy
|
|
2610
|
+
# past the completed-turn bound, bypassing the ordinary stale-pause path.
|
|
2611
|
+
# This fixture pins all three halves of the contract: the declared pause is
|
|
2612
|
+
# absorbed instead of wedged (A), it is still rechecked on the long
|
|
2613
|
+
# PAUSE_RESURFACE_SECS cadence so a forgotten wait cannot rot invisibly (B), and
|
|
2614
|
+
# lifting the declaration on the SAME busy over-age pane restores the wedge
|
|
2615
|
+
# escalation, proving the discriminator is the worker's own declaration and not a
|
|
2616
|
+
# blanket silencing of the escalator (C).
|
|
2617
|
+
test_busy_declared_pause_is_rechecked_not_wedge_escalated() {
|
|
2618
|
+
local dir state fakebin out capture_file window key sig pid statusf back
|
|
2619
|
+
dir=$(make_case busy-declared-pause); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2620
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-review-scout"
|
|
2621
|
+
statusf="$state/review-scout.status"
|
|
2622
|
+
printf 'Working... (7200.4s) lavish-axi poll' > "$capture_file"
|
|
2623
|
+
printf 'window=%s\nkind=scout\nharness=pi\n' "$window" > "$state/review-scout.meta"
|
|
2624
|
+
record_pi_busy "$state" review-scout
|
|
2625
|
+
printf 'paused: hosting the Lavish review, awaiting captain feedback\n' > "$statusf"
|
|
2626
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-review-scout_status"
|
|
2627
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2628
|
+
# No completed turn for hours (the single blocking poll call): age the spawn
|
|
2629
|
+
# record itself, exactly as the never-completed-a-turn fixtures above do.
|
|
2630
|
+
touch -t 200001010000 "$state/review-scout.meta"
|
|
2631
|
+
# No pre-seeded .hash-<key>: a live harness footer ticks, so every poll lands
|
|
2632
|
+
# on the changed-hash branch - the review scout's real masking condition.
|
|
2633
|
+
|
|
2634
|
+
# Phase A: past the bound, with the wedge threshold set as low as it goes, the
|
|
2635
|
+
# declared pause is absorbed on the long cadence and never starts a wedge.
|
|
2636
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2637
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2638
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2639
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2640
|
+
FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2641
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2642
|
+
pid=$!
|
|
2643
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a declared pause on a busy review pane was escalated: $(cat "$out")"; }
|
|
2644
|
+
reap "$pid"
|
|
2645
|
+
[ ! -s "$out" ] || fail "a declared pause on a busy review pane printed a wake reason: $(cat "$out")"
|
|
2646
|
+
[ -e "$state/.paused-$key" ] || fail "the busy-turn bound did not apply the declared-pause cadence"
|
|
2647
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "a declared pause on a busy pane started the wedge timer"
|
|
2648
|
+
[ ! -e "$state/.wedge-escalations-$key" ] || fail "a declared pause on a busy pane incremented the escalation counter"
|
|
2649
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional declared-pause phase-A stop"
|
|
2650
|
+
|
|
2651
|
+
# Phase B: age the pause past the (now normal) long cadence and let the pane
|
|
2652
|
+
# settle on one stable hash, so the still-busy pane takes the repeat-hash
|
|
2653
|
+
# branch whose pause bookkeeping the bound must not wipe. It re-surfaces once
|
|
2654
|
+
# as a recheck, never as a wedge.
|
|
2655
|
+
back=$(( $(date +%s) - 500 ))
|
|
2656
|
+
if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf"
|
|
2657
|
+
else touch -m -d "@$back" "$statusf"; fi
|
|
2658
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-review-scout_status"
|
|
2659
|
+
printf '%s' "$(hash_text "$(cat "$capture_file")")" > "$state/.hash-$key"
|
|
2660
|
+
printf '1\n' > "$state/.count-$key"
|
|
2661
|
+
: > "$out"
|
|
2662
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2663
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2664
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2665
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=240 \
|
|
2666
|
+
FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2667
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2668
|
+
pid=$!
|
|
2669
|
+
wait_for_exit "$pid" 100 || { reap "$pid"; fail "a declared pause past the long cadence was never rechecked"; }
|
|
2670
|
+
grep -F "awaiting external" "$out" >/dev/null || fail "the recheck was not labeled a declared-pause recheck: $(cat "$out")"
|
|
2671
|
+
grep -F "possible wedge" "$out" >/dev/null && fail "a declared pause on a busy pane was mislabeled a possible wedge: $(cat "$out")"
|
|
2672
|
+
[ -e "$state/.paused-resurfaced-$key" ] || fail "the declared-pause re-surface throttle was cleared by the busy-turn bound"
|
|
2673
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "a declared-pause recheck used the wedge timer"
|
|
2674
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the declared-pause recheck"
|
|
2675
|
+
|
|
2676
|
+
# Phase C: the pause is lifted on the SAME busy, over-age pane. Nothing else
|
|
2677
|
+
# changes, so a still-absorbed pane here would mean the bound was silenced
|
|
2678
|
+
# rather than taught the declaration. It must wedge-escalate exactly as before.
|
|
2679
|
+
printf 'working: review closed, resuming the sweep\n' > "$statusf"
|
|
2680
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-review-scout_status"
|
|
2681
|
+
: > "$out"
|
|
2682
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2683
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2684
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2685
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=999 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2686
|
+
FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2687
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2688
|
+
pid=$!
|
|
2689
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "a lifted pause escalated before the wedge threshold: $(cat "$out")"; }
|
|
2690
|
+
reap "$pid"
|
|
2691
|
+
[ -s "$state/.stale-since-$key" ] || fail "a lifted pause did not restore the busy-turn wedge timer"
|
|
2692
|
+
[ ! -e "$state/.paused-$key" ] || fail "a lifted pause left stale declared-pause bookkeeping behind"
|
|
2693
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional lifted-pause priming stop"
|
|
2694
|
+
|
|
2695
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2696
|
+
: > "$out"
|
|
2697
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2698
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2699
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2700
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2701
|
+
FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2702
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2703
|
+
pid=$!
|
|
2704
|
+
wait_for_exit "$pid" 100 || { reap "$pid"; fail "a lifted pause on an over-age busy pane no longer wedge-escalates"; }
|
|
2705
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "the restored busy-turn escalation did not flag a possible wedge: $(cat "$out")"
|
|
2706
|
+
pass "a busy pane under a declared pause is rechecked on the long cadence, and lifting the pause restores the wedge escalation"
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
# --- declared pause + busy pane + AWAY MODE: the bound must hand off, not decorate
|
|
2710
|
+
# Away mode is daemon-owned: the watcher reverts to one-shot and lets the daemon
|
|
2711
|
+
# classify. The busy-turn bound used to be the one stale path that ignored that,
|
|
2712
|
+
# running the wedge timer under afk and handing the daemon a wake already decorated
|
|
2713
|
+
# as a possible wedge. That decoration outranks the daemon's own pause verdict, so a
|
|
2714
|
+
# crew that declared the wait itself was wedge-escalated once per
|
|
2715
|
+
# FM_STALE_ESCALATE_SECS for as long as the wait lasted, with the escalation count
|
|
2716
|
+
# climbing into demand-deep-inspection on a pane nobody needed to inspect.
|
|
2717
|
+
# Phase A pins the handoff: the plain window identity, no wedge timer, no escalation
|
|
2718
|
+
# counter, and no normal-mode pause bookkeeping (the daemon owns that in away mode).
|
|
2719
|
+
# Phase B re-arms on the same unchanged pane and pins the one-shot: a second wake
|
|
2720
|
+
# here is what the climbing ladder looked like. Phase C drives the discriminator
|
|
2721
|
+
# apart on the SAME afk, busy, over-age pane - lifting the declaration restores the
|
|
2722
|
+
# wedge escalation, so this is the worker's declaration being honored rather than
|
|
2723
|
+
# away mode silencing the escalator.
|
|
2724
|
+
test_afk_busy_declared_pause_hands_off_plain_stale() {
|
|
2725
|
+
local dir state fakebin out capture_file window key sig pid statusf
|
|
2726
|
+
dir=$(make_case afk-busy-declared-pause); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2727
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-afk-review-scout"
|
|
2728
|
+
statusf="$state/afk-review-scout.status"
|
|
2729
|
+
printf 'Working... (7200.4s) lavish-axi poll' > "$capture_file"
|
|
2730
|
+
printf 'window=%s\nkind=scout\nharness=pi\n' "$window" > "$state/afk-review-scout.meta"
|
|
2731
|
+
record_pi_busy "$state" afk-review-scout
|
|
2732
|
+
printf 'paused: hosting the Lavish review, awaiting captain feedback\n' > "$statusf"
|
|
2733
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-afk-review-scout_status"
|
|
2734
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2735
|
+
touch -t 200001010000 "$state/afk-review-scout.meta"
|
|
2736
|
+
date '+%s' > "$state/.afk"
|
|
2737
|
+
|
|
2738
|
+
# Phase A: past the bound, with the wedge threshold as low as it goes, the
|
|
2739
|
+
# declaration is handed to the daemon undecorated instead of being wedge-timed.
|
|
2740
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2741
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2742
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2743
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2744
|
+
FM_POLL=0.2 FM_SIGNAL_GRACE=1 \
|
|
2745
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2746
|
+
pid=$!
|
|
2747
|
+
wait_for_exit "$pid" 150 || { reap "$pid"; fail "the away-mode busy-turn bound never handed the declared pause to the daemon"; }
|
|
2748
|
+
grep -Fx "stale: $window" "$out" >/dev/null \
|
|
2749
|
+
|| fail "the away-mode busy-turn bound did not hand off the plain window identity: $(cat "$out")"
|
|
2750
|
+
grep -F "possible wedge" "$out" >/dev/null \
|
|
2751
|
+
&& fail "away mode decorated a declared pause as a possible wedge: $(cat "$out")"
|
|
2752
|
+
[ ! -e "$state/.stale-since-$key" ] \
|
|
2753
|
+
|| fail "the away-mode handoff started the wedge timer on a declared pause"
|
|
2754
|
+
[ ! -e "$state/.wedge-escalations-$key" ] \
|
|
2755
|
+
|| fail "the away-mode handoff incremented the wedge escalation count on a declared pause"
|
|
2756
|
+
[ ! -e "$state/.paused-$key" ] \
|
|
2757
|
+
|| fail "the away-mode handoff recorded normal-mode pause tracking instead of leaving it to the daemon"
|
|
2758
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the away-mode declared-pause handoff"
|
|
2759
|
+
|
|
2760
|
+
# Phase B: re-arm on the same unchanged pane. The bound has already handed this
|
|
2761
|
+
# stale hash off, so it must stay silent rather than re-waking the daemon - a
|
|
2762
|
+
# second wake here is the escalation ladder the wedge timer used to climb.
|
|
2763
|
+
: > "$out"
|
|
2764
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2765
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2766
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2767
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2768
|
+
FM_POLL=0.2 FM_SIGNAL_GRACE=1 \
|
|
2769
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2770
|
+
pid=$!
|
|
2771
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "the away-mode bound re-woke on an already-handed-off declared pause: $(cat "$out")"; }
|
|
2772
|
+
reap "$pid"
|
|
2773
|
+
[ ! -s "$out" ] || fail "the away-mode bound re-surfaced an already-handed-off declared pause: $(cat "$out")"
|
|
2774
|
+
[ ! -e "$state/.wedge-escalations-$key" ] \
|
|
2775
|
+
|| fail "re-arming on an unchanged declared pause started a wedge escalation ladder"
|
|
2776
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional away-mode re-arm stop"
|
|
2777
|
+
|
|
2778
|
+
# Phase C: lift the declaration on the SAME afk, busy, over-age pane. Nothing else
|
|
2779
|
+
# changes, so a wedge escalation here proves the declaration was the discriminator.
|
|
2780
|
+
printf 'working: resumed the review write-up\n' > "$statusf"
|
|
2781
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-afk-review-scout_status"
|
|
2782
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2783
|
+
: > "$out"
|
|
2784
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2785
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2786
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2787
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2788
|
+
FM_POLL=0.2 FM_SIGNAL_GRACE=1 \
|
|
2789
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2790
|
+
pid=$!
|
|
2791
|
+
wait_for_exit "$pid" 150 || { reap "$pid"; fail "a lifted pause on an away-mode over-age busy pane no longer wedge-escalates"; }
|
|
2792
|
+
grep -F "possible wedge" "$out" >/dev/null \
|
|
2793
|
+
|| fail "the restored away-mode busy-turn escalation did not flag a possible wedge: $(cat "$out")"
|
|
2794
|
+
pass "away mode hands a busy declared pause to the daemon as a plain stale, and lifting the declaration restores the wedge escalation"
|
|
2795
|
+
}
|
|
2796
|
+
|
|
2797
|
+
# --- declared pause + busy pane + AWAY MODE + a TICKING footer: one wake per declaration
|
|
2798
|
+
# The static-pane case above cannot tell a hash-keyed one-shot from a
|
|
2799
|
+
# declaration-keyed one, because its capture never changes between polls. The
|
|
2800
|
+
# incident pane's harness footer ticks on every capture, so a one-shot keyed on the
|
|
2801
|
+
# pane hash re-fires on every poll, and the daemon, which relaunches the watcher
|
|
2802
|
+
# after each handled wake, is woken in a loop for the whole declared wait. This
|
|
2803
|
+
# fixture's fake tmux renders a fresh footer on EVERY capture-pane and asserts that
|
|
2804
|
+
# divergence outright on every re-arm (.hash-<key> moves, .count-<key> never
|
|
2805
|
+
# climbs), so the one-wake assertion across five silent re-arms cannot pass
|
|
2806
|
+
# vacuously on a pane that happened to sit still. Round 1 also starts from an
|
|
2807
|
+
# undeclared wedge timer and escalation count, which the handoff must clear the
|
|
2808
|
+
# way the normal-mode absorber does, so lifting the declaration later starts the
|
|
2809
|
+
# wedge path from a fresh timer rather than resuming a stale count.
|
|
2810
|
+
test_afk_busy_declared_pause_ticking_pane_hands_off_once() {
|
|
2811
|
+
local dir state fakebin out drain_out window key sig pid statusf ticks round prev_hash cur_hash prev_ticks
|
|
2812
|
+
dir=$(make_case afk-busy-declared-pause-ticking); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2813
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; window="test:fm-afk-ticking-scout"
|
|
2814
|
+
statusf="$state/afk-ticking-scout.status"; ticks="$dir/ticks"
|
|
2815
|
+
cat > "$fakebin/tmux" <<'SH'
|
|
2816
|
+
#!/usr/bin/env bash
|
|
2817
|
+
set -u
|
|
2818
|
+
case "${1:-}" in
|
|
2819
|
+
list-windows)
|
|
2820
|
+
[ -n "${FM_FAKE_TMUX_WINDOW:-}" ] && printf '%s\n' "${FM_FAKE_TMUX_WINDOW#*:}"
|
|
2821
|
+
exit 0 ;;
|
|
2822
|
+
capture-pane)
|
|
2823
|
+
n=$(( $(cat "$FM_FAKE_TMUX_TICKS" 2>/dev/null || echo 0) + 1 ))
|
|
2824
|
+
echo "$n" > "$FM_FAKE_TMUX_TICKS"
|
|
2825
|
+
printf 'Working... (%d.%ds) lavish-axi poll' "$(( 7200 + n ))" "$(( n % 10 ))"
|
|
2826
|
+
exit 0 ;;
|
|
2827
|
+
display-message)
|
|
2828
|
+
case "$*" in
|
|
2829
|
+
*pane_current_command*) printf '%s\n' "${FM_FAKE_TMUX_CURRENT_COMMAND:-}"; exit 0 ;;
|
|
2830
|
+
esac ;;
|
|
2831
|
+
esac
|
|
2832
|
+
exit 1
|
|
2833
|
+
SH
|
|
2834
|
+
chmod +x "$fakebin/tmux"
|
|
2835
|
+
printf 'window=%s\nkind=scout\nharness=pi\n' "$window" > "$state/afk-ticking-scout.meta"
|
|
2836
|
+
record_pi_busy "$state" afk-ticking-scout
|
|
2837
|
+
printf 'paused: hosting the Lavish review, awaiting captain feedback\n' > "$statusf"
|
|
2838
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-afk-ticking-scout_status"
|
|
2839
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2840
|
+
touch -t 200001010000 "$state/afk-ticking-scout.meta"
|
|
2841
|
+
date '+%s' > "$state/.afk"
|
|
2842
|
+
# An undeclared busy phase already ran the wedge timer and escalated twice
|
|
2843
|
+
# before the crew declared the wait.
|
|
2844
|
+
echo $(( $(date +%s) - 500 )) > "$state/.stale-since-$key"
|
|
2845
|
+
printf '2\n' > "$state/.wedge-escalations-$key"
|
|
2846
|
+
date +%s > "$state/.writing-since-$key"
|
|
2847
|
+
|
|
2848
|
+
# Round 1: the declaration is handed off once, undecorated, and the undeclared
|
|
2849
|
+
# phase's wedge bookkeeping is cleared with it.
|
|
2850
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_TICKS="$ticks" \
|
|
2851
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2852
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2853
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2854
|
+
FM_POLL=0.2 FM_SIGNAL_GRACE=1 \
|
|
2855
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2856
|
+
pid=$!
|
|
2857
|
+
wait_for_exit "$pid" 150 || { reap "$pid"; fail "the away-mode busy-turn bound never handed a ticking declared pause to the daemon"; }
|
|
2858
|
+
grep -Fx "stale: $window" "$out" >/dev/null \
|
|
2859
|
+
|| fail "the away-mode busy-turn bound did not hand off the plain window identity for a ticking pane: $(cat "$out")"
|
|
2860
|
+
grep -F "possible wedge" "$out" >/dev/null \
|
|
2861
|
+
&& fail "away mode decorated a ticking declared pause as a possible wedge: $(cat "$out")"
|
|
2862
|
+
[ ! -e "$state/.stale-since-$key" ] \
|
|
2863
|
+
|| fail "the away-mode handoff left the undeclared phase's wedge timer in place"
|
|
2864
|
+
[ ! -e "$state/.wedge-escalations-$key" ] \
|
|
2865
|
+
|| fail "the away-mode handoff left the undeclared phase's escalation count in place"
|
|
2866
|
+
[ ! -e "$state/.writing-since-$key" ] \
|
|
2867
|
+
|| fail "the away-mode handoff left the undeclared phase's write-deferral chain in place"
|
|
2868
|
+
[ ! -e "$state/.paused-$key" ] \
|
|
2869
|
+
|| fail "the away-mode handoff recorded normal-mode pause tracking on a ticking pane"
|
|
2870
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the ticking declared-pause handoff"
|
|
2871
|
+
|
|
2872
|
+
# Rounds 2-6: five consecutive re-arms on the same standing declaration. Every
|
|
2873
|
+
# capture renders a new footer, so every poll lands on the changed-hash branch -
|
|
2874
|
+
# the exact shape a hash-keyed one-shot re-fires on. Each round proves the pane
|
|
2875
|
+
# really moved before it asserts silence, so the case cannot go vacuous.
|
|
2876
|
+
round=2
|
|
2877
|
+
while [ "$round" -le 6 ]; do
|
|
2878
|
+
prev_hash=$(cat "$state/.hash-$key" 2>/dev/null || true)
|
|
2879
|
+
prev_ticks=$(cat "$ticks" 2>/dev/null || echo 0)
|
|
2880
|
+
: > "$out"
|
|
2881
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_TICKS="$ticks" \
|
|
2882
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
2883
|
+
FM_FAKE_CREW_STATE='state: working · source: pane · harness busy (pi-ext)' \
|
|
2884
|
+
FM_BUSY_TURN_MAX_SECS=1 FM_STALE_ESCALATE_SECS=1 FM_PAUSE_RESURFACE_SECS=999 \
|
|
2885
|
+
FM_POLL=0.2 FM_SIGNAL_GRACE=1 \
|
|
2886
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2887
|
+
pid=$!
|
|
2888
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "re-arm $round on a ticking declared pause re-woke the daemon: $(cat "$out")"; }
|
|
2889
|
+
reap "$pid"
|
|
2890
|
+
cur_hash=$(cat "$state/.hash-$key" 2>/dev/null || true)
|
|
2891
|
+
[ "$(cat "$ticks" 2>/dev/null || echo 0)" -gt "$prev_ticks" ] \
|
|
2892
|
+
|| fail "re-arm $round never captured the pane, so its silence proves nothing"
|
|
2893
|
+
[ -n "$cur_hash" ] && [ "$cur_hash" != "$prev_hash" ] \
|
|
2894
|
+
|| fail "re-arm $round saw the same pane hash as the round before, so it cannot tell a hash-keyed one-shot from a declaration-keyed one"
|
|
2895
|
+
[ "$(cat "$state/.count-$key" 2>/dev/null || echo missing)" = 0 ] \
|
|
2896
|
+
|| fail "re-arm $round settled on a stable hash instead of ticking on every poll"
|
|
2897
|
+
[ ! -s "$out" ] || fail "re-arm $round re-surfaced a standing declared pause on a ticking pane: $(cat "$out")"
|
|
2898
|
+
[ ! -e "$state/.stale-since-$key" ] \
|
|
2899
|
+
|| fail "re-arm $round started the wedge timer on a standing declared pause"
|
|
2900
|
+
[ ! -e "$state/.wedge-escalations-$key" ] \
|
|
2901
|
+
|| fail "re-arm $round climbed the wedge escalation ladder on a standing declared pause"
|
|
2902
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional re-arm $round stop"
|
|
2903
|
+
round=$((round + 1))
|
|
2904
|
+
done
|
|
2905
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || true
|
|
2906
|
+
grep "$(printf '\tstale\t')" "$drain_out" >/dev/null \
|
|
2907
|
+
&& fail "the silent re-arms still queued a stale row for the standing declaration: $(cat "$drain_out")"
|
|
2908
|
+
pass "away mode wakes the daemon once per declaration for a busy pane whose footer ticks on every capture"
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
# Behavioral proof that the production default (no FM_BUSY_TURN_MAX_SECS override
|
|
2912
|
+
# anywhere in this env) is 3600s: a completed turn 5 minutes old must not start a
|
|
2913
|
+
# wedge timer, while one 66 minutes old must - bracketing the default around 3600
|
|
2914
|
+
# without waiting a literal hour.
|
|
2915
|
+
test_busy_pane_default_turn_age_bound_is_3600s() {
|
|
2916
|
+
local dir state fakebin out capture_file window key pane_hash sig pid
|
|
2917
|
+
dir=$(make_case busy-default-turn-age); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2918
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"; window="test:fm-busy-default"
|
|
2919
|
+
printf 'Working...' > "$capture_file"
|
|
2920
|
+
printf 'window=%s\nkind=ship\nharness=pi\n' "$window" > "$state/busy-default.meta"
|
|
2921
|
+
record_pi_busy "$state" busy-default
|
|
2922
|
+
printf 'working: setup complete\n' > "$state/busy-default.status"
|
|
2923
|
+
sig=$(seen_sig "$state/busy-default.status"); printf '%s' "$sig" > "$state/.seen-busy-default_status"
|
|
2924
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2925
|
+
pane_hash=$(hash_text "Working...")
|
|
2926
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2927
|
+
printf '1\n' > "$state/.count-$key"
|
|
2928
|
+
|
|
2929
|
+
set_mtime $(( $(date +%s) - 300 )) "$state/busy-default.turn-ended"
|
|
2930
|
+
prime_turnend_seen "$state/busy-default.turn-ended"
|
|
2931
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2932
|
+
FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2933
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2934
|
+
pid=$!
|
|
2935
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2936
|
+
reap "$pid"; fail "a 5-minute-old completed turn tripped the default busy-turn-age bound: $(cat "$out")"
|
|
2937
|
+
fi
|
|
2938
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "a 5-minute-old completed turn started a wedge timer under the default bound"
|
|
2939
|
+
reap "$pid"
|
|
2940
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional five-minute-bound stop"
|
|
2941
|
+
|
|
2942
|
+
set_mtime $(( $(date +%s) - 4000 )) "$state/busy-default.turn-ended"
|
|
2943
|
+
prime_turnend_seen "$state/busy-default.turn-ended"
|
|
2944
|
+
: > "$out"
|
|
2945
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2946
|
+
FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2947
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2948
|
+
pid=$!
|
|
2949
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
2950
|
+
reap "$pid"; fail "a 66-minute-old completed turn escalated before the wedge threshold under the default bound: $(cat "$out")"
|
|
2951
|
+
fi
|
|
2952
|
+
[ -s "$state/.stale-since-$key" ] || fail "a 66-minute-old completed turn did not start a wedge timer under the default bound (default is not 3600s)"
|
|
2953
|
+
reap "$pid"
|
|
2954
|
+
pass "the production default busy-turn-age bound is 3600s (5min under does not wedge, 66min over does)"
|
|
2955
|
+
}
|
|
2956
|
+
|
|
2957
|
+
test_nonterminal_stale_repairs_missing_or_corrupt_timer() {
|
|
2958
|
+
local dir state fakebin out capture_file window key pane_hash sig pid since
|
|
2959
|
+
dir=$(make_case nonterminal-stale-timer-repair); state="$dir/state"; fakebin="$dir/fakebin"
|
|
2960
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
2961
|
+
window="test:fm-quiet-timer"
|
|
2962
|
+
printf 'idle building output' > "$capture_file"
|
|
2963
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/quiet-timer.meta"
|
|
2964
|
+
printf 'working: still compiling\n' > "$state/quiet-timer.status"
|
|
2965
|
+
sig=$(seen_sig "$state/quiet-timer.status"); printf '%s' "$sig" > "$state/.seen-quiet-timer_status"
|
|
2966
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
2967
|
+
pane_hash=$(hash_text "idle building output")
|
|
2968
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
2969
|
+
printf '1\n' > "$state/.count-$key"
|
|
2970
|
+
printf '%s' "$pane_hash" > "$state/.stale-$key"
|
|
2971
|
+
|
|
2972
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2973
|
+
FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2974
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2975
|
+
pid=$!
|
|
2976
|
+
wait_numeric_file "$state/.stale-since-$key" 30 || { reap "$pid"; fail "matching stale suppressor with missing timer did not initialize stale-since"; }
|
|
2977
|
+
if ! kill -0 "$pid" 2>/dev/null; then
|
|
2978
|
+
wait "$pid" 2>/dev/null || true
|
|
2979
|
+
fail "watcher exited while repairing a missing stale-since timer: $(cat "$out")"
|
|
2980
|
+
fi
|
|
2981
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "missing stale-since repair enqueued a wake"; }
|
|
2982
|
+
reap "$pid"
|
|
2983
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional missing-timer repair stop"
|
|
2984
|
+
|
|
2985
|
+
printf 'corrupt\n' > "$state/.stale-since-$key"
|
|
2986
|
+
: > "$out"
|
|
2987
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
2988
|
+
FM_STATE_OVERRIDE="$state" FM_STALE_ESCALATE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
2989
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
2990
|
+
pid=$!
|
|
2991
|
+
wait_numeric_file "$state/.stale-since-$key" 30 || { reap "$pid"; fail "matching stale suppressor with corrupt timer did not repair stale-since"; }
|
|
2992
|
+
since=$(cat "$state/.stale-since-$key" 2>/dev/null || true)
|
|
2993
|
+
[ "$since" != "corrupt" ] || { reap "$pid"; fail "corrupt stale-since value was left in place"; }
|
|
2994
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "corrupt stale-since repair enqueued a wake"; }
|
|
2995
|
+
reap "$pid"
|
|
2996
|
+
pass "matching non-terminal stale suppressors repair missing or corrupt stale-since timers"
|
|
2997
|
+
}
|
|
2998
|
+
|
|
2999
|
+
# --- quiet pane, worktree still being written: deferred, never wedge-escalated -
|
|
3000
|
+
# The live 2026-08-14 case: one crew produced eight consecutive possible-wedge
|
|
3001
|
+
# escalations in an afternoon, three of them demanding deep inspection, while it
|
|
3002
|
+
# was demonstrably writing source, then tests, then documentation. The detector's
|
|
3003
|
+
# two inputs (pane quietness, run step) cannot see that, so the pane looks frozen.
|
|
3004
|
+
# Both halves of the contract are asserted on the SAME fixture, because the whole
|
|
3005
|
+
# point is that only the worktree evidence differs: writing defers, silent
|
|
3006
|
+
# escalates on the unchanged schedule.
|
|
3007
|
+
# Every wait below is the file's standard one (wait_poll_cycle for an absorbing
|
|
3008
|
+
# watcher, a 100-tick wait_for_exit for an escalating one), because the poll these
|
|
3009
|
+
# tests assert on is the ONE poll that spawns the bounded worktree walk: on a
|
|
3010
|
+
# loaded runner it outlives a fixed liveness budget, and a round reaped before it
|
|
3011
|
+
# finished reports a lost deferral instead of the deferral under test.
|
|
3012
|
+
test_wedge_escalation_deferred_while_worktree_is_written() {
|
|
3013
|
+
local dir state fakebin out drain_out capture_file window key pane_hash sig pid wt back
|
|
3014
|
+
dir=$(make_case wedge-worktree-writes); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3015
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
3016
|
+
window="test:fm-writing"; wt="$dir/wt"
|
|
3017
|
+
mkdir -p "$wt/src"
|
|
3018
|
+
printf 'idle building output' > "$capture_file"
|
|
3019
|
+
printf 'window=%s\nkind=ship\nworktree=%s\n' "$window" "$wt" > "$state/writing.meta"
|
|
3020
|
+
printf 'working: implementing\n' > "$state/writing.status"
|
|
3021
|
+
sig=$(seen_sig "$state/writing.status"); printf '%s' "$sig" > "$state/.seen-writing_status"
|
|
3022
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
3023
|
+
pane_hash=$(hash_text "idle building output")
|
|
3024
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
3025
|
+
printf '1\n' > "$state/.count-$key"
|
|
3026
|
+
# Already-classified hash with an idle window that opened 500s ago, so the very
|
|
3027
|
+
# first stale poll lands straight on the at-threshold wedge branch (this repeat
|
|
3028
|
+
# path never re-reads crew state, so the worktree evidence is the only input
|
|
3029
|
+
# that can change the outcome).
|
|
3030
|
+
printf '%s' "$pane_hash" > "$state/.stale-$key"
|
|
3031
|
+
back=$(( $(date +%s) - 500 ))
|
|
3032
|
+
echo "$back" > "$state/.stale-since-$key"
|
|
3033
|
+
set_mtime "$back" "$state/.stale-since-$key"
|
|
3034
|
+
|
|
3035
|
+
# Phase A: the crew wrote a file after the idle window opened. Deferred.
|
|
3036
|
+
printf 'int main(void) { return 0; }\n' > "$wt/src/main.c"
|
|
3037
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3038
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \
|
|
3039
|
+
FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3040
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3041
|
+
pid=$!
|
|
3042
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
3043
|
+
reap "$pid"; fail "watcher wedge-escalated a quiet pane whose worktree was being written: $(cat "$out")"
|
|
3044
|
+
fi
|
|
3045
|
+
[ ! -s "$out" ] || { reap "$pid"; fail "a written-worktree deferral printed a wake reason: $(cat "$out")"; }
|
|
3046
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "a written-worktree deferral enqueued a wake"; }
|
|
3047
|
+
[ -e "$state/.writing-since-$key" ] || { reap "$pid"; fail "the write-deferral chain marker was not recorded"; }
|
|
3048
|
+
[ ! -e "$state/.wedge-escalations-$key" ] || { reap "$pid"; fail "a deferral advanced the wedge escalation counter"; }
|
|
3049
|
+
[ "$(cat "$state/.stale-since-$key" 2>/dev/null || echo 0)" -gt "$back" ] \
|
|
3050
|
+
|| { reap "$pid"; fail "a deferral did not restart the idle timer, so the next window cannot re-probe"; }
|
|
3051
|
+
reap "$pid"
|
|
3052
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional phase-A watcher stop"
|
|
3053
|
+
|
|
3054
|
+
# Phase B: same fixture, same quiet pane, but nothing written during this idle
|
|
3055
|
+
# window (the crew really is stalled). The unchanged schedule must still fire.
|
|
3056
|
+
set_mtime "$(( $(date +%s) - 900 ))" "$wt/src/main.c"
|
|
3057
|
+
echo "$back" > "$state/.stale-since-$key"
|
|
3058
|
+
set_mtime "$back" "$state/.stale-since-$key"
|
|
3059
|
+
: > "$out"
|
|
3060
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3061
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \
|
|
3062
|
+
FM_PAUSE_RESURFACE_SECS=999 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3063
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3064
|
+
pid=$!
|
|
3065
|
+
wait_for_exit "$pid" 100 || fail "a stalled crew that wrote nothing did not wedge-escalate on the existing schedule"
|
|
3066
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "the stalled-crew escalation did not print a stale wake"
|
|
3067
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "the stalled-crew escalation did not flag a possible wedge"
|
|
3068
|
+
[ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || true)" = 1 ] || fail "the stalled-crew escalation was not counted"
|
|
3069
|
+
[ ! -e "$state/.stale-since-$key" ] || fail "the idle timer was not cleared after a real escalation"
|
|
3070
|
+
[ ! -e "$state/.writing-since-$key" ] || fail "the write-deferral chain outlived a real escalation"
|
|
3071
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the stalled-crew escalation failed"
|
|
3072
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the stalled-crew escalation was not queued"
|
|
3073
|
+
pass "a quiet pane writing its own worktree is deferred, while one writing nothing still wedge-escalates on the unchanged schedule"
|
|
3074
|
+
}
|
|
3075
|
+
|
|
3076
|
+
# A deferral is not silence. A worktree can churn without real progress (a
|
|
3077
|
+
# rewritten log, a build touching the same file), so the whole deferral chain ages
|
|
3078
|
+
# and re-surfaces once per PAUSE_RESURFACE_SECS - the same bounded cadence a
|
|
3079
|
+
# declared pause uses - labeled as a recheck rather than a wedge.
|
|
3080
|
+
test_write_deferral_resurfaces_on_the_bounded_cadence() {
|
|
3081
|
+
local dir state fakebin out drain_out capture_file window key pane_hash sig pid wt back
|
|
3082
|
+
dir=$(make_case wedge-worktree-resurface); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3083
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
3084
|
+
window="test:fm-churn"; wt="$dir/wt"
|
|
3085
|
+
mkdir -p "$wt/src"
|
|
3086
|
+
printf 'idle building output' > "$capture_file"
|
|
3087
|
+
printf 'window=%s\nkind=ship\nworktree=%s\n' "$window" "$wt" > "$state/churn.meta"
|
|
3088
|
+
printf 'working: implementing\n' > "$state/churn.status"
|
|
3089
|
+
sig=$(seen_sig "$state/churn.status"); printf '%s' "$sig" > "$state/.seen-churn_status"
|
|
3090
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
3091
|
+
pane_hash=$(hash_text "idle building output")
|
|
3092
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
3093
|
+
printf '1\n' > "$state/.count-$key"
|
|
3094
|
+
printf '%s' "$pane_hash" > "$state/.stale-$key"
|
|
3095
|
+
back=$(( $(date +%s) - 500 ))
|
|
3096
|
+
echo "$back" > "$state/.stale-since-$key"
|
|
3097
|
+
set_mtime "$back" "$state/.stale-since-$key"
|
|
3098
|
+
# This pane has been deferring on write evidence for 500s already.
|
|
3099
|
+
: > "$state/.writing-since-$key"
|
|
3100
|
+
set_mtime "$back" "$state/.writing-since-$key"
|
|
3101
|
+
printf 'churn\n' > "$wt/src/main.c"
|
|
3102
|
+
|
|
3103
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3104
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_STALE_ESCALATE_SECS=240 \
|
|
3105
|
+
FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3106
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3107
|
+
pid=$!
|
|
3108
|
+
wait_for_exit "$pid" 100 || fail "a long-running write deferral never re-surfaced on the bounded cadence"
|
|
3109
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "the write-deferral recheck did not print a stale wake"
|
|
3110
|
+
grep -F "writing its worktree" "$out" >/dev/null || fail "the write-deferral recheck was not labeled as such"
|
|
3111
|
+
grep -F "possible wedge" "$out" >/dev/null && fail "a write-deferral recheck was mislabeled a possible wedge"
|
|
3112
|
+
[ -e "$state/.writing-resurfaced-$key" ] || fail "the write-deferral re-surface throttle marker was not recorded"
|
|
3113
|
+
[ ! -e "$state/.wedge-escalations-$key" ] || fail "a write-deferral recheck advanced the wedge escalation counter"
|
|
3114
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the write-deferral recheck failed"
|
|
3115
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the write-deferral recheck was not queued"
|
|
3116
|
+
pass "a write deferral re-surfaces once on the bounded pause cadence, so a churning worktree cannot stay invisible"
|
|
3117
|
+
}
|
|
3118
|
+
|
|
3119
|
+
# The worktree recorded for a secondmate is a provisioned firstmate home, and that
|
|
3120
|
+
# home runs its OWN supervision inside itself: its watcher beacon, pane hashes and
|
|
3121
|
+
# heartbeats keep state/ churning whether or not the mate produced anything. Reading
|
|
3122
|
+
# that as crew progress would quietly relax the kind-agnostic busy-turn backstop from
|
|
3123
|
+
# the escalation cadence to the hourly recheck for work that produced nothing, so the
|
|
3124
|
+
# probe must report no evidence and the unchanged schedule must still fire.
|
|
3125
|
+
test_secondmate_home_supervision_churn_is_not_write_evidence() {
|
|
3126
|
+
local dir state fakebin out drain_out capture_file window key sig pid home back
|
|
3127
|
+
dir=$(make_case secondmate-home-churn); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3128
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
3129
|
+
window="test:fm-mate"; home="$dir/mate-home"
|
|
3130
|
+
mkdir -p "$home/state"
|
|
3131
|
+
printf 'sm-mate\n' > "$home/.fm-secondmate-home"
|
|
3132
|
+
printf 'Working... (12.3s)' > "$capture_file"
|
|
3133
|
+
printf 'window=%s\nkind=ship\nharness=pi\nworktree=%s\n' "$window" "$home" > "$state/mate.meta"
|
|
3134
|
+
record_pi_busy "$state" mate
|
|
3135
|
+
# An ordinary crew recording a provisioned mate home is the route that actually
|
|
3136
|
+
# reaches the probe: a kind=secondmate window of its own is triaged only under a
|
|
3137
|
+
# declared pause, and a declared pause takes the bounded recheck cadence instead of
|
|
3138
|
+
# the wedge timer. The home marker alone is what excludes the walk, so the exclusion
|
|
3139
|
+
# is what this asserts. A busy pane is bounded by its completed-turn age; no turn
|
|
3140
|
+
# ever completed here, so the spawn record itself is aged past the bound that routes
|
|
3141
|
+
# it into the wedge timer.
|
|
3142
|
+
printf 'working: implementing\n' > "$state/mate.status"
|
|
3143
|
+
sig=$(seen_sig "$state/mate.status"); printf '%s' "$sig" > "$state/.seen-mate_status"
|
|
3144
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
3145
|
+
set_mtime "$(( $(date +%s) - 4000 ))" "$state/mate.meta"
|
|
3146
|
+
back=$(( $(date +%s) - 500 ))
|
|
3147
|
+
echo "$back" > "$state/.stale-since-$key"
|
|
3148
|
+
set_mtime "$back" "$state/.stale-since-$key"
|
|
3149
|
+
# The only thing written since the idle window opened is the mate home's own
|
|
3150
|
+
# supervision bookkeeping.
|
|
3151
|
+
printf 'beat\n' > "$home/state/.last-watcher-beat"
|
|
3152
|
+
|
|
3153
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3154
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
3155
|
+
FM_STALE_ESCALATE_SECS=240 FM_BUSY_TURN_MAX_SECS=1 FM_PAUSE_RESURFACE_SECS=999 \
|
|
3156
|
+
FM_POLL=1 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3157
|
+
pid=$!
|
|
3158
|
+
wait_for_exit "$pid" 100 || fail "a mate home's own supervision churn deferred an escalation it must not defer"
|
|
3159
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "the mate-home escalation did not print a stale wake"
|
|
3160
|
+
grep -F "possible wedge" "$out" >/dev/null || fail "the mate-home escalation did not flag a possible wedge"
|
|
3161
|
+
[ ! -e "$state/.writing-since-$key" ] || fail "a mate's provisioned home was probed as if it were a code tree"
|
|
3162
|
+
[ "$(cat "$state/.wedge-escalations-$key" 2>/dev/null || true)" = 1 ] || fail "the mate escalation was not counted"
|
|
3163
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the mate escalation failed"
|
|
3164
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "$window" >/dev/null || fail "the mate escalation was not queued"
|
|
3165
|
+
pass "a secondmate's own home supervision churn is not crew write evidence, so a pane recording that home keeps the unchanged escalation schedule"
|
|
3166
|
+
}
|
|
3167
|
+
|
|
3168
|
+
# A write deferral is a bounded chain, not a permanent one: its .writing-since
|
|
3169
|
+
# marker ages the whole chain so a churning worktree still re-surfaces once per
|
|
3170
|
+
# PAUSE_RESURFACE_SECS. That only holds while the chain belongs to the CURRENT quiet
|
|
3171
|
+
# stretch, so every path that restarts the idle-window timer must drop it too. The
|
|
3172
|
+
# reachable case is a pane that deferred on write evidence and later has its timer
|
|
3173
|
+
# repaired: a long-finished chain would make the first deferral of the new window
|
|
3174
|
+
# re-surface immediately instead of after a fresh window.
|
|
3175
|
+
test_timer_repair_drops_a_finished_write_deferral_chain() {
|
|
3176
|
+
local dir state fakebin out capture_file window key pane_hash sig pid wt back
|
|
3177
|
+
dir=$(make_case wedge-write-chain-timer-repair); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3178
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
3179
|
+
window="test:fm-chain-repair"; wt="$dir/wt"
|
|
3180
|
+
mkdir -p "$wt/src"
|
|
3181
|
+
printf 'idle building output' > "$capture_file"
|
|
3182
|
+
printf 'window=%s\nkind=ship\nworktree=%s\n' "$window" "$wt" > "$state/chain-repair.meta"
|
|
3183
|
+
printf 'working: implementing\n' > "$state/chain-repair.status"
|
|
3184
|
+
sig=$(seen_sig "$state/chain-repair.status"); printf '%s' "$sig" > "$state/.seen-chain-repair_status"
|
|
3185
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
3186
|
+
pane_hash=$(hash_text "idle building output")
|
|
3187
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
3188
|
+
printf '1\n' > "$state/.count-$key"
|
|
3189
|
+
printf '%s' "$pane_hash" > "$state/.stale-$key"
|
|
3190
|
+
# A deferral chain left over from an earlier quiet stretch, already well past the
|
|
3191
|
+
# bounded re-surface window.
|
|
3192
|
+
back=$(( $(date +%s) - 5000 ))
|
|
3193
|
+
: > "$state/.writing-since-$key"
|
|
3194
|
+
set_mtime "$back" "$state/.writing-since-$key"
|
|
3195
|
+
# The idle-window timer is corrupt, so this poll repairs it and opens a NEW quiet
|
|
3196
|
+
# window without probing the worktree at all.
|
|
3197
|
+
printf 'corrupt\n' > "$state/.stale-since-$key"
|
|
3198
|
+
|
|
3199
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3200
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
3201
|
+
FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3202
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3203
|
+
pid=$!
|
|
3204
|
+
# Watcher startup performs bounded recovery scans before its first stale poll;
|
|
3205
|
+
# give this positive marker assertion the same loaded-runner budget as the
|
|
3206
|
+
# suite's other startup-sensitive waits instead of failing after only 3s.
|
|
3207
|
+
wait_numeric_file "$state/.stale-since-$key" 100 \
|
|
3208
|
+
|| { reap "$pid"; fail "the corrupt idle-window timer was not repaired"; }
|
|
3209
|
+
[ ! -e "$state/.writing-since-$key" ] \
|
|
3210
|
+
|| { reap "$pid"; fail "an idle-window timer repair kept a finished write-deferral chain"; }
|
|
3211
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "the idle-window timer repair enqueued a wake"; }
|
|
3212
|
+
reap "$pid"
|
|
3213
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional timer-repair watcher stop"
|
|
3214
|
+
|
|
3215
|
+
# The new quiet window now crosses the escalation threshold while the crew writes
|
|
3216
|
+
# its worktree. That deferral must get a FRESH re-surface window rather than
|
|
3217
|
+
# inheriting the finished chain's age.
|
|
3218
|
+
back=$(( $(date +%s) - 500 ))
|
|
3219
|
+
echo "$back" > "$state/.stale-since-$key"
|
|
3220
|
+
set_mtime "$back" "$state/.stale-since-$key"
|
|
3221
|
+
printf 'int main(void) { return 0; }\n' > "$wt/src/main.c"
|
|
3222
|
+
: > "$out"
|
|
3223
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3224
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
3225
|
+
FM_STALE_ESCALATE_SECS=240 FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3226
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3227
|
+
pid=$!
|
|
3228
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
3229
|
+
reap "$pid"
|
|
3230
|
+
fail "the first deferral of a new quiet window re-surfaced at once, so it inherited a finished chain: $(cat "$out")"
|
|
3231
|
+
fi
|
|
3232
|
+
[ ! -s "$out" ] || { reap "$pid"; fail "a fresh write deferral printed a wake reason: $(cat "$out")"; }
|
|
3233
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "a fresh write deferral enqueued a wake"; }
|
|
3234
|
+
[ -e "$state/.writing-since-$key" ] || { reap "$pid"; fail "the new deferral recorded no chain marker"; }
|
|
3235
|
+
[ ! -e "$state/.writing-resurfaced-$key" ] \
|
|
3236
|
+
|| { reap "$pid"; fail "a fresh write deferral spent its bounded re-surface on the first poll"; }
|
|
3237
|
+
reap "$pid"
|
|
3238
|
+
pass "an idle-window timer repair drops a finished write-deferral chain, so the next deferral gets a fresh re-surface window"
|
|
3239
|
+
}
|
|
3240
|
+
|
|
3241
|
+
# The same chain must not outlive either first-sight path through a captain-relevant
|
|
3242
|
+
# status line, because both also open a new idle window: the provably-working absorb
|
|
3243
|
+
# and the plain surface.
|
|
3244
|
+
test_terminal_first_sight_drops_a_finished_write_deferral_chain() {
|
|
3245
|
+
local dir state fakebin out capture_file window key pane_hash sig pid wt back
|
|
3246
|
+
dir=$(make_case wedge-write-chain-first-sight); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3247
|
+
out="$dir/watch.out"; capture_file="$dir/pane.txt"
|
|
3248
|
+
window="test:fm-chain-firstsight"; wt="$dir/wt"
|
|
3249
|
+
mkdir -p "$wt/src"
|
|
3250
|
+
printf 'no-mistakes axi run: validating...' > "$capture_file"
|
|
3251
|
+
printf 'window=%s\nkind=ship\nworktree=%s\n' "$window" "$wt" > "$state/chain-first.meta"
|
|
3252
|
+
printf 'done: implementation complete, ready to validate\n' > "$state/chain-first.status"
|
|
3253
|
+
sig=$(seen_sig "$state/chain-first.status"); printf '%s' "$sig" > "$state/.seen-chain-first_status"
|
|
3254
|
+
key=$(printf '%s' "$window" | tr ':/.' '___')
|
|
3255
|
+
pane_hash=$(hash_text "no-mistakes axi run: validating...")
|
|
3256
|
+
printf '%s' "$pane_hash" > "$state/.hash-$key"
|
|
3257
|
+
printf '1\n' > "$state/.count-$key"
|
|
3258
|
+
back=$(( $(date +%s) - 5000 ))
|
|
3259
|
+
: > "$state/.writing-since-$key"
|
|
3260
|
+
set_mtime "$back" "$state/.writing-since-$key"
|
|
3261
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
3262
|
+
|
|
3263
|
+
# First sight of this hash, absorbed because the active run outranks the stale
|
|
3264
|
+
# captain-relevant line. The absorb opens a new idle window, so the finished chain
|
|
3265
|
+
# must go with it.
|
|
3266
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3267
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
3268
|
+
FM_STALE_ESCALATE_SECS=999 FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3269
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3270
|
+
pid=$!
|
|
3271
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
3272
|
+
reap "$pid"; fail "the overridden terminal status was not absorbed on first sight: $(cat "$out")"
|
|
3273
|
+
fi
|
|
3274
|
+
[ "$(cat "$state/.stale-$key" 2>/dev/null || true)" = "$pane_hash" ] \
|
|
3275
|
+
|| { reap "$pid"; fail "the first-sight absorb did not advance the stale suppressor"; }
|
|
3276
|
+
[ ! -e "$state/.writing-since-$key" ] \
|
|
3277
|
+
|| { reap "$pid"; fail "the provably-working first-sight absorb kept a finished write-deferral chain"; }
|
|
3278
|
+
reap "$pid"
|
|
3279
|
+
ack_stopped_cycle "$state" || fail "could not acknowledge the intentional first-sight absorb stop"
|
|
3280
|
+
|
|
3281
|
+
# Same pane, first sight again, but nothing overrides the status line now, so it
|
|
3282
|
+
# surfaces. That path drops the idle-window timer, so it must drop the chain too.
|
|
3283
|
+
rm -f "$state/.stale-$key" "$state/.stale-since-$key"
|
|
3284
|
+
printf '1\n' > "$state/.count-$key"
|
|
3285
|
+
: > "$state/.writing-since-$key"
|
|
3286
|
+
set_mtime "$back" "$state/.writing-since-$key"
|
|
3287
|
+
FM_FAKE_CREW_STATE='state: unknown · source: none · no run, no busy pane'
|
|
3288
|
+
: > "$out"
|
|
3289
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3290
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" \
|
|
3291
|
+
FM_STALE_ESCALATE_SECS=999 FM_PAUSE_RESURFACE_SECS=240 FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3292
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3293
|
+
pid=$!
|
|
3294
|
+
wait_for_exit "$pid" 100 || fail "a first-sight captain-relevant status was not surfaced"
|
|
3295
|
+
grep -F "stale: $window" "$out" >/dev/null || fail "the first-sight surface did not print a stale wake"
|
|
3296
|
+
[ ! -e "$state/.writing-since-$key" ] \
|
|
3297
|
+
|| fail "the first-sight surface kept a finished write-deferral chain"
|
|
3298
|
+
unset FM_FAKE_CREW_STATE
|
|
3299
|
+
pass "both first-sight paths through a captain-relevant status drop a finished write-deferral chain with the idle window"
|
|
3300
|
+
}
|
|
3301
|
+
|
|
3302
|
+
# --- triage debug log stays size capped -------------------------------------
|
|
3303
|
+
|
|
3304
|
+
test_triage_log_size_cap_accepts_spaced_wc_counts() {
|
|
3305
|
+
local dir state fakebin out status_file pid lines i
|
|
3306
|
+
dir=$(make_case triage-log-spaced-wc); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out"
|
|
3307
|
+
i=1
|
|
3308
|
+
while [ "$i" -le 3000 ]; do
|
|
3309
|
+
printf 'old line %04d\n' "$i" >> "$state/.watch-triage.log"
|
|
3310
|
+
i=$((i + 1))
|
|
3311
|
+
done
|
|
3312
|
+
cat > "$fakebin/wc" <<'SH'
|
|
3313
|
+
#!/usr/bin/env bash
|
|
3314
|
+
set -u
|
|
3315
|
+
if [ "${1:-}" = "-c" ]; then
|
|
3316
|
+
printf ' 999999\n'
|
|
3317
|
+
exit 0
|
|
3318
|
+
fi
|
|
3319
|
+
exit 127
|
|
3320
|
+
SH
|
|
3321
|
+
chmod +x "$fakebin/wc"
|
|
3322
|
+
status_file="$state/task.status"
|
|
3323
|
+
printf 'working: compiling step 2\n' > "$status_file"
|
|
3324
|
+
# Provably working so the no-verb signal is absorbed (which is what writes the
|
|
3325
|
+
# triage log line under test).
|
|
3326
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
3327
|
+
PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3328
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 FM_WATCH_TRIAGE_LOG_MAX_BYTES=1 "$WATCH" > "$out" &
|
|
3329
|
+
pid=$!
|
|
3330
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
3331
|
+
reap "$pid"; fail "watcher exited for a benign signal while testing log capping: $(cat "$out")"
|
|
3332
|
+
fi
|
|
3333
|
+
i=0
|
|
3334
|
+
while [ "$i" -lt 30 ]; do
|
|
3335
|
+
lines=$(awk 'END { print NR + 0 }' "$state/.watch-triage.log")
|
|
3336
|
+
[ "$lines" -le 2000 ] && break
|
|
3337
|
+
sleep 0.1
|
|
3338
|
+
i=$((i + 1))
|
|
3339
|
+
done
|
|
3340
|
+
[ "$lines" -le 2000 ] || { reap "$pid"; fail "triage log was not capped when wc emitted a spaced byte count (lines=$lines)"; }
|
|
3341
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "benign signal enqueued a wake while testing log capping"; }
|
|
3342
|
+
reap "$pid"
|
|
3343
|
+
pass "triage log capping handles wc byte counts with leading spaces"
|
|
3344
|
+
}
|
|
3345
|
+
|
|
3346
|
+
# --- process-event delivery -------------------------------------------------
|
|
3347
|
+
# A durably captured process-event result publishes an ordinary `check` wake on
|
|
3348
|
+
# the durable queue. The watcher must deliver that queued wake proactively -
|
|
3349
|
+
# print an actionable reason and exit into the same rewake path every other
|
|
3350
|
+
# actionable wake uses - rather than leaving it to be found by a manual drain.
|
|
3351
|
+
|
|
3352
|
+
# Run the runner against a case home. FM_ROOT_OVERRIDE (exported by the shared
|
|
3353
|
+
# wake harness to keep the drain's tangle check inert) would otherwise point the
|
|
3354
|
+
# runner at a root with no installed adapters, and the claim root must stay
|
|
3355
|
+
# inside the case so nothing here can observe a real home's source ownership.
|
|
3356
|
+
pe_case() { # <dir> <command>...
|
|
3357
|
+
local dir=$1
|
|
3358
|
+
shift
|
|
3359
|
+
(unset FM_ROOT_OVERRIDE
|
|
3360
|
+
FM_PROCEVENT_CLAIM_ROOT="$dir/claims" FM_HOME="$dir" "$ROOT/bin/fm-procevent.sh" "$@")
|
|
3361
|
+
}
|
|
3362
|
+
|
|
3363
|
+
# Capture one real process-event result into <dir>'s home, then retire the
|
|
3364
|
+
# source so the fixture holds exactly the reported end state: one durably
|
|
3365
|
+
# captured, unhandled, queued result and no remaining poll work.
|
|
3366
|
+
seed_captured_procevent_result() { # <dir>
|
|
3367
|
+
local dir=$1 i=0
|
|
3368
|
+
pe_case "$dir" register lavish delivery-src -- \
|
|
3369
|
+
/bin/sh -c 'printf "session:\n file: /a.html\n status: waiting\n"' >/dev/null || return 1
|
|
3370
|
+
pe_case "$dir" reconcile >/dev/null || return 1
|
|
3371
|
+
while [ "$i" -lt 100 ]; do
|
|
3372
|
+
[ -s "$dir/state/.wake-queue" ] && break
|
|
3373
|
+
sleep 0.1
|
|
3374
|
+
i=$((i + 1))
|
|
3375
|
+
done
|
|
3376
|
+
pe_case "$dir" retire delivery-src >/dev/null || return 1
|
|
3377
|
+
[ -s "$dir/state/.wake-queue" ]
|
|
3378
|
+
}
|
|
3379
|
+
|
|
3380
|
+
# The watcher, scoped by FM_HOME rather than FM_STATE_OVERRIDE, so the
|
|
3381
|
+
# per-cycle reconcile it launches resolves the same home's state.
|
|
3382
|
+
procevent_watch_bg() { # <dir> <out>
|
|
3383
|
+
local dir=$1 out=$2
|
|
3384
|
+
PATH="$dir/fakebin:$PATH" FM_HOME="$dir" FM_PROCEVENT_CLAIM_ROOT="$dir/claims" \
|
|
3385
|
+
FM_CREW_STATE_BIN="$dir/fakebin/fm-crew-state.sh" \
|
|
3386
|
+
FM_POLL=0.2 FM_SIGNAL_GRACE=1 FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3387
|
+
}
|
|
3388
|
+
|
|
3389
|
+
test_procevent_captured_result_surfaces_proactively() {
|
|
3390
|
+
local dir state out drain_out pid beacon_age
|
|
3391
|
+
dir=$(make_case procevent-delivery); state="$dir/state"
|
|
3392
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
3393
|
+
seed_captured_procevent_result "$dir" || fail "the fixture captured no process-event result"
|
|
3394
|
+
grep -F "procevent lavish delivery-src 1" "$state/.wake-queue" >/dev/null \
|
|
3395
|
+
|| fail "the captured result was never published to the durable queue"
|
|
3396
|
+
|
|
3397
|
+
procevent_watch_bg "$dir" "$out"
|
|
3398
|
+
pid=$!
|
|
3399
|
+
wait_for_exit "$pid" 100 \
|
|
3400
|
+
|| fail "a healthy watcher never surfaced a durably captured process-event result: $(cat "$out")"
|
|
3401
|
+
grep -F "check:" "$out" >/dev/null \
|
|
3402
|
+
|| fail "the process-event wake was not reported as an actionable check: $(cat "$out")"
|
|
3403
|
+
grep -F "procevent:delivery-src:1" "$out" >/dev/null \
|
|
3404
|
+
|| fail "the actionable reason did not name the queued result: $(cat "$out")"
|
|
3405
|
+
beacon_age=$(FM_STATE_OVERRIDE="$state" bash -c \
|
|
3406
|
+
'. "$1/bin/fm-wake-lib.sh"; fm_path_age "$2"' _ "$ROOT" "$state/.last-watcher-beat")
|
|
3407
|
+
[ "$beacon_age" -lt 60 ] || fail "the surfacing watcher was not a healthy one (beacon age ${beacon_age}s)"
|
|
3408
|
+
|
|
3409
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the process-event wake failed"
|
|
3410
|
+
grep "$(printf '\tcheck\t')" "$drain_out" | grep -F "procevent lavish delivery-src 1" >/dev/null \
|
|
3411
|
+
|| fail "the process-event result was not queued for the drain that follows the wake"
|
|
3412
|
+
pass "a captured process-event result wakes a healthy watcher proactively, with no manual drain"
|
|
3413
|
+
}
|
|
3414
|
+
|
|
3415
|
+
test_procevent_unacknowledged_result_redrains_until_handled() {
|
|
3416
|
+
local dir state out replay_out replay_err pid before after sequence generation
|
|
3417
|
+
dir=$(make_case procevent-redrain); state="$dir/state"
|
|
3418
|
+
out="$dir/watch.out"; replay_out="$dir/replay.out"; replay_err="$dir/replay.err"
|
|
3419
|
+
seed_captured_procevent_result "$dir" || fail "the fixture captured no process-event result"
|
|
3420
|
+
|
|
3421
|
+
procevent_watch_bg "$dir" "$out"
|
|
3422
|
+
pid=$!
|
|
3423
|
+
wait_for_exit "$pid" 100 || fail "the first proactive wake never happened: $(cat "$out")"
|
|
3424
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>&1 || fail "drain after the first process-event wake failed"
|
|
3425
|
+
|
|
3426
|
+
# An interrupted handler leaves the captured result durable. The successor
|
|
3427
|
+
# must re-surface it through recovery, then its drain must print the same row.
|
|
3428
|
+
: > "$out"
|
|
3429
|
+
procevent_watch_bg "$dir" "$out"
|
|
3430
|
+
pid=$!
|
|
3431
|
+
wait_for_exit "$pid" 100 \
|
|
3432
|
+
|| fail "an unacknowledged process-event result was not re-surfaced on re-arm: $(cat "$out")"
|
|
3433
|
+
grep -F 'check: rearm-resurface' "$out" >/dev/null \
|
|
3434
|
+
|| fail "the successor did not report recovery for the unacknowledged result: $(cat "$out")"
|
|
3435
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$replay_out" 2> "$replay_err" \
|
|
3436
|
+
|| fail "the successor could not re-drain the unacknowledged process-event result"
|
|
3437
|
+
grep "$(printf '\tcheck\t')" "$replay_out" | grep -F 'procevent lavish delivery-src 1' >/dev/null \
|
|
3438
|
+
|| fail "the successor drain did not re-print the durable process-event row"
|
|
3439
|
+
|
|
3440
|
+
pe_case "$dir" handled delivery-src 1 >/dev/null || fail "could not acknowledge the captured result"
|
|
3441
|
+
sequence=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through \([0-9][0-9]*\) --recovery-generation [A-Za-z0-9._-][A-Za-z0-9._-]*$/\1/p' "$replay_err")
|
|
3442
|
+
generation=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through [0-9][0-9]* --recovery-generation \([A-Za-z0-9._-][A-Za-z0-9._-]*\)$/\1/p' "$replay_err")
|
|
3443
|
+
[ -n "$sequence" ] && [ -n "$generation" ] \
|
|
3444
|
+
|| fail "the replay drain omitted its post-handling acknowledgement boundary"
|
|
3445
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" --ack-through "$sequence" --recovery-generation "$generation" \
|
|
3446
|
+
|| fail "completed process-event handling could not acknowledge the replay"
|
|
3447
|
+
[ ! -s "$state/.wake-queue" ] || fail "acknowledged process-event replay remained durable"
|
|
3448
|
+
|
|
3449
|
+
before=$(awk 'END { print NR + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0)
|
|
3450
|
+
: > "$out"
|
|
3451
|
+
procevent_watch_bg "$dir" "$out"
|
|
3452
|
+
pid=$!
|
|
3453
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
3454
|
+
fail "a handled process-event result woke the watcher: $(cat "$out")"
|
|
3455
|
+
fi
|
|
3456
|
+
reap "$pid"
|
|
3457
|
+
after=$(awk 'END { print NR + 0 }' "$state/.wake-queue" 2>/dev/null || echo 0)
|
|
3458
|
+
[ "$after" = "$before" ] || fail "a handled result was announced again ($before -> $after queued records)"
|
|
3459
|
+
pass "an unacknowledged process-event result re-drains until handling is acknowledged"
|
|
3460
|
+
}
|
|
3461
|
+
|
|
3462
|
+
test_procevent_marker_keys_are_injective() {
|
|
3463
|
+
local dir state out pid marker_count
|
|
3464
|
+
dir=$(make_case procevent-marker-identity); state="$dir/state"; out="$dir/watch.out"
|
|
3465
|
+
append_wake "$state" check "procevent:a.b:1" "check: procevent fixture a.b 1"
|
|
3466
|
+
append_wake "$state" check "procevent:a_b:1" "check: procevent fixture a_b 1"
|
|
3467
|
+
procevent_watch_bg "$dir" "$out"
|
|
3468
|
+
pid=$!
|
|
3469
|
+
wait_for_exit "$pid" 100 || fail "colliding-looking process-event keys were not surfaced"
|
|
3470
|
+
grep -F "procevent:a.b:1" "$out" >/dev/null || fail "the dotted queue key was suppressed"
|
|
3471
|
+
grep -F "procevent:a_b:1" "$out" >/dev/null || fail "the underscored queue key was suppressed"
|
|
3472
|
+
marker_count=$(find "$state" -maxdepth 1 -name '.seen-procevent-*' -type f | awk 'END { print NR + 0 }')
|
|
3473
|
+
[ "$marker_count" = 2 ] || fail "distinct queue keys produced $marker_count seen markers"
|
|
3474
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>&1 || fail "marker identity fixture drain failed"
|
|
3475
|
+
pass "complete process-event queue keys map to distinct seen markers"
|
|
3476
|
+
}
|
|
3477
|
+
|
|
3478
|
+
install_marker_mv_fault() { # <dir>
|
|
3479
|
+
local dir=$1
|
|
3480
|
+
REAL_MV=$(command -v mv)
|
|
3481
|
+
export REAL_MV
|
|
3482
|
+
cat > "$dir/fakebin/mv" <<'SH'
|
|
3483
|
+
#!/usr/bin/env bash
|
|
3484
|
+
dest=${!#}
|
|
3485
|
+
case "$dest" in
|
|
3486
|
+
*/.seen-procevent-*)
|
|
3487
|
+
case "${FM_MARKER_MV_MODE:-}" in
|
|
3488
|
+
pause)
|
|
3489
|
+
printf '1\n' > "$FM_MARKER_MV_READY"
|
|
3490
|
+
while [ ! -e "$FM_MARKER_MV_RELEASE" ]; do sleep 0.02; done
|
|
3491
|
+
;;
|
|
3492
|
+
kill-before) kill -KILL "$PPID"; exit 1 ;;
|
|
3493
|
+
kill-after) "$REAL_MV" "$@" || exit; kill -KILL "$PPID"; exit 1 ;;
|
|
3494
|
+
fail) exit 1 ;;
|
|
3495
|
+
esac
|
|
3496
|
+
;;
|
|
3497
|
+
esac
|
|
3498
|
+
exec "$REAL_MV" "$@"
|
|
3499
|
+
SH
|
|
3500
|
+
chmod +x "$dir/fakebin/mv"
|
|
3501
|
+
}
|
|
3502
|
+
|
|
3503
|
+
test_procevent_surface_serializes_with_drain() {
|
|
3504
|
+
local dir state out drain_out ready release pid drain_pid
|
|
3505
|
+
dir=$(make_case procevent-drain-race); state="$dir/state"; out="$dir/watch.out"
|
|
3506
|
+
drain_out="$dir/drain.out"; ready="$dir/marker-ready"; release="$dir/marker-release"
|
|
3507
|
+
append_wake "$state" check "procevent:drain-race:1" "check: procevent fixture drain-race 1"
|
|
3508
|
+
install_marker_mv_fault "$dir"
|
|
3509
|
+
FM_MARKER_MV_MODE=pause FM_MARKER_MV_READY="$ready" FM_MARKER_MV_RELEASE="$release" \
|
|
3510
|
+
procevent_watch_bg "$dir" "$out"
|
|
3511
|
+
pid=$!
|
|
3512
|
+
wait_numeric_file "$ready" 100 || fail "the watcher never reached its marker commit boundary"
|
|
3513
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" &
|
|
3514
|
+
drain_pid=$!
|
|
3515
|
+
wait_live "$drain_pid" 10 || fail "a concurrent drain split the surfacing transition"
|
|
3516
|
+
[ -s "$state/.wake-queue" ] || fail "the concurrent drain consumed the record before marker commit"
|
|
3517
|
+
touch "$release"
|
|
3518
|
+
wait "$pid" || fail "the paused watcher did not finish surfacing"
|
|
3519
|
+
wait "$drain_pid" || fail "the concurrent drain failed after surfacing committed"
|
|
3520
|
+
grep -F "procevent:drain-race:1" "$drain_out" >/dev/null \
|
|
3521
|
+
|| fail "the serialized drain lost the process-event record"
|
|
3522
|
+
pass "queue revalidation, proactive output, and marker commit serialize with drain"
|
|
3523
|
+
}
|
|
3524
|
+
|
|
3525
|
+
test_procevent_surface_crash_boundaries() {
|
|
3526
|
+
local dir state out fifo pid reader marker exit_status replay_err sequence generation
|
|
3527
|
+
dir=$(make_case procevent-output-fail); state="$dir/state"; out="$dir/watch.out"; fifo="$dir/output.fifo"
|
|
3528
|
+
append_wake "$state" check "procevent:output-fail:1" "check: procevent fixture output-fail 1"
|
|
3529
|
+
mkfifo "$fifo"
|
|
3530
|
+
sh -c ': < "$1"' _ "$fifo" & reader=$!
|
|
3531
|
+
PATH="$dir/fakebin:$PATH" FM_HOME="$dir" FM_PROCEVENT_CLAIM_ROOT="$dir/claims" \
|
|
3532
|
+
FM_CREW_STATE_BIN="$dir/fakebin/fm-crew-state.sh" FM_POLL=0.2 FM_SIGNAL_GRACE=1 \
|
|
3533
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$fifo" &
|
|
3534
|
+
pid=$!
|
|
3535
|
+
wait "$reader" || true
|
|
3536
|
+
wait_for_exit "$pid" 100
|
|
3537
|
+
exit_status=$?
|
|
3538
|
+
[ "$exit_status" -ne 124 ] || fail "the watcher survived a failed actionable output write"
|
|
3539
|
+
marker=$(find "$state" -maxdepth 1 -name '.seen-procevent-*' -type f | head -1)
|
|
3540
|
+
[ -z "$marker" ] || fail "failed output committed a suppression marker"
|
|
3541
|
+
[ -s "$state/.wake-queue" ] || fail "failed output consumed the durable queue record"
|
|
3542
|
+
procevent_watch_bg "$dir" "$out"; pid=$!
|
|
3543
|
+
wait_for_exit "$pid" 100 || fail "the record was not replayable after output failure"
|
|
3544
|
+
grep -F "procevent:output-fail:1" "$out" >/dev/null || fail "output failure lost proactive replay"
|
|
3545
|
+
|
|
3546
|
+
dir=$(make_case procevent-before-marker); state="$dir/state"; out="$dir/watch.out"
|
|
3547
|
+
append_wake "$state" check "procevent:before-marker:1" "check: procevent fixture before-marker 1"
|
|
3548
|
+
install_marker_mv_fault "$dir"
|
|
3549
|
+
FM_MARKER_MV_MODE=kill-before procevent_watch_bg "$dir" "$out"; pid=$!
|
|
3550
|
+
wait_for_exit "$pid" 100
|
|
3551
|
+
exit_status=$?
|
|
3552
|
+
[ "$exit_status" -ne 124 ] || fail "the watcher survived the injected pre-marker crash"
|
|
3553
|
+
grep -F "procevent:before-marker:1" "$out" >/dev/null || fail "the pre-marker crash happened before output"
|
|
3554
|
+
marker=$(find "$state" -maxdepth 1 -name '.seen-procevent-*' -type f | head -1)
|
|
3555
|
+
[ -z "$marker" ] || fail "a pre-marker crash committed suppression"
|
|
3556
|
+
procevent_watch_bg "$dir" "$out.replay"; pid=$!
|
|
3557
|
+
wait_for_exit "$pid" 100 || fail "a pre-marker crash was not replayable"
|
|
3558
|
+
|
|
3559
|
+
dir=$(make_case procevent-after-marker); state="$dir/state"; out="$dir/watch.out"
|
|
3560
|
+
append_wake "$state" check "procevent:after-marker:1" "check: procevent fixture after-marker 1"
|
|
3561
|
+
install_marker_mv_fault "$dir"
|
|
3562
|
+
FM_MARKER_MV_MODE=kill-after procevent_watch_bg "$dir" "$out"; pid=$!
|
|
3563
|
+
wait_for_exit "$pid" 100
|
|
3564
|
+
exit_status=$?
|
|
3565
|
+
[ "$exit_status" -ne 124 ] || fail "the watcher survived the injected post-marker crash"
|
|
3566
|
+
grep -F "procevent:after-marker:1" "$out" >/dev/null || fail "the post-marker crash lost actionable output"
|
|
3567
|
+
marker=$(find "$state" -maxdepth 1 -name '.seen-procevent-*' -type f | head -1)
|
|
3568
|
+
[ -n "$marker" ] || fail "the post-marker crash did not reach marker commit"
|
|
3569
|
+
: > "$out.replay"
|
|
3570
|
+
procevent_watch_bg "$dir" "$out.replay"; pid=$!
|
|
3571
|
+
wait_for_exit "$pid" 100 \
|
|
3572
|
+
|| fail "an unacknowledged delivered record was not re-surfaced on re-arm: $(cat "$out.replay")"
|
|
3573
|
+
grep -F 'check: rearm-resurface' "$out.replay" >/dev/null \
|
|
3574
|
+
|| fail "the successor did not recover the delivered-but-unacknowledged record: $(cat "$out.replay")"
|
|
3575
|
+
replay_err="$out.replay.err"
|
|
3576
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$out.replay.drain" 2> "$replay_err" \
|
|
3577
|
+
|| fail "post-marker successor drain failed"
|
|
3578
|
+
grep "$(printf '\tcheck\t')" "$out.replay.drain" | grep -F 'procevent fixture after-marker 1' >/dev/null \
|
|
3579
|
+
|| fail "post-marker successor did not re-drain the durable record"
|
|
3580
|
+
sequence=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through \([0-9][0-9]*\) --recovery-generation [A-Za-z0-9._-][A-Za-z0-9._-]*$/\1/p' "$replay_err")
|
|
3581
|
+
generation=$(sed -n 's/^WAKE_ACK_REQUIRED:.*--ack-through [0-9][0-9]* --recovery-generation \([A-Za-z0-9._-][A-Za-z0-9._-]*\)$/\1/p' "$replay_err")
|
|
3582
|
+
[ -n "$sequence" ] && [ -n "$generation" ] \
|
|
3583
|
+
|| fail "post-marker replay omitted its post-handling acknowledgement boundary"
|
|
3584
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" --ack-through "$sequence" --recovery-generation "$generation" \
|
|
3585
|
+
|| fail "post-marker replay acknowledgement failed"
|
|
3586
|
+
[ ! -s "$state/.wake-queue" ] || fail "post-marker acknowledgement left the durable record queued"
|
|
3587
|
+
pass "surfacing failures replay until post-handling acknowledgement"
|
|
3588
|
+
}
|
|
3589
|
+
|
|
3590
|
+
test_procevent_marker_failure_exits_and_replays() {
|
|
3591
|
+
local dir state out pid marker output_count
|
|
3592
|
+
dir=$(make_case procevent-marker-failure); state="$dir/state"; out="$dir/watch.out"
|
|
3593
|
+
append_wake "$state" check "procevent:marker-failure:1" "check: procevent fixture marker-failure 1"
|
|
3594
|
+
install_marker_mv_fault "$dir"
|
|
3595
|
+
FM_MARKER_MV_MODE=fail procevent_watch_bg "$dir" "$out"
|
|
3596
|
+
pid=$!
|
|
3597
|
+
wait_for_exit "$pid" 100 || fail "marker failure did not end the actionable watcher cycle successfully"
|
|
3598
|
+
output_count=$(grep -Fc "procevent:marker-failure:1" "$out" || true)
|
|
3599
|
+
[ "$output_count" = 1 ] || fail "marker failure printed the actionable reason $output_count times"
|
|
3600
|
+
marker=$(find "$state" -maxdepth 1 -name '.seen-procevent-*' -type f | head -1)
|
|
3601
|
+
[ -z "$marker" ] || fail "marker failure committed suppression"
|
|
3602
|
+
[ ! -e "$state/.wake-queue.lock" ] && [ ! -L "$state/.wake-queue.lock" ] \
|
|
3603
|
+
|| fail "marker failure left the queue lock held"
|
|
3604
|
+
procevent_watch_bg "$dir" "$out.replay"
|
|
3605
|
+
pid=$!
|
|
3606
|
+
wait_for_exit "$pid" 100 || fail "marker failure did not leave the durable record replayable"
|
|
3607
|
+
grep -F "procevent:marker-failure:1" "$out.replay" >/dev/null \
|
|
3608
|
+
|| fail "marker failure lost the later proactive replay"
|
|
3609
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" >/dev/null 2>&1 || fail "marker-failure fixture drain failed"
|
|
3610
|
+
pass "marker failure exits through the shared wake owner, releases its lock, and replays later"
|
|
3611
|
+
}
|
|
3612
|
+
|
|
3613
|
+
# --- heartbeat: no-change absorbed, backstop surfaces a missed status --------
|
|
3614
|
+
|
|
3615
|
+
test_heartbeat_no_change_absorbed() {
|
|
3616
|
+
local dir state fakebin out pid i sig
|
|
3617
|
+
dir=$(make_case heartbeat-absorb); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out"
|
|
3618
|
+
printf 'working: routine heartbeat history\n' > "$state/routine.status"
|
|
3619
|
+
sig=$(seen_sig "$state/routine.status"); printf '%s' "$sig" > "$state/.seen-routine_status"
|
|
3620
|
+
# A quiet fleet with a fast heartbeat cadence.
|
|
3621
|
+
PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3622
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" &
|
|
3623
|
+
pid=$!
|
|
3624
|
+
if ! wait_poll_cycle "$state" "$pid"; then
|
|
3625
|
+
reap "$pid"; fail "watcher exited for a no-change heartbeat (should absorb): $(cat "$out")"
|
|
3626
|
+
fi
|
|
3627
|
+
# The heartbeat fires on the first poll whose .last-heartbeat has aged past
|
|
3628
|
+
# FM_HEARTBEAT, which need not be the first completed cycle, so wait for the
|
|
3629
|
+
# absorbed heartbeat itself rather than assuming one cycle produced it.
|
|
3630
|
+
i=0
|
|
3631
|
+
while [ "$i" -lt 200 ]; do
|
|
3632
|
+
[ "$(cat "$state/.heartbeat-streak" 2>/dev/null || echo 0)" -ge 1 ] && break
|
|
3633
|
+
kill -0 "$pid" 2>/dev/null || break
|
|
3634
|
+
sleep 0.1
|
|
3635
|
+
i=$((i + 1))
|
|
3636
|
+
done
|
|
3637
|
+
[ ! -s "$out" ] || fail "no-change heartbeat printed a wake reason: $(cat "$out")"
|
|
3638
|
+
[ ! -s "$state/.wake-queue" ] || fail "no-change heartbeat enqueued a durable wake record"
|
|
3639
|
+
[ "$(cat "$state/.heartbeat-streak" 2>/dev/null || echo 0)" -ge 1 ] || fail "heartbeat backoff streak did not advance while absorbing"
|
|
3640
|
+
[ "$(status_presentation_marker_offset "$state/.hb-surfaced-routine" "$state/routine.status")" = \
|
|
3641
|
+
"$(size_of "$state/routine.status")" ] \
|
|
3642
|
+
|| fail "routine heartbeat classification did not commit its captured endpoint"
|
|
3643
|
+
reap "$pid"
|
|
3644
|
+
pass "a heartbeat with no captain-relevant change is absorbed and backs off the cadence"
|
|
3645
|
+
}
|
|
3646
|
+
|
|
3647
|
+
test_heartbeat_backstop_surfaces_a_masked_status() {
|
|
3648
|
+
local dir state fakebin out sig pid
|
|
3649
|
+
dir=$(make_case heartbeat-masked); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3650
|
+
out="$dir/watch.out"
|
|
3651
|
+
# Same miss as below, but the captain-relevant event is followed by a routine
|
|
3652
|
+
# append, so its last line reads benign. The backstop must still catch it.
|
|
3653
|
+
printf 'working: setup\nneeds-decision: pick A or B\nworking: tidying the branch\n' \
|
|
3654
|
+
> "$state/miss.status"
|
|
3655
|
+
sig=$(seen_sig "$state/miss.status"); printf '%s' "$sig" > "$state/.seen-miss_status"
|
|
3656
|
+
PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3657
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" &
|
|
3658
|
+
pid=$!
|
|
3659
|
+
wait_for_exit "$pid" 100 \
|
|
3660
|
+
|| fail "heartbeat backstop missed a decision hidden behind a later working: line"
|
|
3661
|
+
grep -Fx "heartbeat" "$out" >/dev/null || fail "backstop did not exit with a heartbeat wake"
|
|
3662
|
+
[ "$(status_presentation_marker_offset "$state/.hb-surfaced-miss" "$state/miss.status")" = \
|
|
3663
|
+
"$(size_of "$state/miss.status")" ] \
|
|
3664
|
+
|| fail "backstop did not record the masked status as surfaced through its end"
|
|
3665
|
+
pass "the heartbeat backstop surfaces a captain event hidden behind a later routine append"
|
|
3666
|
+
}
|
|
3667
|
+
|
|
3668
|
+
test_heartbeat_backstop_surfaces_unsurfaced_status() {
|
|
3669
|
+
local dir state fakebin out drain_out sig pid
|
|
3670
|
+
dir=$(make_case heartbeat-backstop); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3671
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
3672
|
+
# A captain-relevant status whose .seen-* signature ALREADY matches (so the
|
|
3673
|
+
# per-poll signal scan stays quiet) but which was never surfaced (no
|
|
3674
|
+
# .hb-surfaced-* marker). This stands in for a per-wake-path miss; the heartbeat
|
|
3675
|
+
# fleet-scan backstop must catch it and wake firstmate.
|
|
3676
|
+
printf 'done: PR https://example.test/pr/5\n' > "$state/miss.status"
|
|
3677
|
+
sig=$(seen_sig "$state/miss.status"); printf '%s' "$sig" > "$state/.seen-miss_status"
|
|
3678
|
+
PATH="$fakebin:$PATH" FM_STATE_OVERRIDE="$state" FM_POLL=1 FM_SIGNAL_GRACE=1 \
|
|
3679
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=1 "$WATCH" > "$out" &
|
|
3680
|
+
pid=$!
|
|
3681
|
+
wait_for_exit "$pid" 100 || fail "heartbeat backstop did not surface an unsurfaced captain-relevant status"
|
|
3682
|
+
grep -Fx "heartbeat" "$out" >/dev/null || fail "backstop did not exit with a heartbeat wake"
|
|
3683
|
+
[ "$(status_presentation_marker_offset "$state/.hb-surfaced-miss" "$state/miss.status")" = \
|
|
3684
|
+
"$(size_of "$state/miss.status")" ] \
|
|
3685
|
+
|| fail "backstop did not record the status as surfaced through its end (would re-fire next heartbeat)"
|
|
3686
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the backstop heartbeat failed"
|
|
3687
|
+
grep "$(printf '\theartbeat\t')" "$drain_out" >/dev/null || fail "backstop heartbeat was not queued"
|
|
3688
|
+
pass "heartbeat backstop fail-safe surfaces a captain-relevant status the per-wake path missed"
|
|
3689
|
+
}
|
|
3690
|
+
|
|
3691
|
+
# --- beacon stays fresh while absorbing -------------------------------------
|
|
3692
|
+
|
|
3693
|
+
test_beacon_stays_fresh_while_absorbing() {
|
|
3694
|
+
local dir state fakebin out status_file pid m1 m2 now
|
|
3695
|
+
dir=$(make_case beacon-fresh); state="$dir/state"; fakebin="$dir/fakebin"; out="$dir/watch.out"
|
|
3696
|
+
status_file="$state/task.status"
|
|
3697
|
+
printf 'working: a\n' > "$status_file"
|
|
3698
|
+
# Provably working so the working: notes are absorbed (the path that must keep the
|
|
3699
|
+
# beacon fresh).
|
|
3700
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
3701
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
3702
|
+
pid=$!
|
|
3703
|
+
# Wait on the beacon itself rather than a fixed liveness budget: the watcher's
|
|
3704
|
+
# bounded startup can outlast a short wait, and reading an absent beacon would
|
|
3705
|
+
# report a missing beacon that simply had not been written yet.
|
|
3706
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "watcher exited while absorbing the first benign signal"; }
|
|
3707
|
+
m1=$(file_mtime "$state/.last-watcher-beat")
|
|
3708
|
+
# A second benign signal keeps it absorbing; the beacon must keep advancing.
|
|
3709
|
+
printf 'working: b\n' >> "$status_file"
|
|
3710
|
+
wait_poll_cycle "$state" "$pid" || { reap "$pid"; fail "watcher exited while absorbing a second benign signal"; }
|
|
3711
|
+
m2=$(file_mtime "$state/.last-watcher-beat")
|
|
3712
|
+
now=$(date +%s)
|
|
3713
|
+
if [ -z "$m1" ] || [ -z "$m2" ]; then
|
|
3714
|
+
reap "$pid"
|
|
3715
|
+
fail "watcher beacon missing while absorbing"
|
|
3716
|
+
fi
|
|
3717
|
+
[ "$m2" -ge "$m1" ] || { reap "$pid"; fail "beacon mtime regressed while absorbing"; }
|
|
3718
|
+
[ "$(( now - m2 ))" -lt 10 ] || { reap "$pid"; fail "beacon went stale while absorbing (age $(( now - m2 ))s)"; }
|
|
3719
|
+
[ ! -s "$state/.wake-queue" ] || { reap "$pid"; fail "absorbing benign signals enqueued a wake"; }
|
|
3720
|
+
reap "$pid"
|
|
3721
|
+
pass "the liveness beacon stays fresh while the watcher absorbs benign wakes (fm-guard never false-alarms)"
|
|
3722
|
+
}
|
|
3723
|
+
|
|
3724
|
+
# --- afk coherence: the daemon owns triage; the watcher does not double-triage ---
|
|
3725
|
+
|
|
3726
|
+
test_afk_signal_records_heartbeat_endpoint() {
|
|
3727
|
+
local dir state fakebin out status_file pid
|
|
3728
|
+
dir=$(make_case afk-heartbeat-endpoint); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3729
|
+
out="$dir/watch.out"; status_file="$state/task.status"
|
|
3730
|
+
printf 'needs-decision: choose release target\nworking: preparing both targets\n' > "$status_file"
|
|
3731
|
+
date '+%s' > "$state/.afk"
|
|
3732
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
3733
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
3734
|
+
pid=$!
|
|
3735
|
+
wait_for_exit "$pid" 100 || fail "afk watcher did not hand the actionable signal to the daemon"
|
|
3736
|
+
[ "$(status_presentation_marker_offset "$state/.hb-surfaced-task" "$status_file")" = \
|
|
3737
|
+
"$(size_of "$status_file")" ] \
|
|
3738
|
+
|| fail "afk signal did not record the endpoint handed to the daemon"
|
|
3739
|
+
unset FM_FAKE_CREW_STATE
|
|
3740
|
+
pass "an afk signal records its captured heartbeat endpoint"
|
|
3741
|
+
}
|
|
3742
|
+
|
|
3743
|
+
test_afk_present_reverts_watcher_to_one_shot() {
|
|
3744
|
+
local dir state fakebin out drain_out status_file pid
|
|
3745
|
+
dir=$(make_case afk-coherence); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3746
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"
|
|
3747
|
+
status_file="$state/task.status"
|
|
3748
|
+
printf 'working: routine note\n' > "$status_file"
|
|
3749
|
+
date '+%s' > "$state/.afk" # away mode: the supervise-daemon owns triage
|
|
3750
|
+
# Set a PROVABLY-WORKING verdict: if afk failed to bypass the provably-working
|
|
3751
|
+
# check, this no-verb signal would be absorbed (not surfaced). The test asserting
|
|
3752
|
+
# a surface therefore also proves afk reverts to one-shot and skips the costly read.
|
|
3753
|
+
export FM_FAKE_CREW_STATE='state: working · source: run-step · validating (running)'
|
|
3754
|
+
watch_bg "$state" "$fakebin" "$out"
|
|
3755
|
+
pid=$!
|
|
3756
|
+
wait_for_exit "$pid" 100 || fail "with .afk present the watcher did not exit one-shot for a benign signal"
|
|
3757
|
+
grep -F "signal: $status_file" "$out" >/dev/null || fail "afk-mode watcher did not surface the signal for the daemon"
|
|
3758
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after the afk-mode signal failed"
|
|
3759
|
+
grep "$(printf '\tsignal\t')" "$drain_out" | grep -F "$status_file" >/dev/null \
|
|
3760
|
+
|| fail "afk-mode benign signal was not queued for the daemon to classify"
|
|
3761
|
+
pass "with .afk present the watcher reverts to one-shot so the daemon owns triage (no double-triage)"
|
|
3762
|
+
}
|
|
3763
|
+
|
|
3764
|
+
# A paused pane can first appear as a changed hash. In AFK mode that initial path
|
|
3765
|
+
# must still hand off the plain window identity to the daemon, rather than running
|
|
3766
|
+
# the normal-mode pause re-surface and decorating the stale identity.
|
|
3767
|
+
test_afk_paused_changed_pane_hands_off_plain_stale() {
|
|
3768
|
+
local dir state fakebin out drain_out capture_file statusf window key sig pid back
|
|
3769
|
+
dir=$(make_case afk-paused-changed-pane); state="$dir/state"; fakebin="$dir/fakebin"
|
|
3770
|
+
out="$dir/watch.out"; drain_out="$dir/drain.out"; capture_file="$dir/pane.txt"
|
|
3771
|
+
window="test:fm-afk-held"
|
|
3772
|
+
printf 'idle, awaiting upstream\n' > "$capture_file"
|
|
3773
|
+
printf 'window=%s\nkind=ship\n' "$window" > "$state/afk-held.meta"
|
|
3774
|
+
statusf="$state/afk-held.status"
|
|
3775
|
+
printf 'paused: awaiting the upstream tool release\n' > "$statusf"
|
|
3776
|
+
back=$(( $(date +%s) - 500 ))
|
|
3777
|
+
if [ "$(uname)" = Darwin ]; then touch -mt "$(date -r "$back" '+%Y%m%d%H%M.%S')" "$statusf"
|
|
3778
|
+
else touch -m -d "@$back" "$statusf"; fi
|
|
3779
|
+
sig=$(seen_sig "$statusf"); printf '%s' "$sig" > "$state/.seen-afk-held_status"
|
|
3780
|
+
date '+%s' > "$state/.afk"
|
|
3781
|
+
key=$(printf '%s' "$window" | tr '.:/' '___')
|
|
3782
|
+
|
|
3783
|
+
# Deliberately do not seed .hash-*: this is the changed-pane path that used to
|
|
3784
|
+
# call handle_paused_stale before AFK's one-shot daemon handoff.
|
|
3785
|
+
PATH="$fakebin:$PATH" FM_FAKE_TMUX_WINDOW="$window" FM_FAKE_TMUX_CAPTURE="$capture_file" \
|
|
3786
|
+
FM_FAKE_CREW_STATE='state: paused · source: status-log · awaiting the upstream tool release' \
|
|
3787
|
+
FM_STATE_OVERRIDE="$state" FM_CREW_STATE_BIN="$fakebin/fm-crew-state.sh" FM_PAUSE_RESURFACE_SECS=240 FM_POLL=0.2 FM_SIGNAL_GRACE=1 \
|
|
3788
|
+
FM_CHECK_INTERVAL=999999 FM_HEARTBEAT=999999 "$WATCH" > "$out" &
|
|
3789
|
+
pid=$!
|
|
3790
|
+
wait_for_exit "$pid" 100 || fail "AFK paused changed pane did not hand off a stale wake"
|
|
3791
|
+
grep -Fx "stale: $window" "$out" >/dev/null || fail "AFK paused stale did not preserve its plain window identity: $(cat "$out")"
|
|
3792
|
+
grep -F "awaiting external" "$out" >/dev/null && fail "AFK watcher decorated a stale identity instead of handing it to the daemon"
|
|
3793
|
+
[ ! -e "$state/.paused-$key" ] || fail "AFK watcher recorded normal-mode pause tracking instead of handing off"
|
|
3794
|
+
FM_STATE_OVERRIDE="$state" "$DRAIN" > "$drain_out" 2>/dev/null || fail "drain after AFK paused stale failed"
|
|
3795
|
+
grep "$(printf '\tstale\t')" "$drain_out" | grep -F "stale: $window" >/dev/null \
|
|
3796
|
+
|| fail "AFK paused stale was not queued with the plain window identity"
|
|
3797
|
+
pass "AFK changed paused panes hand off plain stale identities for daemon-owned pause triage"
|
|
3798
|
+
}
|
|
3799
|
+
|
|
3800
|
+
test_status_span_actionable_classifier
|
|
3801
|
+
test_status_span_survives_a_later_routine_append
|
|
3802
|
+
test_status_span_respects_decision_closure
|
|
3803
|
+
test_malformed_seen_signature_reads_the_whole_log
|
|
3804
|
+
test_stale_is_terminal_classifier
|
|
3805
|
+
test_classifier_primitives
|
|
3806
|
+
test_crew_is_provably_working_classifier
|
|
3807
|
+
test_status_is_paused_classifier
|
|
3808
|
+
test_crew_absorb_class_classifier
|
|
3809
|
+
test_crew_worktree_written_since_classifier
|
|
3810
|
+
test_empty_write_prune_widens_the_probe
|
|
3811
|
+
test_empty_write_prune_from_the_environment_widens_the_probe
|
|
3812
|
+
test_worktree_write_probe_is_wall_clock_bounded
|
|
3813
|
+
test_signal_crew_provably_working_classifier
|
|
3814
|
+
test_secondmate_status_signal_never_absorbed_classifier
|
|
3815
|
+
test_provably_working_signal_absorbed
|
|
3816
|
+
test_turn_ended_provably_working_absorbed
|
|
3817
|
+
test_turn_ended_not_working_surfaced
|
|
3818
|
+
test_turn_ended_churning_pane_absorbed
|
|
3819
|
+
test_turn_ended_churn_resets_prior_stale_classification
|
|
3820
|
+
test_turn_ended_churn_resets_wedge_state_before_stale_poll
|
|
3821
|
+
test_turn_ended_still_pane_surfaced
|
|
3822
|
+
test_turn_ended_malformed_prior_hash_surfaced
|
|
3823
|
+
test_turn_ended_trailing_newline_prior_hash_surfaced
|
|
3824
|
+
test_secondmate_turn_ended_churning_pane_surfaced
|
|
3825
|
+
test_turn_ended_colliding_window_key_surfaced
|
|
3826
|
+
test_turn_ended_duplicate_endpoint_records_surfaced
|
|
3827
|
+
test_turn_ended_mixed_positive_evidence_batch_absorbed
|
|
3828
|
+
test_turn_ended_mixed_positive_evidence_batch_default_off
|
|
3829
|
+
test_status_and_turn_end_batch_never_uses_churn_evidence
|
|
3830
|
+
test_turn_ended_churn_absorb_off_by_default
|
|
3831
|
+
test_turn_ended_churn_absorb_bounded
|
|
3832
|
+
test_turn_ended_churn_timer_write_failure_surfaced
|
|
3833
|
+
test_turn_ended_invalid_churn_bound_surfaced
|
|
3834
|
+
test_turn_ended_oversized_churn_bound_surfaced
|
|
3835
|
+
test_turn_ended_invalid_churn_deadline_surfaced
|
|
3836
|
+
test_turn_ended_surfaced_batch_opens_no_partial_deadline
|
|
3837
|
+
test_working_note_not_working_surfaced
|
|
3838
|
+
test_secondmate_status_note_surfaced_despite_busy_agent
|
|
3839
|
+
test_self_announced_close_does_not_rewake_but_next_note_does
|
|
3840
|
+
test_actionable_signal_surfaced
|
|
3841
|
+
test_actionable_signal_survives_a_later_routine_append
|
|
3842
|
+
test_release_completion_survives_a_later_routine_append
|
|
3843
|
+
test_routine_appends_after_a_classified_event_stay_absorbed
|
|
3844
|
+
test_unreadable_status_reports_once_per_file_state
|
|
3845
|
+
test_permission_recovery_surfaces_preserved_status
|
|
3846
|
+
test_terminal_stale_surfaced
|
|
3847
|
+
test_stale_terminal_status_overridden_by_active_run
|
|
3848
|
+
test_nonterminal_stale_provably_working_absorbed_then_escalated
|
|
3849
|
+
test_wedge_escalation_marks_demand_deep_inspection_after_threshold
|
|
3850
|
+
test_wedge_escalation_resets_when_pane_becomes_active
|
|
3851
|
+
test_busy_pane_below_turn_age_bound_is_absorbed
|
|
3852
|
+
test_busy_pane_stable_hash_escalates_past_turn_age_bound
|
|
3853
|
+
test_busy_pane_changing_hash_escalates_past_turn_age_bound
|
|
3854
|
+
test_busy_pane_turn_end_touch_resets_age
|
|
3855
|
+
test_busy_pane_repeated_escalation_reaches_demand_deep_inspection
|
|
3856
|
+
test_busy_pane_default_turn_age_bound_is_3600s
|
|
3857
|
+
test_busy_declared_pause_is_rechecked_not_wedge_escalated
|
|
3858
|
+
test_afk_busy_declared_pause_hands_off_plain_stale
|
|
3859
|
+
test_afk_busy_declared_pause_ticking_pane_hands_off_once
|
|
3860
|
+
test_nonterminal_stale_not_working_surfaced
|
|
3861
|
+
test_nonterminal_stale_paused_absorbed_then_resurfaced
|
|
3862
|
+
test_exited_declared_pause_is_bounded_but_live_gate_surfaces
|
|
3863
|
+
test_secondmate_paused_resurfaces_in_normal_mode
|
|
3864
|
+
test_secondmate_captain_held_resurfaces_in_normal_mode
|
|
3865
|
+
test_secondmate_nonpaused_stale_remains_suppressed
|
|
3866
|
+
test_secondmate_unpause_clears_pause_tracking
|
|
3867
|
+
test_nonterminal_stale_pause_transitions_reclassify_unchanged_hash
|
|
3868
|
+
test_nonterminal_paused_rechecks_authoritative_state
|
|
3869
|
+
test_paused_authoritative_working_preserves_wedge_timer
|
|
3870
|
+
test_nonterminal_stale_repairs_missing_or_corrupt_timer
|
|
3871
|
+
test_wedge_escalation_deferred_while_worktree_is_written
|
|
3872
|
+
test_write_deferral_resurfaces_on_the_bounded_cadence
|
|
3873
|
+
test_secondmate_home_supervision_churn_is_not_write_evidence
|
|
3874
|
+
test_timer_repair_drops_a_finished_write_deferral_chain
|
|
3875
|
+
test_terminal_first_sight_drops_a_finished_write_deferral_chain
|
|
3876
|
+
test_triage_log_size_cap_accepts_spaced_wc_counts
|
|
3877
|
+
test_procevent_captured_result_surfaces_proactively
|
|
3878
|
+
test_procevent_unacknowledged_result_redrains_until_handled
|
|
3879
|
+
test_procevent_marker_keys_are_injective
|
|
3880
|
+
test_procevent_surface_serializes_with_drain
|
|
3881
|
+
test_procevent_surface_crash_boundaries
|
|
3882
|
+
test_procevent_marker_failure_exits_and_replays
|
|
3883
|
+
test_heartbeat_no_change_absorbed
|
|
3884
|
+
test_heartbeat_backstop_surfaces_unsurfaced_status
|
|
3885
|
+
test_heartbeat_backstop_surfaces_a_masked_status
|
|
3886
|
+
test_beacon_stays_fresh_while_absorbing
|
|
3887
|
+
test_afk_signal_records_heartbeat_endpoint
|
|
3888
|
+
test_afk_present_reverts_watcher_to_one_shot
|
|
3889
|
+
test_afk_paused_changed_pane_hands_off_plain_stale
|